From 300e80f1f8922c6e373858789530048b903da11c Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Wed, 12 Mar 2025 14:15:15 +0000 Subject: [PATCH 01/72] add prebuilds system user database changes and associated changes Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz.go | 28 ++++++++++ coderd/database/dump.sql | 20 ++++++++ .../migrations/000301_system_user.down.sql | 20 ++++++++ .../migrations/000301_system_user.up.sql | 51 +++++++++++++++++++ coderd/database/modelmethods.go | 1 + coderd/database/modelqueries.go | 1 + coderd/database/models.go | 2 + coderd/database/queries.sql.go | 34 +++++++++---- coderd/prebuilds/id.go | 5 ++ docs/admin/security/audit-logs.md | 2 +- enterprise/audit/table.go | 1 + enterprise/coderd/groups_test.go | 17 ++++++- enterprise/coderd/roles_test.go | 8 +++ enterprise/coderd/templates_test.go | 9 +++- 14 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 coderd/database/migrations/000301_system_user.down.sql create mode 100644 coderd/database/migrations/000301_system_user.up.sql create mode 100644 coderd/prebuilds/id.go diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 9c88e986cbffc..796127c83259b 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -18,6 +18,7 @@ import ( "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/rbac/rolestore" @@ -358,6 +359,27 @@ var ( }), Scope: rbac.ScopeAll, }.WithCachedASTValue() + + subjectPrebuildsOrchestrator = rbac.Subject{ + FriendlyName: "Prebuilds Orchestrator", + ID: prebuilds.OwnerID.String(), + Roles: rbac.Roles([]rbac.Role{ + { + Identifier: rbac.RoleIdentifier{Name: "prebuilds-orchestrator"}, + DisplayName: "Coder", + Site: rbac.Permissions(map[string][]policy.Action{ + // May use template, read template-related info, & insert template-related resources (preset prebuilds). + rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionUse}, + // May CRUD workspaces, and start/stop them. + rbac.ResourceWorkspace.Type: { + policy.ActionCreate, policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, + policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, + }, + }), + }, + }), + Scope: rbac.ScopeAll, + }.WithCachedASTValue() ) // AsProvisionerd returns a context with an actor that has permissions required @@ -412,6 +434,12 @@ func AsSystemReadProvisionerDaemons(ctx context.Context) context.Context { return context.WithValue(ctx, authContextKey{}, subjectSystemReadProvisionerDaemons) } +// AsPrebuildsOrchestrator returns a context with an actor that has permissions +// to read orchestrator workspace prebuilds. +func AsPrebuildsOrchestrator(ctx context.Context) context.Context { + return context.WithValue(ctx, authContextKey{}, subjectPrebuildsOrchestrator) +} + var AsRemoveActor = rbac.Subject{ ID: "remove-actor", } diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 492aaefc12aa5..6961b1386e176 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -445,6 +445,17 @@ BEGIN END; $$; +CREATE FUNCTION prevent_system_user_changes() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF OLD.is_system = true THEN + RAISE EXCEPTION 'Cannot modify or delete system users'; + END IF; + RETURN OLD; +END; +$$; + CREATE FUNCTION protect_deleting_organizations() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -854,6 +865,7 @@ CREATE TABLE users ( github_com_user_id bigint, hashed_one_time_passcode bytea, one_time_passcode_expires_at timestamp with time zone, + is_system boolean DEFAULT false, CONSTRAINT one_time_passcode_set CHECK ((((hashed_one_time_passcode IS NULL) AND (one_time_passcode_expires_at IS NULL)) OR ((hashed_one_time_passcode IS NOT NULL) AND (one_time_passcode_expires_at IS NOT NULL)))) ); @@ -867,6 +879,8 @@ COMMENT ON COLUMN users.hashed_one_time_passcode IS 'A hash of the one-time-pass COMMENT ON COLUMN users.one_time_passcode_expires_at IS 'The time when the one-time-passcode expires.'; +COMMENT ON COLUMN users.is_system IS 'Determines if a user is a system user, and therefore cannot login or perform normal actions'; + CREATE VIEW group_members_expanded AS WITH all_members AS ( SELECT group_members.user_id, @@ -2362,6 +2376,8 @@ COMMENT ON INDEX template_usage_stats_start_time_template_id_user_id_idx IS 'Ind CREATE UNIQUE INDEX templates_organization_id_name_idx ON templates USING btree (organization_id, lower((name)::text)) WHERE (deleted = false); +CREATE INDEX user_is_system_idx ON users USING btree (is_system); + CREATE UNIQUE INDEX user_links_linked_id_login_type_idx ON user_links USING btree (linked_id, login_type) WHERE (linked_id <> ''::text); CREATE UNIQUE INDEX users_email_lower_idx ON users USING btree (lower(email)) WHERE (deleted = false); @@ -2450,6 +2466,10 @@ CREATE OR REPLACE VIEW provisioner_job_stats AS CREATE TRIGGER inhibit_enqueue_if_disabled BEFORE INSERT ON notification_messages FOR EACH ROW EXECUTE FUNCTION inhibit_enqueue_if_disabled(); +CREATE TRIGGER prevent_system_user_deletions BEFORE DELETE ON users FOR EACH ROW WHEN ((old.is_system = true)) EXECUTE FUNCTION prevent_system_user_changes(); + +CREATE TRIGGER prevent_system_user_updates BEFORE UPDATE ON users FOR EACH ROW WHEN ((old.is_system = true)) EXECUTE FUNCTION prevent_system_user_changes(); + CREATE TRIGGER protect_deleting_organizations BEFORE UPDATE ON organizations FOR EACH ROW WHEN (((new.deleted = true) AND (old.deleted = false))) EXECUTE FUNCTION protect_deleting_organizations(); CREATE TRIGGER remove_organization_member_custom_role BEFORE DELETE ON custom_roles FOR EACH ROW EXECUTE FUNCTION remove_organization_member_role(); diff --git a/coderd/database/migrations/000301_system_user.down.sql b/coderd/database/migrations/000301_system_user.down.sql new file mode 100644 index 0000000000000..1117ddb2f2513 --- /dev/null +++ b/coderd/database/migrations/000301_system_user.down.sql @@ -0,0 +1,20 @@ +-- Remove system user from organizations +DELETE FROM organization_members +WHERE user_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; + +-- Drop triggers first +DROP TRIGGER IF EXISTS prevent_system_user_updates ON users; +DROP TRIGGER IF EXISTS prevent_system_user_deletions ON users; + +-- Drop function +DROP FUNCTION IF EXISTS prevent_system_user_changes(); + +-- Delete system user +DELETE FROM users +WHERE id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; + +-- Drop index +DROP INDEX IF EXISTS user_is_system_idx; + +-- Drop column +ALTER TABLE users DROP COLUMN IF EXISTS is_system; diff --git a/coderd/database/migrations/000301_system_user.up.sql b/coderd/database/migrations/000301_system_user.up.sql new file mode 100644 index 0000000000000..0edb25ef076d6 --- /dev/null +++ b/coderd/database/migrations/000301_system_user.up.sql @@ -0,0 +1,51 @@ +ALTER TABLE users + ADD COLUMN is_system bool DEFAULT false; + +CREATE INDEX user_is_system_idx ON users USING btree (is_system); + +COMMENT ON COLUMN users.is_system IS 'Determines if a user is a system user, and therefore cannot login or perform normal actions'; + +-- TODO: tried using "none" for login type, but the migration produced this error: 'unsafe use of new value "none" of enum type login_type' +-- -> not sure why though? it exists on the login_type enum. +INSERT INTO users (id, email, username, name, created_at, updated_at, status, rbac_roles, hashed_password, is_system, login_type) +VALUES ('c42fdf75-3097-471c-8c33-fb52454d81c0', 'prebuilds@system', 'prebuilds', 'Prebuilds Owner', now(), now(), + 'active', '{}', 'none', true, 'password'::login_type); + +-- Create function to check system user modifications +CREATE OR REPLACE FUNCTION prevent_system_user_changes() + RETURNS TRIGGER AS +$$ +BEGIN + IF OLD.is_system = true THEN + RAISE EXCEPTION 'Cannot modify or delete system users'; + END IF; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +-- Create trigger to prevent updates to system users +CREATE TRIGGER prevent_system_user_updates + BEFORE UPDATE ON users + FOR EACH ROW + WHEN (OLD.is_system = true) +EXECUTE FUNCTION prevent_system_user_changes(); + +-- Create trigger to prevent deletion of system users +CREATE TRIGGER prevent_system_user_deletions + BEFORE DELETE ON users + FOR EACH ROW + WHEN (OLD.is_system = true) +EXECUTE FUNCTION prevent_system_user_changes(); + +-- TODO: do we *want* to use the default org here? how do we handle multi-org? +WITH default_org AS (SELECT id + FROM organizations + WHERE is_default = true + LIMIT 1) +INSERT +INTO organization_members (organization_id, user_id, created_at, updated_at) +SELECT default_org.id, + 'c42fdf75-3097-471c-8c33-fb52454d81c0', -- The system user responsible for prebuilds. + NOW(), + NOW() +FROM default_org; diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index a9dbc3e530994..5b197a0649dcf 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -423,6 +423,7 @@ func ConvertUserRows(rows []GetUsersRow) []User { AvatarURL: r.AvatarURL, Deleted: r.Deleted, LastSeenAt: r.LastSeenAt, + IsSystem: r.IsSystem, } } diff --git a/coderd/database/modelqueries.go b/coderd/database/modelqueries.go index cc19de5132f37..506cd7a69fe34 100644 --- a/coderd/database/modelqueries.go +++ b/coderd/database/modelqueries.go @@ -421,6 +421,7 @@ func (q *sqlQuerier) GetAuthorizedUsers(ctx context.Context, arg GetUsersParams, &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, &i.Count, ); err != nil { return nil, err diff --git a/coderd/database/models.go b/coderd/database/models.go index e0064916b0135..8e98510389118 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3186,6 +3186,8 @@ type User struct { HashedOneTimePasscode []byte `db:"hashed_one_time_passcode" json:"hashed_one_time_passcode"` // The time when the one-time-passcode expires. OneTimePasscodeExpiresAt sql.NullTime `db:"one_time_passcode_expires_at" json:"one_time_passcode_expires_at"` + // Determines if a user is a system user, and therefore cannot login or perform normal actions + IsSystem sql.NullBool `db:"is_system" json:"is_system"` } type UserConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b394a0b0121ec..2c5138dbee4af 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -11451,7 +11451,7 @@ func (q *sqlQuerier) GetUserAppearanceSettings(ctx context.Context, userID uuid. const getUserByEmailOrUsername = `-- name: GetUserByEmailOrUsername :one SELECT - id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system FROM users WHERE @@ -11487,13 +11487,14 @@ func (q *sqlQuerier) GetUserByEmailOrUsername(ctx context.Context, arg GetUserBy &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } const getUserByID = `-- name: GetUserByID :one SELECT - id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system FROM users WHERE @@ -11523,6 +11524,7 @@ func (q *sqlQuerier) GetUserByID(ctx context.Context, id uuid.UUID) (User, error &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -11545,7 +11547,7 @@ func (q *sqlQuerier) GetUserCount(ctx context.Context) (int64, error) { const getUsers = `-- name: GetUsers :many SELECT - id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, COUNT(*) OVER() AS count + id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system, COUNT(*) OVER() AS count FROM users WHERE @@ -11659,6 +11661,7 @@ type GetUsersRow struct { GithubComUserID sql.NullInt64 `db:"github_com_user_id" json:"github_com_user_id"` HashedOneTimePasscode []byte `db:"hashed_one_time_passcode" json:"hashed_one_time_passcode"` OneTimePasscodeExpiresAt sql.NullTime `db:"one_time_passcode_expires_at" json:"one_time_passcode_expires_at"` + IsSystem sql.NullBool `db:"is_system" json:"is_system"` Count int64 `db:"count" json:"count"` } @@ -11701,6 +11704,7 @@ func (q *sqlQuerier) GetUsers(ctx context.Context, arg GetUsersParams) ([]GetUse &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, &i.Count, ); err != nil { return nil, err @@ -11717,7 +11721,7 @@ func (q *sqlQuerier) GetUsers(ctx context.Context, arg GetUsersParams) ([]GetUse } const getUsersByIDs = `-- name: GetUsersByIDs :many -SELECT id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at FROM users WHERE id = ANY($1 :: uuid [ ]) +SELECT id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system FROM users WHERE id = ANY($1 :: uuid [ ]) ` // This shouldn't check for deleted, because it's frequently used @@ -11750,6 +11754,7 @@ func (q *sqlQuerier) GetUsersByIDs(ctx context.Context, ids []uuid.UUID) ([]User &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ); err != nil { return nil, err } @@ -11783,7 +11788,7 @@ VALUES -- if the status passed in is empty, fallback to dormant, which is what -- we were doing before. COALESCE(NULLIF($10::text, '')::user_status, 'dormant'::user_status) - ) RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + ) RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type InsertUserParams struct { @@ -11831,6 +11836,7 @@ func (q *sqlQuerier) InsertUser(ctx context.Context, arg InsertUserParams) (User &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -11996,7 +12002,7 @@ SET last_seen_at = $2, updated_at = $3 WHERE - id = $1 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + id = $1 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserLastSeenAtParams struct { @@ -12026,6 +12032,7 @@ func (q *sqlQuerier) UpdateUserLastSeenAt(ctx context.Context, arg UpdateUserLas &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -12043,7 +12050,7 @@ SET '':: bytea END WHERE - id = $2 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + id = $2 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserLoginTypeParams struct { @@ -12072,6 +12079,7 @@ func (q *sqlQuerier) UpdateUserLoginType(ctx context.Context, arg UpdateUserLogi &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -12087,7 +12095,7 @@ SET name = $6 WHERE id = $1 -RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at +RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserProfileParams struct { @@ -12127,6 +12135,7 @@ func (q *sqlQuerier) UpdateUserProfile(ctx context.Context, arg UpdateUserProfil &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -12138,7 +12147,7 @@ SET quiet_hours_schedule = $2 WHERE id = $1 -RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at +RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserQuietHoursScheduleParams struct { @@ -12167,6 +12176,7 @@ func (q *sqlQuerier) UpdateUserQuietHoursSchedule(ctx context.Context, arg Updat &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -12179,7 +12189,7 @@ SET rbac_roles = ARRAY(SELECT DISTINCT UNNEST($1 :: text[])) WHERE id = $2 -RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at +RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserRolesParams struct { @@ -12208,6 +12218,7 @@ func (q *sqlQuerier) UpdateUserRoles(ctx context.Context, arg UpdateUserRolesPar &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } @@ -12219,7 +12230,7 @@ SET status = $2, updated_at = $3 WHERE - id = $1 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at + id = $1 RETURNING id, email, username, hashed_password, created_at, updated_at, status, rbac_roles, login_type, avatar_url, deleted, last_seen_at, quiet_hours_schedule, name, github_com_user_id, hashed_one_time_passcode, one_time_passcode_expires_at, is_system ` type UpdateUserStatusParams struct { @@ -12249,6 +12260,7 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP &i.GithubComUserID, &i.HashedOneTimePasscode, &i.OneTimePasscodeExpiresAt, + &i.IsSystem, ) return i, err } diff --git a/coderd/prebuilds/id.go b/coderd/prebuilds/id.go new file mode 100644 index 0000000000000..bde76e3f7bf14 --- /dev/null +++ b/coderd/prebuilds/id.go @@ -0,0 +1,5 @@ +package prebuilds + +import "github.com/google/uuid" + +var OwnerID = uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0") diff --git a/docs/admin/security/audit-logs.md b/docs/admin/security/audit-logs.md index 778e9f9c2e26e..47f3b8757a7bb 100644 --- a/docs/admin/security/audit-logs.md +++ b/docs/admin/security/audit-logs.md @@ -28,7 +28,7 @@ We track the following resources: | RoleSyncSettings
| |
FieldTracked
fieldtrue
mappingtrue
| | Template
write, delete | |
FieldTracked
active_version_idtrue
activity_bumptrue
allow_user_autostarttrue
allow_user_autostoptrue
allow_user_cancel_workspace_jobstrue
autostart_block_days_of_weektrue
autostop_requirement_days_of_weektrue
autostop_requirement_weekstrue
created_atfalse
created_bytrue
created_by_avatar_urlfalse
created_by_usernamefalse
default_ttltrue
deletedfalse
deprecatedtrue
descriptiontrue
display_nametrue
failure_ttltrue
group_acltrue
icontrue
idtrue
max_port_sharing_leveltrue
nametrue
organization_display_namefalse
organization_iconfalse
organization_idfalse
organization_namefalse
provisionertrue
require_active_versiontrue
time_til_dormanttrue
time_til_dormant_autodeletetrue
updated_atfalse
user_acltrue
| | TemplateVersion
create, write | |
FieldTracked
archivedtrue
created_atfalse
created_bytrue
created_by_avatar_urlfalse
created_by_usernamefalse
external_auth_providersfalse
idtrue
job_idfalse
messagefalse
nametrue
organization_idfalse
readmetrue
source_example_idfalse
template_idtrue
updated_atfalse
| -| User
create, write, delete | |
FieldTracked
avatar_urlfalse
created_atfalse
deletedtrue
emailtrue
github_com_user_idfalse
hashed_one_time_passcodefalse
hashed_passwordtrue
idtrue
last_seen_atfalse
login_typetrue
nametrue
one_time_passcode_expires_attrue
quiet_hours_scheduletrue
rbac_rolestrue
statustrue
updated_atfalse
usernametrue
| +| User
create, write, delete | |
FieldTracked
avatar_urlfalse
created_atfalse
deletedtrue
emailtrue
github_com_user_idfalse
hashed_one_time_passcodefalse
hashed_passwordtrue
idtrue
is_systemtrue
last_seen_atfalse
login_typetrue
nametrue
one_time_passcode_expires_attrue
quiet_hours_scheduletrue
rbac_rolestrue
statustrue
updated_atfalse
usernametrue
| | WorkspaceAgent
connect, disconnect | |
FieldTracked
api_versionfalse
architecturefalse
auth_instance_idfalse
auth_tokenfalse
connection_timeout_secondsfalse
created_atfalse
directoryfalse
disconnected_atfalse
display_appsfalse
display_orderfalse
environment_variablesfalse
expanded_directoryfalse
first_connected_atfalse
idfalse
instance_metadatafalse
last_connected_atfalse
last_connected_replica_idfalse
lifecycle_statefalse
logs_lengthfalse
logs_overflowedfalse
motd_filefalse
namefalse
operating_systemfalse
ready_atfalse
resource_idfalse
resource_metadatafalse
started_atfalse
subsystemsfalse
troubleshooting_urlfalse
updated_atfalse
versionfalse
| | WorkspaceApp
open, close | |
FieldTracked
agent_idfalse
commandfalse
created_atfalse
display_namefalse
display_orderfalse
externalfalse
healthfalse
healthcheck_intervalfalse
healthcheck_thresholdfalse
healthcheck_urlfalse
hiddenfalse
iconfalse
idfalse
open_infalse
sharing_levelfalse
slugfalse
subdomainfalse
urlfalse
| | WorkspaceBuild
start, stop | |
FieldTracked
build_numberfalse
created_atfalse
daily_costfalse
deadlinefalse
idfalse
initiator_by_avatar_urlfalse
initiator_by_usernamefalse
initiator_idfalse
job_idfalse
max_deadlinefalse
provisioner_statefalse
reasonfalse
template_version_idtrue
template_version_preset_idfalse
transitionfalse
updated_atfalse
workspace_idfalse
| diff --git a/enterprise/audit/table.go b/enterprise/audit/table.go index 6fd3f46308975..84cc7d451b4f1 100644 --- a/enterprise/audit/table.go +++ b/enterprise/audit/table.go @@ -151,6 +151,7 @@ var auditableResourcesTypes = map[any]map[string]Action{ "github_com_user_id": ActionIgnore, "hashed_one_time_passcode": ActionIgnore, "one_time_passcode_expires_at": ActionTrack, + "is_system": ActionTrack, // Should never change, but track it anyway. }, &database.WorkspaceTable{}: { "id": ActionTrack, diff --git a/enterprise/coderd/groups_test.go b/enterprise/coderd/groups_test.go index 1baf62211dcd9..a6c9212b955f8 100644 --- a/enterprise/coderd/groups_test.go +++ b/enterprise/coderd/groups_test.go @@ -6,6 +6,9 @@ import ( "testing" "time" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/prebuilds" + "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -819,8 +822,14 @@ func TestGroup(t *testing.T) { }) t.Run("everyoneGroupReturnsEmpty", func(t *testing.T) { + // TODO (sasswart): this test seems to have drifted from its original intention. evaluate and remove/fix t.Parallel() + // TODO: we should not be returning the prebuilds user in Group, and this is not returned in dbmem. + if !dbtestutil.WillUsePostgres() { + t.Skip("This test requires postgres") + } + client, user := coderdenttest.New(t, &coderdenttest.Options{LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ codersdk.FeatureTemplateRBAC: 1, @@ -829,16 +838,20 @@ func TestGroup(t *testing.T) { userAdminClient, _ := coderdtest.CreateAnotherUser(t, client, user.OrganizationID, rbac.RoleUserAdmin()) _, user1 := coderdtest.CreateAnotherUser(t, client, user.OrganizationID) _, user2 := coderdtest.CreateAnotherUser(t, client, user.OrganizationID) - ctx := testutil.Context(t, testutil.WaitLong) + + // nolint:gocritic // "This client is operating as the owner user" is fine in this case. + prebuildsUser, err := client.User(ctx, prebuilds.OwnerID.String()) + require.NoError(t, err) // The 'Everyone' group always has an ID that matches the organization ID. group, err := userAdminClient.Group(ctx, user.OrganizationID) require.NoError(t, err) - require.Len(t, group.Members, 4) + require.Len(t, group.Members, 5) require.Equal(t, "Everyone", group.Name) require.Equal(t, user.OrganizationID, group.OrganizationID) require.Contains(t, group.Members, user1.ReducedUser) require.Contains(t, group.Members, user2.ReducedUser) + require.Contains(t, group.Members, prebuildsUser.ReducedUser) }) } diff --git a/enterprise/coderd/roles_test.go b/enterprise/coderd/roles_test.go index 57b66a368248c..b2d7da47a608d 100644 --- a/enterprise/coderd/roles_test.go +++ b/enterprise/coderd/roles_test.go @@ -7,6 +7,8 @@ import ( "slices" "testing" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -333,6 +335,12 @@ func TestCustomOrganizationRole(t *testing.T) { // Verify deleting a custom role cascades to all members t.Run("DeleteRoleCascadeMembers", func(t *testing.T) { t.Parallel() + + // TODO: we should not be returning the prebuilds user in OrganizationMembers, and this is not returned in dbmem. + if !dbtestutil.WillUsePostgres() { + t.Skip("This test requires postgres") + } + owner, first := coderdenttest.New(t, &coderdenttest.Options{ LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index a40ed7b64a6db..e66adebca4680 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -18,6 +18,7 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/notifications/notificationstest" "github.com/coder/coder/v2/coderd/rbac" @@ -922,6 +923,12 @@ func TestTemplateACL(t *testing.T) { t.Run("everyoneGroup", func(t *testing.T) { t.Parallel() + + // TODO: we should not be returning the prebuilds user in TemplateACL, and this is not returned in dbmem. + if !dbtestutil.WillUsePostgres() { + t.Skip("This test requires postgres") + } + client, user := coderdenttest.New(t, &coderdenttest.Options{LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ codersdk.FeatureTemplateRBAC: 1, @@ -940,7 +947,7 @@ func TestTemplateACL(t *testing.T) { require.NoError(t, err) require.Len(t, acl.Groups, 1) - require.Len(t, acl.Groups[0].Members, 2) + require.Len(t, acl.Groups[0].Members, 3) // orgAdmin + TemplateAdmin + prebuilds user require.Len(t, acl.Users, 0) }) From b7882374e12052d769ee595be880925e668cfce3 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 11:44:32 +0000 Subject: [PATCH 02/72] optionally prevent system users from counting to user count Signed-off-by: Danny Kopping --- cli/server.go | 2 +- coderd/database/dbauthz/dbauthz.go | 17 +++++---- coderd/database/dbauthz/dbauthz_test.go | 6 ++-- coderd/database/dbmem/dbmem.go | 11 ++++-- coderd/database/dbmetrics/querymetrics.go | 13 +++---- coderd/database/dbmock/dbmock.go | 24 ++++++------- .../migrations/000301_system_user.down.sql | 4 +++ coderd/database/modelqueries.go | 1 + coderd/database/querier.go | 6 ++-- coderd/database/querier_test.go | 1 + coderd/database/queries.sql.go | 35 ++++++++++++++----- .../database/queries/organizationmembers.sql | 6 ++++ coderd/database/queries/users.sql | 14 ++++++-- coderd/userauth.go | 3 +- coderd/userauth_test.go | 3 +- coderd/users.go | 4 +-- coderd/users_test.go | 3 +- enterprise/coderd/license/license.go | 2 +- enterprise/dbcrypt/cliutil.go | 5 +-- 19 files changed, 105 insertions(+), 55 deletions(-) diff --git a/cli/server.go b/cli/server.go index 745794a236200..7a47b1d4e1135 100644 --- a/cli/server.go +++ b/cli/server.go @@ -1894,7 +1894,7 @@ func getGithubOAuth2ConfigParams(ctx context.Context, db database.Store, vals *c if defaultEligibleNotSet { // nolint:gocritic // User count requires system privileges - userCount, err := db.GetUserCount(dbauthz.AsSystemRestricted(ctx)) + userCount, err := db.GetUserCount(dbauthz.AsSystemRestricted(ctx), false) if err != nil { return nil, xerrors.Errorf("get user count: %w", err) } diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 796127c83259b..b3770f07d7362 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1084,13 +1084,13 @@ func (q *querier) ActivityBumpWorkspace(ctx context.Context, arg database.Activi return update(q.log, q.auth, fetch, q.db.ActivityBumpWorkspace)(ctx, arg) } -func (q *querier) AllUserIDs(ctx context.Context) ([]uuid.UUID, error) { +func (q *querier) AllUserIDs(ctx context.Context, includeSystem bool) ([]uuid.UUID, error) { // Although this technically only reads users, only system-related functions should be // allowed to call this. if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } - return q.db.AllUserIDs(ctx) + return q.db.AllUserIDs(ctx, includeSystem) } func (q *querier) ArchiveUnusedTemplateVersions(ctx context.Context, arg database.ArchiveUnusedTemplateVersionsParams) ([]uuid.UUID, error) { @@ -1343,7 +1343,10 @@ func (q *querier) DeleteOldWorkspaceAgentStats(ctx context.Context) error { func (q *querier) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { return deleteQ[database.OrganizationMember](q.log, q.auth, func(ctx context.Context, arg database.DeleteOrganizationMemberParams) (database.OrganizationMember, error) { - member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams(arg))) + member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{ + OrganizationID: arg.OrganizationID, + UserID: arg.UserID, + })) if err != nil { return database.OrganizationMember{}, err } @@ -1529,11 +1532,11 @@ func (q *querier) GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed time.Tim return fetchWithPostFilter(q.auth, policy.ActionRead, q.db.GetAPIKeysLastUsedAfter)(ctx, lastUsed) } -func (q *querier) GetActiveUserCount(ctx context.Context) (int64, error) { +func (q *querier) GetActiveUserCount(ctx context.Context, includeSystem bool) (int64, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return 0, err } - return q.db.GetActiveUserCount(ctx) + return q.db.GetActiveUserCount(ctx, includeSystem) } func (q *querier) GetActiveWorkspaceBuildsByTemplateID(ctx context.Context, templateID uuid.UUID) ([]database.WorkspaceBuild, error) { @@ -2557,11 +2560,11 @@ func (q *querier) GetUserByID(ctx context.Context, id uuid.UUID) (database.User, return fetch(q.log, q.auth, q.db.GetUserByID)(ctx, id) } -func (q *querier) GetUserCount(ctx context.Context) (int64, error) { +func (q *querier) GetUserCount(ctx context.Context, includeSystem bool) (int64, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return 0, err } - return q.db.GetUserCount(ctx) + return q.db.GetUserCount(ctx, includeSystem) } func (q *querier) GetUserLatencyInsights(ctx context.Context, arg database.GetUserLatencyInsightsParams) ([]database.GetUserLatencyInsightsRow, error) { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index ec8ced783fa0a..ca1379fdbdc84 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1664,7 +1664,7 @@ func (s *MethodTestSuite) TestUser() { s.Run("AllUserIDs", s.Subtest(func(db database.Store, check *expects) { a := dbgen.User(s.T(), db, database.User{}) b := dbgen.User(s.T(), db, database.User{}) - check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(slice.New(a.ID, b.ID)) + check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(slice.New(a.ID, b.ID)) })) s.Run("CustomRoles", s.Subtest(func(db database.Store, check *expects) { check.Args(database.CustomRolesParams{}).Asserts(rbac.ResourceAssignRole, policy.ActionRead).Returns([]database.CustomRole{}) @@ -3649,7 +3649,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead) })) s.Run("GetActiveUserCount", s.Subtest(func(db database.Store, check *expects) { - check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0)) + check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0)) })) s.Run("GetUnexpiredLicenses", s.Subtest(func(db database.Store, check *expects) { check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead) @@ -3692,7 +3692,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(time.Now().Add(time.Hour*-1)).Asserts(rbac.ResourceSystem, policy.ActionRead) })) s.Run("GetUserCount", s.Subtest(func(db database.Store, check *expects) { - check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0)) + check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0)) })) s.Run("GetTemplates", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 1ece2571f4960..5ac92468c3188 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1549,7 +1549,7 @@ func (q *FakeQuerier) ActivityBumpWorkspace(ctx context.Context, arg database.Ac return sql.ErrNoRows } -func (q *FakeQuerier) AllUserIDs(_ context.Context) ([]uuid.UUID, error) { +func (q *FakeQuerier) AllUserIDs(_ context.Context, includeSystem bool) ([]uuid.UUID, error) { q.mutex.RLock() defer q.mutex.RUnlock() userIDs := make([]uuid.UUID, 0, len(q.users)) @@ -2644,7 +2644,7 @@ func (q *FakeQuerier) GetAPIKeysLastUsedAfter(_ context.Context, after time.Time return apiKeys, nil } -func (q *FakeQuerier) GetActiveUserCount(_ context.Context) (int64, error) { +func (q *FakeQuerier) GetActiveUserCount(_ context.Context, includeSystem bool) (int64, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -6200,7 +6200,8 @@ func (q *FakeQuerier) GetUserByID(_ context.Context, id uuid.UUID) (database.Use return q.getUserByIDNoLock(id) } -func (q *FakeQuerier) GetUserCount(_ context.Context) (int64, error) { +// nolint:revive // Not a control flag; used for filtering. +func (q *FakeQuerier) GetUserCount(_ context.Context, includeSystem bool) (int64, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -6209,6 +6210,10 @@ func (q *FakeQuerier) GetUserCount(_ context.Context) (int64, error) { if !u.Deleted { existing++ } + + if !includeSystem && u.IsSystem.Valid && u.IsSystem.Bool { + continue + } } return existing, nil } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 407d9e48bfcf8..bae68852bbf2b 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/coderd/rbac/policy" @@ -115,9 +116,9 @@ func (m queryMetricsStore) ActivityBumpWorkspace(ctx context.Context, arg databa return r0 } -func (m queryMetricsStore) AllUserIDs(ctx context.Context) ([]uuid.UUID, error) { +func (m queryMetricsStore) AllUserIDs(ctx context.Context, includeSystem bool) ([]uuid.UUID, error) { start := time.Now() - r0, r1 := m.s.AllUserIDs(ctx) + r0, r1 := m.s.AllUserIDs(ctx, includeSystem) m.queryLatencies.WithLabelValues("AllUserIDs").Observe(time.Since(start).Seconds()) return r0, r1 } @@ -514,9 +515,9 @@ func (m queryMetricsStore) GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed return apiKeys, err } -func (m queryMetricsStore) GetActiveUserCount(ctx context.Context) (int64, error) { +func (m queryMetricsStore) GetActiveUserCount(ctx context.Context, includeSystem bool) (int64, error) { start := time.Now() - count, err := m.s.GetActiveUserCount(ctx) + count, err := m.s.GetActiveUserCount(ctx, includeSystem) m.queryLatencies.WithLabelValues("GetActiveUserCount").Observe(time.Since(start).Seconds()) return count, err } @@ -1424,9 +1425,9 @@ func (m queryMetricsStore) GetUserByID(ctx context.Context, id uuid.UUID) (datab return user, err } -func (m queryMetricsStore) GetUserCount(ctx context.Context) (int64, error) { +func (m queryMetricsStore) GetUserCount(ctx context.Context, includeSystem bool) (int64, error) { start := time.Now() - count, err := m.s.GetUserCount(ctx) + count, err := m.s.GetUserCount(ctx, includeSystem) m.queryLatencies.WithLabelValues("GetUserCount").Observe(time.Since(start).Seconds()) return count, err } diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index fbe4d0745fbb0..b2e8eabe4594e 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -103,18 +103,18 @@ func (mr *MockStoreMockRecorder) ActivityBumpWorkspace(ctx, arg any) *gomock.Cal } // AllUserIDs mocks base method. -func (m *MockStore) AllUserIDs(ctx context.Context) ([]uuid.UUID, error) { +func (m *MockStore) AllUserIDs(ctx context.Context, includeSystem bool) ([]uuid.UUID, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllUserIDs", ctx) + ret := m.ctrl.Call(m, "AllUserIDs", ctx, includeSystem) ret0, _ := ret[0].([]uuid.UUID) ret1, _ := ret[1].(error) return ret0, ret1 } // AllUserIDs indicates an expected call of AllUserIDs. -func (mr *MockStoreMockRecorder) AllUserIDs(ctx any) *gomock.Call { +func (mr *MockStoreMockRecorder) AllUserIDs(ctx, includeSystem any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserIDs", reflect.TypeOf((*MockStore)(nil).AllUserIDs), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserIDs", reflect.TypeOf((*MockStore)(nil).AllUserIDs), ctx, includeSystem) } // ArchiveUnusedTemplateVersions mocks base method. @@ -923,18 +923,18 @@ func (mr *MockStoreMockRecorder) GetAPIKeysLastUsedAfter(ctx, lastUsed any) *gom } // GetActiveUserCount mocks base method. -func (m *MockStore) GetActiveUserCount(ctx context.Context) (int64, error) { +func (m *MockStore) GetActiveUserCount(ctx context.Context, includeSystem bool) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetActiveUserCount", ctx) + ret := m.ctrl.Call(m, "GetActiveUserCount", ctx, includeSystem) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // GetActiveUserCount indicates an expected call of GetActiveUserCount. -func (mr *MockStoreMockRecorder) GetActiveUserCount(ctx any) *gomock.Call { +func (mr *MockStoreMockRecorder) GetActiveUserCount(ctx, includeSystem any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveUserCount", reflect.TypeOf((*MockStore)(nil).GetActiveUserCount), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetActiveUserCount", reflect.TypeOf((*MockStore)(nil).GetActiveUserCount), ctx, includeSystem) } // GetActiveWorkspaceBuildsByTemplateID mocks base method. @@ -2978,18 +2978,18 @@ func (mr *MockStoreMockRecorder) GetUserByID(ctx, id any) *gomock.Call { } // GetUserCount mocks base method. -func (m *MockStore) GetUserCount(ctx context.Context) (int64, error) { +func (m *MockStore) GetUserCount(ctx context.Context, includeSystem bool) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetUserCount", ctx) + ret := m.ctrl.Call(m, "GetUserCount", ctx, includeSystem) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // GetUserCount indicates an expected call of GetUserCount. -func (mr *MockStoreMockRecorder) GetUserCount(ctx any) *gomock.Call { +func (mr *MockStoreMockRecorder) GetUserCount(ctx, includeSystem any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCount", reflect.TypeOf((*MockStore)(nil).GetUserCount), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCount", reflect.TypeOf((*MockStore)(nil).GetUserCount), ctx, includeSystem) } // GetUserLatencyInsights mocks base method. diff --git a/coderd/database/migrations/000301_system_user.down.sql b/coderd/database/migrations/000301_system_user.down.sql index 1117ddb2f2513..c286cfa193745 100644 --- a/coderd/database/migrations/000301_system_user.down.sql +++ b/coderd/database/migrations/000301_system_user.down.sql @@ -9,6 +9,10 @@ DROP TRIGGER IF EXISTS prevent_system_user_deletions ON users; -- Drop function DROP FUNCTION IF EXISTS prevent_system_user_changes(); +-- Delete user status changes +DELETE FROM user_status_changes +WHERE user_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; + -- Delete system user DELETE FROM users WHERE id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; diff --git a/coderd/database/modelqueries.go b/coderd/database/modelqueries.go index 506cd7a69fe34..a16c26fc6840c 100644 --- a/coderd/database/modelqueries.go +++ b/coderd/database/modelqueries.go @@ -393,6 +393,7 @@ func (q *sqlQuerier) GetAuthorizedUsers(ctx context.Context, arg GetUsersParams, arg.LastSeenAfter, arg.CreatedBefore, arg.CreatedAfter, + arg.IncludeSystem, arg.OffsetOpt, arg.LimitOpt, ) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index d72469650f0ea..0de928932652b 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -49,7 +49,7 @@ type sqlcQuerier interface { // We only bump when 5% of the deadline has elapsed. ActivityBumpWorkspace(ctx context.Context, arg ActivityBumpWorkspaceParams) error // AllUserIDs returns all UserIDs regardless of user status or deletion. - AllUserIDs(ctx context.Context) ([]uuid.UUID, error) + AllUserIDs(ctx context.Context, includeSystem bool) ([]uuid.UUID, error) // Archiving templates is a soft delete action, so is reversible. // Archiving prevents the version from being used and discovered // by listing. @@ -124,7 +124,7 @@ type sqlcQuerier interface { GetAPIKeysByLoginType(ctx context.Context, loginType LoginType) ([]APIKey, error) GetAPIKeysByUserID(ctx context.Context, arg GetAPIKeysByUserIDParams) ([]APIKey, error) GetAPIKeysLastUsedAfter(ctx context.Context, lastUsed time.Time) ([]APIKey, error) - GetActiveUserCount(ctx context.Context) (int64, error) + GetActiveUserCount(ctx context.Context, includeSystem bool) (int64, error) GetActiveWorkspaceBuildsByTemplateID(ctx context.Context, templateID uuid.UUID) ([]WorkspaceBuild, error) GetAllTailnetAgents(ctx context.Context) ([]TailnetAgent, error) // For PG Coordinator HTMLDebug @@ -309,7 +309,7 @@ type sqlcQuerier interface { GetUserAppearanceSettings(ctx context.Context, userID uuid.UUID) (string, error) GetUserByEmailOrUsername(ctx context.Context, arg GetUserByEmailOrUsernameParams) (User, error) GetUserByID(ctx context.Context, id uuid.UUID) (User, error) - GetUserCount(ctx context.Context) (int64, error) + GetUserCount(ctx context.Context, includeSystem bool) (int64, error) // GetUserLatencyInsights returns the median and 95th percentile connection // latency that users have experienced. The result can be filtered on // template_ids, meaning only user data from workspaces based on those templates diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 837068f1fa03e..7606e90b5107c 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 2c5138dbee4af..2a448ca836c35 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5216,11 +5216,18 @@ WHERE user_id = $2 ELSE true END + -- Filter by system type + AND CASE + WHEN $3::bool THEN TRUE + ELSE + is_system = false + END ` type OrganizationMembersParams struct { OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` UserID uuid.UUID `db:"user_id" json:"user_id"` + IncludeSystem bool `db:"include_system" json:"include_system"` } type OrganizationMembersRow struct { @@ -5237,7 +5244,7 @@ type OrganizationMembersRow struct { // - Use just 'user_id' to get all orgs a user is a member of // - Use both to get a specific org member row func (q *sqlQuerier) OrganizationMembers(ctx context.Context, arg OrganizationMembersParams) ([]OrganizationMembersRow, error) { - rows, err := q.db.QueryContext(ctx, organizationMembers, arg.OrganizationID, arg.UserID) + rows, err := q.db.QueryContext(ctx, organizationMembers, arg.OrganizationID, arg.UserID, arg.IncludeSystem) if err != nil { return nil, err } @@ -11325,11 +11332,12 @@ func (q *sqlQuerier) UpdateUserLinkedID(ctx context.Context, arg UpdateUserLinke const allUserIDs = `-- name: AllUserIDs :many SELECT DISTINCT id FROM USERS + WHERE CASE WHEN $1::bool THEN TRUE ELSE is_system = false END ` // AllUserIDs returns all UserIDs regardless of user status or deletion. -func (q *sqlQuerier) AllUserIDs(ctx context.Context) ([]uuid.UUID, error) { - rows, err := q.db.QueryContext(ctx, allUserIDs) +func (q *sqlQuerier) AllUserIDs(ctx context.Context, includeSystem bool) ([]uuid.UUID, error) { + rows, err := q.db.QueryContext(ctx, allUserIDs, includeSystem) if err != nil { return nil, err } @@ -11358,10 +11366,11 @@ FROM users WHERE status = 'active'::user_status AND deleted = false + AND CASE WHEN $1::bool THEN TRUE ELSE is_system = false END ` -func (q *sqlQuerier) GetActiveUserCount(ctx context.Context) (int64, error) { - row := q.db.QueryRowContext(ctx, getActiveUserCount) +func (q *sqlQuerier) GetActiveUserCount(ctx context.Context, includeSystem bool) (int64, error) { + row := q.db.QueryRowContext(ctx, getActiveUserCount, includeSystem) var count int64 err := row.Scan(&count) return count, err @@ -11536,10 +11545,11 @@ FROM users WHERE deleted = false + AND CASE WHEN $1::bool THEN TRUE ELSE is_system = false END ` -func (q *sqlQuerier) GetUserCount(ctx context.Context) (int64, error) { - row := q.db.QueryRowContext(ctx, getUserCount) +func (q *sqlQuerier) GetUserCount(ctx context.Context, includeSystem bool) (int64, error) { + row := q.db.QueryRowContext(ctx, getUserCount, includeSystem) var count int64 err := row.Scan(&count) return count, err @@ -11618,16 +11628,21 @@ WHERE created_at >= $8 ELSE true END + AND CASE + WHEN $9::bool THEN TRUE + ELSE + is_system = false + END -- End of filters -- Authorize Filter clause will be injected below in GetAuthorizedUsers -- @authorize_filter ORDER BY -- Deterministic and consistent ordering of all users. This is to ensure consistent pagination. - LOWER(username) ASC OFFSET $9 + LOWER(username) ASC OFFSET $10 LIMIT -- A null limit means "no limit", so 0 means return all - NULLIF($10 :: int, 0) + NULLIF($11 :: int, 0) ` type GetUsersParams struct { @@ -11639,6 +11654,7 @@ type GetUsersParams struct { LastSeenAfter time.Time `db:"last_seen_after" json:"last_seen_after"` CreatedBefore time.Time `db:"created_before" json:"created_before"` CreatedAfter time.Time `db:"created_after" json:"created_after"` + IncludeSystem bool `db:"include_system" json:"include_system"` OffsetOpt int32 `db:"offset_opt" json:"offset_opt"` LimitOpt int32 `db:"limit_opt" json:"limit_opt"` } @@ -11676,6 +11692,7 @@ func (q *sqlQuerier) GetUsers(ctx context.Context, arg GetUsersParams) ([]GetUse arg.LastSeenAfter, arg.CreatedBefore, arg.CreatedAfter, + arg.IncludeSystem, arg.OffsetOpt, arg.LimitOpt, ) diff --git a/coderd/database/queries/organizationmembers.sql b/coderd/database/queries/organizationmembers.sql index a92cd681eabf6..9d570bc1c49ee 100644 --- a/coderd/database/queries/organizationmembers.sql +++ b/coderd/database/queries/organizationmembers.sql @@ -22,6 +22,12 @@ WHERE WHEN @user_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = @user_id ELSE true + END + -- Filter by system type + AND CASE + WHEN @include_system::bool THEN TRUE + ELSE + is_system = false END; -- name: InsertOrganizationMember :one diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index 79f19c1784155..4900e70262e3c 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -46,7 +46,8 @@ SELECT FROM users WHERE - deleted = false; + deleted = false + AND CASE WHEN @include_system::bool THEN TRUE ELSE is_system = false END; -- name: GetActiveUserCount :one SELECT @@ -54,7 +55,8 @@ SELECT FROM users WHERE - status = 'active'::user_status AND deleted = false; + status = 'active'::user_status AND deleted = false + AND CASE WHEN @include_system::bool THEN TRUE ELSE is_system = false END; -- name: InsertUser :one INSERT INTO @@ -223,6 +225,11 @@ WHERE created_at >= @created_after ELSE true END + AND CASE + WHEN @include_system::bool THEN TRUE + ELSE + is_system = false + END -- End of filters -- Authorize Filter clause will be injected below in GetAuthorizedUsers @@ -319,7 +326,8 @@ RETURNING id, email, username, last_seen_at; -- AllUserIDs returns all UserIDs regardless of user status or deletion. -- name: AllUserIDs :many -SELECT DISTINCT id FROM USERS; +SELECT DISTINCT id FROM USERS + WHERE CASE WHEN @include_system::bool THEN TRUE ELSE is_system = false END; -- name: UpdateUserHashedOneTimePasscode :exec UPDATE diff --git a/coderd/userauth.go b/coderd/userauth.go index 3c1481b1f9039..a6c7c1d319fc9 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -24,6 +24,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/idpsync" "github.com/coder/coder/v2/coderd/jwtutils" @@ -1665,7 +1666,7 @@ func (api *API) oauthLogin(r *http.Request, params *oauthLoginParams) ([]*http.C } // nolint:gocritic // Getting user count is a system function. - userCount, err := tx.GetUserCount(dbauthz.AsSystemRestricted(ctx)) + userCount, err := tx.GetUserCount(dbauthz.AsSystemRestricted(ctx), false) if err != nil { return xerrors.Errorf("unable to fetch user count: %w", err) } diff --git a/coderd/userauth_test.go b/coderd/userauth_test.go index ee6ee957ba861..4b67320164fc2 100644 --- a/coderd/userauth_test.go +++ b/coderd/userauth_test.go @@ -28,6 +28,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/coderdtest" @@ -304,7 +305,7 @@ func TestUserOAuth2Github(t *testing.T) { ctx := testutil.Context(t, testutil.WaitLong) // nolint:gocritic // Unit test - count, err := db.GetUserCount(dbauthz.AsSystemRestricted(ctx)) + count, err := db.GetUserCount(dbauthz.AsSystemRestricted(ctx), false) require.NoError(t, err) require.Equal(t, int64(1), count) diff --git a/coderd/users.go b/coderd/users.go index bbb10c4787a27..f6b961b7c2f8a 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -85,7 +85,7 @@ func (api *API) userDebugOIDC(rw http.ResponseWriter, r *http.Request) { func (api *API) firstUser(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() // nolint:gocritic // Getting user count is a system function. - userCount, err := api.Database.GetUserCount(dbauthz.AsSystemRestricted(ctx)) + userCount, err := api.Database.GetUserCount(dbauthz.AsSystemRestricted(ctx), false) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error fetching user count.", @@ -128,7 +128,7 @@ func (api *API) postFirstUser(rw http.ResponseWriter, r *http.Request) { // This should only function for the first user. // nolint:gocritic // Getting user count is a system function. - userCount, err := api.Database.GetUserCount(dbauthz.AsSystemRestricted(ctx)) + userCount, err := api.Database.GetUserCount(dbauthz.AsSystemRestricted(ctx), false) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error fetching user count.", diff --git a/coderd/users_test.go b/coderd/users_test.go index 2d85a9823a587..378aaab2d3a70 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -9,12 +9,13 @@ import ( "testing" "time" + "github.com/coder/serpent" + "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/notifications/notificationstest" "github.com/coder/coder/v2/coderd/rbac/policy" - "github.com/coder/serpent" "github.com/golang-jwt/jwt/v4" "github.com/google/uuid" diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index 6f0e827eb3320..fbd53dcaac58c 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -33,7 +33,7 @@ func Entitlements( } // nolint:gocritic // Getting active user count is a system function. - activeUserCount, err := db.GetActiveUserCount(dbauthz.AsSystemRestricted(ctx)) + activeUserCount, err := db.GetActiveUserCount(dbauthz.AsSystemRestricted(ctx), false) // Don't include system user in license count. if err != nil { return codersdk.Entitlements{}, xerrors.Errorf("query active user count: %w", err) } diff --git a/enterprise/dbcrypt/cliutil.go b/enterprise/dbcrypt/cliutil.go index 120b41972de05..a94760d3d6e65 100644 --- a/enterprise/dbcrypt/cliutil.go +++ b/enterprise/dbcrypt/cliutil.go @@ -7,6 +7,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/database" ) @@ -19,7 +20,7 @@ func Rotate(ctx context.Context, log slog.Logger, sqlDB *sql.DB, ciphers []Ciphe return xerrors.Errorf("create cryptdb: %w", err) } - userIDs, err := db.AllUserIDs(ctx) + userIDs, err := db.AllUserIDs(ctx, false) if err != nil { return xerrors.Errorf("get users: %w", err) } @@ -109,7 +110,7 @@ func Decrypt(ctx context.Context, log slog.Logger, sqlDB *sql.DB, ciphers []Ciph } cryptDB.primaryCipherDigest = "" - userIDs, err := db.AllUserIDs(ctx) + userIDs, err := db.AllUserIDs(ctx, false) if err != nil { return xerrors.Errorf("get users: %w", err) } From 812259528b6440140ec0e8cc5febd362a5b33540 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 19:19:40 +0000 Subject: [PATCH 03/72] appease the linter Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz.go | 2 ++ coderd/database/dbmem/dbmem.go | 12 +++++++++++- coderd/httpmw/organizationparam.go | 1 + coderd/idpsync/role.go | 2 ++ coderd/members.go | 1 + coderd/users.go | 1 + enterprise/coderd/groups.go | 1 + 7 files changed, 19 insertions(+), 1 deletion(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index b3770f07d7362..38f6af62a4705 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1346,6 +1346,7 @@ func (q *querier) DeleteOrganizationMember(ctx context.Context, arg database.Del member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: arg.OrganizationID, UserID: arg.UserID, + IncludeSystem: false, })) if err != nil { return database.OrganizationMember{}, err @@ -3776,6 +3777,7 @@ func (q *querier) UpdateMemberRoles(ctx context.Context, arg database.UpdateMemb member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: arg.OrgID, UserID: arg.UserID, + IncludeSystem: false, })) if err != nil { return database.OrganizationMember{}, err diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 5ac92468c3188..0c39738069cb0 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1549,11 +1549,16 @@ func (q *FakeQuerier) ActivityBumpWorkspace(ctx context.Context, arg database.Ac return sql.ErrNoRows } +// nolint:revive // It's not a control flag, it's a filter. func (q *FakeQuerier) AllUserIDs(_ context.Context, includeSystem bool) ([]uuid.UUID, error) { q.mutex.RLock() defer q.mutex.RUnlock() userIDs := make([]uuid.UUID, 0, len(q.users)) for idx := range q.users { + if !includeSystem && q.users[idx].IsSystem.Valid && q.users[idx].IsSystem.Bool { + continue + } + userIDs = append(userIDs, q.users[idx].ID) } return userIDs, nil @@ -2644,12 +2649,17 @@ func (q *FakeQuerier) GetAPIKeysLastUsedAfter(_ context.Context, after time.Time return apiKeys, nil } +// nolint:revive // It's not a control flag, it's a filter. func (q *FakeQuerier) GetActiveUserCount(_ context.Context, includeSystem bool) (int64, error) { q.mutex.RLock() defer q.mutex.RUnlock() active := int64(0) for _, u := range q.users { + if !includeSystem && u.IsSystem.Valid && u.IsSystem.Bool { + continue + } + if u.Status == database.UserStatusActive && !u.Deleted { active++ } @@ -6200,7 +6210,7 @@ func (q *FakeQuerier) GetUserByID(_ context.Context, id uuid.UUID) (database.Use return q.getUserByIDNoLock(id) } -// nolint:revive // Not a control flag; used for filtering. +// nolint:revive // It's not a control flag, it's a filter. func (q *FakeQuerier) GetUserCount(_ context.Context, includeSystem bool) (int64, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/httpmw/organizationparam.go b/coderd/httpmw/organizationparam.go index 2eba0dcedf5b8..18938ec1e792d 100644 --- a/coderd/httpmw/organizationparam.go +++ b/coderd/httpmw/organizationparam.go @@ -126,6 +126,7 @@ func ExtractOrganizationMemberParam(db database.Store) func(http.Handler) http.H organizationMember, err := database.ExpectOne(db.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: organization.ID, UserID: user.ID, + IncludeSystem: false, })) if httpapi.Is404Error(err) { httpapi.ResourceNotFound(rw) diff --git a/coderd/idpsync/role.go b/coderd/idpsync/role.go index 22e0edc3bc662..54ec787661826 100644 --- a/coderd/idpsync/role.go +++ b/coderd/idpsync/role.go @@ -10,6 +10,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/rbac" @@ -91,6 +92,7 @@ func (s AGPLIDPSync) SyncRoles(ctx context.Context, db database.Store, user data orgMemberships, err := tx.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: uuid.Nil, UserID: user.ID, + IncludeSystem: false, }) if err != nil { return xerrors.Errorf("get organizations by user id: %w", err) diff --git a/coderd/members.go b/coderd/members.go index 1852e6448408f..d1c4cdf01770c 100644 --- a/coderd/members.go +++ b/coderd/members.go @@ -160,6 +160,7 @@ func (api *API) listMembers(rw http.ResponseWriter, r *http.Request) { members, err := api.Database.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: organization.ID, UserID: uuid.Nil, + IncludeSystem: false, }) if httpapi.Is404Error(err) { httpapi.ResourceNotFound(rw) diff --git a/coderd/users.go b/coderd/users.go index f6b961b7c2f8a..240ccb786019c 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -1191,6 +1191,7 @@ func (api *API) userRoles(rw http.ResponseWriter, r *http.Request) { memberships, err := api.Database.OrganizationMembers(ctx, database.OrganizationMembersParams{ UserID: user.ID, OrganizationID: uuid.Nil, + IncludeSystem: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/enterprise/coderd/groups.go b/enterprise/coderd/groups.go index 6b94adb2c5b78..fef4a1bbef04b 100644 --- a/enterprise/coderd/groups.go +++ b/enterprise/coderd/groups.go @@ -170,6 +170,7 @@ func (api *API) patchGroup(rw http.ResponseWriter, r *http.Request) { _, err := database.ExpectOne(api.Database.OrganizationMembers(ctx, database.OrganizationMembersParams{ OrganizationID: group.OrganizationID, UserID: uuid.MustParse(id), + IncludeSystem: false, })) if errors.Is(err, sql.ErrNoRows) { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ From bfb7c2817e94a5e6cad3bfeeb38cf4a36681526f Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 19:49:32 +0000 Subject: [PATCH 04/72] add unit test for system user behaviour Signed-off-by: Danny Kopping --- coderd/users_test.go | 92 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/coderd/users_test.go b/coderd/users_test.go index 378aaab2d3a70..0b83d33e4b435 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -2,6 +2,7 @@ package coderd_test import ( "context" + "database/sql" "fmt" "net/http" "slices" @@ -13,8 +14,10 @@ import ( "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" + "github.com/coder/coder/v2/coderd/database/migrations" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/notifications/notificationstest" + "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/golang-jwt/jwt/v4" @@ -2415,3 +2418,92 @@ func BenchmarkUsersMe(b *testing.B) { require.NoError(b, err) } } + +func TestSystemUserBehaviour(t *testing.T) { + // Setup. + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + + sqlDB := testSQLDB(t) + err := migrations.Up(sqlDB) // coderd/database/migrations/00030*_system_user.up.sql will create a system user. + require.NoError(t, err, "migrations") + + db := database.New(sqlDB) + + // ================================================================================================================= + + // When: retrieving users with the include_system flag enabled. + other := dbgen.User(t, db, database.User{}) + users, err := db.GetUsers(ctx, database.GetUsersParams{ + IncludeSystem: true, + }) + + // Then: system users are returned, alongside other users. + require.NoError(t, err) + require.Len(t, users, 2) + + var systemUser, regularUser database.GetUsersRow + for _, u := range users { + if u.IsSystem.Bool { + systemUser = u + } else { + regularUser = u + } + } + require.NotNil(t, systemUser) + require.NotNil(t, regularUser) + + require.True(t, systemUser.IsSystem.Bool) + require.Equal(t, systemUser.ID, prebuilds.OwnerID) + require.False(t, regularUser.IsSystem.Bool) + require.Equal(t, regularUser.ID, other.ID) + + // ================================================================================================================= + + // When: retrieving users with the include_system flag disabled. + users, err = db.GetUsers(ctx, database.GetUsersParams{ + IncludeSystem: false, + }) + + // Then: only regular users are returned. + require.NoError(t, err) + require.Len(t, users, 1) + require.False(t, users[0].IsSystem.Bool) + + // ================================================================================================================= + + // When: attempting to update a system user's name. + _, err = db.UpdateUserProfile(ctx, database.UpdateUserProfileParams{ + ID: systemUser.ID, + Name: "not prebuilds", + }) + // Then: the attempt is rejected by a postgres trigger. + require.ErrorContains(t, err, "Cannot modify or delete system users") + + // When: attempting to delete a system user. + err = db.UpdateUserDeletedByID(ctx, systemUser.ID) + // Then: the attempt is rejected by a postgres trigger. + require.ErrorContains(t, err, "Cannot modify or delete system users") + + // When: attempting to update a user's roles. + _, err = db.UpdateUserRoles(ctx, database.UpdateUserRolesParams{ + ID: systemUser.ID, + GrantedRoles: []string{rbac.RoleAuditor().String()}, + }) + // Then: the attempt is rejected by a postgres trigger. + require.ErrorContains(t, err, "Cannot modify or delete system users") +} + +func testSQLDB(t testing.TB) *sql.DB { + t.Helper() + + connection, err := dbtestutil.Open(t) + require.NoError(t, err) + + db, err := sql.Open("postgres", connection) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + return db +} From 6639167999a35b1c25dcae0a90cca3576e34827c Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 19:55:25 +0000 Subject: [PATCH 05/72] reverting RBAC changes; not relevant here appeasing linter Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz.go | 28 ---------------------------- coderd/users_test.go | 4 ++-- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 38f6af62a4705..34c6999a16a0b 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -18,7 +18,6 @@ import ( "cdr.dev/slog" - "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/rbac/rolestore" @@ -359,27 +358,6 @@ var ( }), Scope: rbac.ScopeAll, }.WithCachedASTValue() - - subjectPrebuildsOrchestrator = rbac.Subject{ - FriendlyName: "Prebuilds Orchestrator", - ID: prebuilds.OwnerID.String(), - Roles: rbac.Roles([]rbac.Role{ - { - Identifier: rbac.RoleIdentifier{Name: "prebuilds-orchestrator"}, - DisplayName: "Coder", - Site: rbac.Permissions(map[string][]policy.Action{ - // May use template, read template-related info, & insert template-related resources (preset prebuilds). - rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionUse}, - // May CRUD workspaces, and start/stop them. - rbac.ResourceWorkspace.Type: { - policy.ActionCreate, policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, - policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, - }, - }), - }, - }), - Scope: rbac.ScopeAll, - }.WithCachedASTValue() ) // AsProvisionerd returns a context with an actor that has permissions required @@ -434,12 +412,6 @@ func AsSystemReadProvisionerDaemons(ctx context.Context) context.Context { return context.WithValue(ctx, authContextKey{}, subjectSystemReadProvisionerDaemons) } -// AsPrebuildsOrchestrator returns a context with an actor that has permissions -// to read orchestrator workspace prebuilds. -func AsPrebuildsOrchestrator(ctx context.Context) context.Context { - return context.WithValue(ctx, authContextKey{}, subjectPrebuildsOrchestrator) -} - var AsRemoveActor = rbac.Subject{ ID: "remove-actor", } diff --git a/coderd/users_test.go b/coderd/users_test.go index 0b83d33e4b435..99b10cba961dd 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -2428,7 +2428,7 @@ func TestSystemUserBehaviour(t *testing.T) { sqlDB := testSQLDB(t) err := migrations.Up(sqlDB) // coderd/database/migrations/00030*_system_user.up.sql will create a system user. require.NoError(t, err, "migrations") - + db := database.New(sqlDB) // ================================================================================================================= @@ -2488,7 +2488,7 @@ func TestSystemUserBehaviour(t *testing.T) { // When: attempting to update a user's roles. _, err = db.UpdateUserRoles(ctx, database.UpdateUserRolesParams{ - ID: systemUser.ID, + ID: systemUser.ID, GrantedRoles: []string{rbac.RoleAuditor().String()}, }) // Then: the attempt is rejected by a postgres trigger. From 769ae1d24c708e4e4ade40195939a4b9acd9f573 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 20:07:29 +0000 Subject: [PATCH 06/72] removing unnecessary changes Signed-off-by: Danny Kopping --- enterprise/coderd/groups_test.go | 1 - enterprise/coderd/roles_test.go | 8 -------- enterprise/coderd/templates_test.go | 1 - 3 files changed, 10 deletions(-) diff --git a/enterprise/coderd/groups_test.go b/enterprise/coderd/groups_test.go index a6c9212b955f8..2f6660c9eb280 100644 --- a/enterprise/coderd/groups_test.go +++ b/enterprise/coderd/groups_test.go @@ -825,7 +825,6 @@ func TestGroup(t *testing.T) { // TODO (sasswart): this test seems to have drifted from its original intention. evaluate and remove/fix t.Parallel() - // TODO: we should not be returning the prebuilds user in Group, and this is not returned in dbmem. if !dbtestutil.WillUsePostgres() { t.Skip("This test requires postgres") } diff --git a/enterprise/coderd/roles_test.go b/enterprise/coderd/roles_test.go index b2d7da47a608d..57b66a368248c 100644 --- a/enterprise/coderd/roles_test.go +++ b/enterprise/coderd/roles_test.go @@ -7,8 +7,6 @@ import ( "slices" "testing" - "github.com/coder/coder/v2/coderd/database/dbtestutil" - "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -335,12 +333,6 @@ func TestCustomOrganizationRole(t *testing.T) { // Verify deleting a custom role cascades to all members t.Run("DeleteRoleCascadeMembers", func(t *testing.T) { t.Parallel() - - // TODO: we should not be returning the prebuilds user in OrganizationMembers, and this is not returned in dbmem. - if !dbtestutil.WillUsePostgres() { - t.Skip("This test requires postgres") - } - owner, first := coderdenttest.New(t, &coderdenttest.Options{ LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ diff --git a/enterprise/coderd/templates_test.go b/enterprise/coderd/templates_test.go index e66adebca4680..424d4279328cf 100644 --- a/enterprise/coderd/templates_test.go +++ b/enterprise/coderd/templates_test.go @@ -924,7 +924,6 @@ func TestTemplateACL(t *testing.T) { t.Run("everyoneGroup", func(t *testing.T) { t.Parallel() - // TODO: we should not be returning the prebuilds user in TemplateACL, and this is not returned in dbmem. if !dbtestutil.WillUsePostgres() { t.Skip("This test requires postgres") } From e7e9c2739ab7b8565cbe022d0d3c52b5c460e8f4 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Thu, 13 Mar 2025 21:10:30 +0000 Subject: [PATCH 07/72] exclude system user db tests from non-linux OSs Signed-off-by: Danny Kopping --- coderd/users_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/coderd/users_test.go b/coderd/users_test.go index 99b10cba961dd..2f3e39cac9552 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "net/http" + "runtime" "slices" "strings" "testing" @@ -2423,6 +2424,10 @@ func TestSystemUserBehaviour(t *testing.T) { // Setup. t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("skipping because non-linux platforms are tricky to run the mock db container on, and there's no platform-dependence on these tests") + } + ctx := testutil.Context(t, testutil.WaitLong) sqlDB := testSQLDB(t) From 39360476439d434dd71e3dd22ffc2b741e8d8fbe Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 17 Mar 2025 05:13:27 +0000 Subject: [PATCH 08/72] Rename prebuild system user reference --- coderd/prebuilds/id.go | 2 +- coderd/users_test.go | 2 +- enterprise/coderd/groups_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/prebuilds/id.go b/coderd/prebuilds/id.go index bde76e3f7bf14..7c2bbe79b7a6f 100644 --- a/coderd/prebuilds/id.go +++ b/coderd/prebuilds/id.go @@ -2,4 +2,4 @@ package prebuilds import "github.com/google/uuid" -var OwnerID = uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0") +var SystemUserID = uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0") diff --git a/coderd/users_test.go b/coderd/users_test.go index 2f3e39cac9552..0ed98647c820f 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -2460,7 +2460,7 @@ func TestSystemUserBehaviour(t *testing.T) { require.NotNil(t, regularUser) require.True(t, systemUser.IsSystem.Bool) - require.Equal(t, systemUser.ID, prebuilds.OwnerID) + require.Equal(t, systemUser.ID, prebuilds.SystemUserID) require.False(t, regularUser.IsSystem.Bool) require.Equal(t, regularUser.ID, other.ID) diff --git a/enterprise/coderd/groups_test.go b/enterprise/coderd/groups_test.go index 2f6660c9eb280..1f5fb6934ac65 100644 --- a/enterprise/coderd/groups_test.go +++ b/enterprise/coderd/groups_test.go @@ -840,7 +840,7 @@ func TestGroup(t *testing.T) { ctx := testutil.Context(t, testutil.WaitLong) // nolint:gocritic // "This client is operating as the owner user" is fine in this case. - prebuildsUser, err := client.User(ctx, prebuilds.OwnerID.String()) + prebuildsUser, err := client.User(ctx, prebuilds.SystemUserID.String()) require.NoError(t, err) // The 'Everyone' group always has an ID that matches the organization ID. group, err := userAdminClient.Group(ctx, user.OrganizationID) From 8bdcafbba60e9ad74429ba867970fb268d8ebae4 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Mon, 17 Mar 2025 07:45:16 +0000 Subject: [PATCH 09/72] ensure that users.IsSystem is not nullable --- coderd/database/dbmem/dbmem.go | 6 +++--- coderd/database/dump.sql | 2 +- coderd/database/migrations/000301_system_user.up.sql | 2 +- coderd/database/models.go | 2 +- coderd/database/queries.sql.go | 2 +- coderd/users_test.go | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 0c39738069cb0..b899309d3910a 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1555,7 +1555,7 @@ func (q *FakeQuerier) AllUserIDs(_ context.Context, includeSystem bool) ([]uuid. defer q.mutex.RUnlock() userIDs := make([]uuid.UUID, 0, len(q.users)) for idx := range q.users { - if !includeSystem && q.users[idx].IsSystem.Valid && q.users[idx].IsSystem.Bool { + if !includeSystem && q.users[idx].IsSystem { continue } @@ -2656,7 +2656,7 @@ func (q *FakeQuerier) GetActiveUserCount(_ context.Context, includeSystem bool) active := int64(0) for _, u := range q.users { - if !includeSystem && u.IsSystem.Valid && u.IsSystem.Bool { + if !includeSystem && u.IsSystem { continue } @@ -6221,7 +6221,7 @@ func (q *FakeQuerier) GetUserCount(_ context.Context, includeSystem bool) (int64 existing++ } - if !includeSystem && u.IsSystem.Valid && u.IsSystem.Bool { + if !includeSystem && u.IsSystem { continue } } diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 6961b1386e176..359bf5472b119 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -865,7 +865,7 @@ CREATE TABLE users ( github_com_user_id bigint, hashed_one_time_passcode bytea, one_time_passcode_expires_at timestamp with time zone, - is_system boolean DEFAULT false, + is_system boolean DEFAULT false NOT NULL, CONSTRAINT one_time_passcode_set CHECK ((((hashed_one_time_passcode IS NULL) AND (one_time_passcode_expires_at IS NULL)) OR ((hashed_one_time_passcode IS NOT NULL) AND (one_time_passcode_expires_at IS NOT NULL)))) ); diff --git a/coderd/database/migrations/000301_system_user.up.sql b/coderd/database/migrations/000301_system_user.up.sql index 0edb25ef076d6..7be8883de567c 100644 --- a/coderd/database/migrations/000301_system_user.up.sql +++ b/coderd/database/migrations/000301_system_user.up.sql @@ -1,5 +1,5 @@ ALTER TABLE users - ADD COLUMN is_system bool DEFAULT false; + ADD COLUMN is_system bool DEFAULT false NOT NULL; CREATE INDEX user_is_system_idx ON users USING btree (is_system); diff --git a/coderd/database/models.go b/coderd/database/models.go index 8e98510389118..d384b988bd1d7 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3187,7 +3187,7 @@ type User struct { // The time when the one-time-passcode expires. OneTimePasscodeExpiresAt sql.NullTime `db:"one_time_passcode_expires_at" json:"one_time_passcode_expires_at"` // Determines if a user is a system user, and therefore cannot login or perform normal actions - IsSystem sql.NullBool `db:"is_system" json:"is_system"` + IsSystem bool `db:"is_system" json:"is_system"` } type UserConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 2a448ca836c35..8c8288f45f7f2 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -11677,7 +11677,7 @@ type GetUsersRow struct { GithubComUserID sql.NullInt64 `db:"github_com_user_id" json:"github_com_user_id"` HashedOneTimePasscode []byte `db:"hashed_one_time_passcode" json:"hashed_one_time_passcode"` OneTimePasscodeExpiresAt sql.NullTime `db:"one_time_passcode_expires_at" json:"one_time_passcode_expires_at"` - IsSystem sql.NullBool `db:"is_system" json:"is_system"` + IsSystem bool `db:"is_system" json:"is_system"` Count int64 `db:"count" json:"count"` } diff --git a/coderd/users_test.go b/coderd/users_test.go index 0ed98647c820f..ca87ce303c003 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -2450,7 +2450,7 @@ func TestSystemUserBehaviour(t *testing.T) { var systemUser, regularUser database.GetUsersRow for _, u := range users { - if u.IsSystem.Bool { + if u.IsSystem { systemUser = u } else { regularUser = u @@ -2459,9 +2459,9 @@ func TestSystemUserBehaviour(t *testing.T) { require.NotNil(t, systemUser) require.NotNil(t, regularUser) - require.True(t, systemUser.IsSystem.Bool) + require.True(t, systemUser.IsSystem) require.Equal(t, systemUser.ID, prebuilds.SystemUserID) - require.False(t, regularUser.IsSystem.Bool) + require.False(t, regularUser.IsSystem) require.Equal(t, regularUser.ID, other.ID) // ================================================================================================================= @@ -2474,7 +2474,7 @@ func TestSystemUserBehaviour(t *testing.T) { // Then: only regular users are returned. require.NoError(t, err) require.Len(t, users, 1) - require.False(t, users[0].IsSystem.Bool) + require.False(t, users[0].IsSystem) // ================================================================================================================= From 412d198dccf01e78f69f4c13370406944e7040c3 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Wed, 12 Mar 2025 14:15:15 +0000 Subject: [PATCH 10/72] feat: add migrations and queries to support prebuilds Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz.go | 86 +++- coderd/database/dbauthz/dbauthz_test.go | 63 ++- coderd/database/dbgen/dbgen.go | 32 ++ coderd/database/dbmem/dbmem.go | 33 ++ coderd/database/dbmetrics/querymetrics.go | 49 +++ coderd/database/dbmock/dbmock.go | 105 +++++ coderd/database/dump.sql | 165 ++++++++ coderd/database/foreign_key_constraint.go | 1 + coderd/database/lock.go | 2 + .../migrations/000302_prebuilds.down.sql | 4 + .../migrations/000302_prebuilds.up.sql | 58 +++ .../000303_preset_prebuilds.down.sql | 2 + .../migrations/000303_preset_prebuilds.up.sql | 13 + .../fixtures/000303_preset_prebuilds.up.sql | 2 + coderd/database/models.go | 66 ++++ coderd/database/querier.go | 7 + coderd/database/queries.sql.go | 367 ++++++++++++++++++ coderd/database/queries/prebuilds.sql | 120 ++++++ coderd/database/unique_constraint.go | 2 + 19 files changed, 1173 insertions(+), 4 deletions(-) create mode 100644 coderd/database/migrations/000302_prebuilds.down.sql create mode 100644 coderd/database/migrations/000302_prebuilds.up.sql create mode 100644 coderd/database/migrations/000303_preset_prebuilds.down.sql create mode 100644 coderd/database/migrations/000303_preset_prebuilds.up.sql create mode 100644 coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql create mode 100644 coderd/database/queries/prebuilds.sql diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 34c6999a16a0b..72f7b489401a7 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -18,6 +18,7 @@ import ( "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/rbac/rolestore" @@ -358,6 +359,27 @@ var ( }), Scope: rbac.ScopeAll, }.WithCachedASTValue() + + subjectPrebuildsOrchestrator = rbac.Subject{ + FriendlyName: "Prebuilds Orchestrator", + ID: prebuilds.OwnerID.String(), + Roles: rbac.Roles([]rbac.Role{ + { + Identifier: rbac.RoleIdentifier{Name: "prebuilds-orchestrator"}, + DisplayName: "Coder", + Site: rbac.Permissions(map[string][]policy.Action{ + // May use template, read template-related info, & insert template-related resources (preset prebuilds). + rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionUse}, + // May CRUD workspaces, and start/stop them. + rbac.ResourceWorkspace.Type: { + policy.ActionCreate, policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, + policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, + }, + }), + }, + }), + Scope: rbac.ScopeAll, + }.WithCachedASTValue() ) // AsProvisionerd returns a context with an actor that has permissions required @@ -412,6 +434,12 @@ func AsSystemReadProvisionerDaemons(ctx context.Context) context.Context { return context.WithValue(ctx, authContextKey{}, subjectSystemReadProvisionerDaemons) } +// AsPrebuildsOrchestrator returns a context with an actor that has permissions +// to read orchestrator workspace prebuilds. +func AsPrebuildsOrchestrator(ctx context.Context) context.Context { + return context.WithValue(ctx, authContextKey{}, subjectPrebuildsOrchestrator) +} + var AsRemoveActor = rbac.Subject{ ID: "remove-actor", } @@ -1106,6 +1134,15 @@ func (q *querier) BulkMarkNotificationMessagesSent(ctx context.Context, arg data return q.db.BulkMarkNotificationMessagesSent(ctx, arg) } +func (q *querier) ClaimPrebuild(ctx context.Context, newOwnerID database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspace); err != nil { + return database.ClaimPrebuildRow{ + ID: uuid.Nil, + }, err + } + return q.db.ClaimPrebuild(ctx, newOwnerID) +} + func (q *querier) CleanTailnetCoordinators(ctx context.Context) error { if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceTailnetCoordinator); err != nil { return err @@ -2020,6 +2057,20 @@ func (q *querier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUI return q.db.GetParameterSchemasByJobID(ctx, jobID) } +func (q *querier) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuildMetricsRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.GetPrebuildMetrics(ctx) +} + +func (q *querier) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.GetPrebuildsInProgress(ctx) +} + func (q *querier) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceID uuid.UUID) (database.TemplateVersionPreset, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { return database.TemplateVersionPreset{}, err @@ -2037,6 +2088,13 @@ func (q *querier) GetPresetParametersByTemplateVersionID(ctx context.Context, te return q.db.GetPresetParametersByTemplateVersionID(ctx, templateVersionID) } +func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.GetPresetsBackoff(ctx, lookback) +} + func (q *querier) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]database.TemplateVersionPreset, error) { // An actor can read template version presets if they can read the related template version. _, err := q.GetTemplateVersionByID(ctx, templateVersionID) @@ -2088,13 +2146,13 @@ func (q *querier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (data // can read the job. _, err := q.GetWorkspaceBuildByJobID(ctx, id) if err != nil { - return database.ProvisionerJob{}, err + return database.ProvisionerJob{}, xerrors.Errorf("fetch related workspace build: %w", err) } case database.ProvisionerJobTypeTemplateVersionDryRun, database.ProvisionerJobTypeTemplateVersionImport: // Authorized call to get template version. _, err := authorizedTemplateVersionFromJob(ctx, q, job) if err != nil { - return database.ProvisionerJob{}, err + return database.ProvisionerJob{}, xerrors.Errorf("fetch related template version: %w", err) } default: return database.ProvisionerJob{}, xerrors.Errorf("unknown job type: %q", job.Type) @@ -2187,6 +2245,13 @@ func (q *querier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Ti return q.db.GetReplicasUpdatedAfter(ctx, updatedAt) } +func (q *querier) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.GetRunningPrebuilds(ctx) +} + func (q *querier) GetRuntimeConfig(ctx context.Context, key string) (string, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return "", err @@ -2311,6 +2376,16 @@ func (q *querier) GetTemplateParameterInsights(ctx context.Context, arg database return q.db.GetTemplateParameterInsights(ctx, arg) } +func (q *querier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]database.GetTemplatePresetsWithPrebuildsRow, error) { + // Although this fetches presets. It filters them by prebuilds and is only of use to the prebuild system. + // As such, we authorize this in line with other prebuild queries, not with other preset queries. + + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.GetTemplatePresetsWithPrebuilds(ctx, templateID) +} + func (q *querier) GetTemplateUsageStats(ctx context.Context, arg database.GetTemplateUsageStatsParams) ([]database.TemplateUsageStat, error) { if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil { return nil, err @@ -3235,6 +3310,13 @@ func (q *querier) InsertPresetParameters(ctx context.Context, arg database.Inser return q.db.InsertPresetParameters(ctx, arg) } +func (q *querier) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { + if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { + return database.TemplateVersionPresetPrebuild{}, err + } + return q.db.InsertPresetPrebuild(ctx, arg) +} + // TODO: We need to create a ProvisionerJob resource type func (q *querier) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { // if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index ca1379fdbdc84..b02c0cbae32c4 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -979,8 +979,8 @@ func (s *MethodTestSuite) TestOrganization() { }) check.Args(database.OrganizationMembersParams{ - OrganizationID: uuid.UUID{}, - UserID: uuid.UUID{}, + OrganizationID: o.ID, + UserID: u.ID, }).Asserts( mem, policy.ActionRead, ) @@ -4642,6 +4642,65 @@ func (s *MethodTestSuite) TestNotifications() { })) } +func (s *MethodTestSuite) TestPrebuilds() { + s.Run("ClaimPrebuild", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.ClaimPrebuildParams{}). + Asserts(rbac.ResourceWorkspace, policy.ActionUpdate). + ErrorsWithInMemDB(dbmem.ErrUnimplemented). + ErrorsWithPG(sql.ErrNoRows) + })) + s.Run("GetPrebuildMetrics", s.Subtest(func(_ database.Store, check *expects) { + check.Args(). + Asserts(rbac.ResourceTemplate, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) + s.Run("GetPrebuildsInProgress", s.Subtest(func(_ database.Store, check *expects) { + check.Args(). + Asserts(rbac.ResourceTemplate, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) + s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) { + check.Args(time.Time{}). + Asserts(rbac.ResourceTemplate, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) + s.Run("GetRunningPrebuilds", s.Subtest(func(_ database.Store, check *expects) { + check.Args(). + Asserts(rbac.ResourceTemplate, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) + s.Run("GetTemplatePresetsWithPrebuilds", s.Subtest(func(db database.Store, check *expects) { + user := dbgen.User(s.T(), db, database.User{}) + check.Args(uuid.NullUUID{UUID: user.ID, Valid: true}). + Asserts(rbac.ResourceTemplate, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) + s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { + org := dbgen.Organization(s.T(), db, database.Organization{}) + user := dbgen.User(s.T(), db, database.User{}) + template := dbgen.Template(s.T(), db, database.Template{ + CreatedBy: user.ID, + OrganizationID: org.ID, + }) + templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ + TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true}, + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ + Name: coderdtest.RandomName(s.T()), + TemplateVersionID: templateVersion.ID, + }) + check.Args(database.InsertPresetPrebuildParams{ + ID: uuid.New(), + PresetID: preset.ID, + DesiredInstances: 1, + }). + Asserts(rbac.ResourceSystem, policy.ActionCreate). + ErrorsWithInMemDB(dbmem.ErrUnimplemented) + })) +} + func (s *MethodTestSuite) TestOAuth2ProviderApps() { s.Run("GetOAuth2ProviderApps", s.Subtest(func(db database.Store, check *expects) { apps := []database.OAuth2ProviderApp{ diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 97940c1a4b76f..c7df7fe3171ee 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -1158,6 +1158,38 @@ func TelemetryItem(t testing.TB, db database.Store, seed database.TelemetryItem) return item } +func Preset(t testing.TB, db database.Store, seed database.InsertPresetParams) database.TemplateVersionPreset { + preset, err := db.InsertPreset(genCtx, database.InsertPresetParams{ + TemplateVersionID: takeFirst(seed.TemplateVersionID, uuid.New()), + Name: takeFirst(seed.Name, testutil.GetRandomName(t)), + CreatedAt: takeFirst(seed.CreatedAt, dbtime.Now()), + }) + require.NoError(t, err, "insert preset") + return preset +} + +func PresetParameter(t testing.TB, db database.Store, seed database.InsertPresetParametersParams) []database.TemplateVersionPresetParameter { + parameters, err := db.InsertPresetParameters(genCtx, database.InsertPresetParametersParams{ + TemplateVersionPresetID: takeFirst(seed.TemplateVersionPresetID, uuid.New()), + Names: takeFirstSlice(seed.Names, []string{testutil.GetRandomName(t)}), + Values: takeFirstSlice(seed.Values, []string{testutil.GetRandomName(t)}), + }) + + require.NoError(t, err, "insert preset parameters") + return parameters +} + +func PresetPrebuild(t testing.TB, db database.Store, seed database.InsertPresetPrebuildParams) database.TemplateVersionPresetPrebuild { + prebuild, err := db.InsertPresetPrebuild(genCtx, database.InsertPresetPrebuildParams{ + ID: takeFirst(seed.ID, uuid.New()), + PresetID: takeFirst(seed.PresetID, uuid.New()), + DesiredInstances: takeFirst(seed.DesiredInstances, 1), + InvalidateAfterSecs: 0, + }) + require.NoError(t, err, "insert preset prebuild") + return prebuild +} + func provisionerJobTiming(t testing.TB, db database.Store, seed database.ProvisionerJobTiming) database.ProvisionerJobTiming { timing, err := db.InsertProvisionerJobTimings(genCtx, database.InsertProvisionerJobTimingsParams{ JobID: takeFirst(seed.JobID, uuid.New()), diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index b899309d3910a..5b05951fc264a 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1714,6 +1714,10 @@ func (*FakeQuerier) BulkMarkNotificationMessagesSent(_ context.Context, arg data return int64(len(arg.IDs)), nil } +func (*FakeQuerier) ClaimPrebuild(_ context.Context, _ database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { + return database.ClaimPrebuildRow{}, ErrUnimplemented +} + func (*FakeQuerier) CleanTailnetCoordinators(_ context.Context) error { return ErrUnimplemented } @@ -4023,6 +4027,14 @@ func (q *FakeQuerier) GetParameterSchemasByJobID(_ context.Context, jobID uuid.U return parameters, nil } +func (*FakeQuerier) GetPrebuildMetrics(_ context.Context) ([]database.GetPrebuildMetricsRow, error) { + return nil, ErrUnimplemented +} + +func (*FakeQuerier) GetPrebuildsInProgress(_ context.Context) ([]database.GetPrebuildsInProgressRow, error) { + return nil, ErrUnimplemented +} + func (q *FakeQuerier) GetPresetByWorkspaceBuildID(_ context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -4065,6 +4077,10 @@ func (q *FakeQuerier) GetPresetParametersByTemplateVersionID(_ context.Context, return parameters, nil } +func (*FakeQuerier) GetPresetsBackoff(_ context.Context, _ time.Time) ([]database.GetPresetsBackoffRow, error) { + return nil, ErrUnimplemented +} + func (q *FakeQuerier) GetPresetsByTemplateVersionID(_ context.Context, templateVersionID uuid.UUID) ([]database.TemplateVersionPreset, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -4728,6 +4744,10 @@ func (q *FakeQuerier) GetReplicasUpdatedAfter(_ context.Context, updatedAt time. return replicas, nil } +func (*FakeQuerier) GetRunningPrebuilds(_ context.Context) ([]database.GetRunningPrebuildsRow, error) { + return nil, ErrUnimplemented +} + func (q *FakeQuerier) GetRuntimeConfig(_ context.Context, key string) (string, error) { q.mutex.Lock() defer q.mutex.Unlock() @@ -5767,6 +5787,10 @@ func (q *FakeQuerier) GetTemplateParameterInsights(ctx context.Context, arg data return rows, nil } +func (*FakeQuerier) GetTemplatePresetsWithPrebuilds(_ context.Context, _ uuid.NullUUID) ([]database.GetTemplatePresetsWithPrebuildsRow, error) { + return nil, ErrUnimplemented +} + func (q *FakeQuerier) GetTemplateUsageStats(_ context.Context, arg database.GetTemplateUsageStatsParams) ([]database.TemplateUsageStat, error) { err := validateDatabaseType(arg) if err != nil { @@ -8542,6 +8566,15 @@ func (q *FakeQuerier) InsertPresetParameters(_ context.Context, arg database.Ins return presetParameters, nil } +func (*FakeQuerier) InsertPresetPrebuild(_ context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { + err := validateDatabaseType(arg) + if err != nil { + return database.TemplateVersionPresetPrebuild{}, err + } + + return database.TemplateVersionPresetPrebuild{}, ErrUnimplemented +} + func (q *FakeQuerier) InsertProvisionerJob(_ context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { if err := validateDatabaseType(arg); err != nil { return database.ProvisionerJob{}, err diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index bae68852bbf2b..bd3480a2ff31c 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -158,6 +158,13 @@ func (m queryMetricsStore) BulkMarkNotificationMessagesSent(ctx context.Context, return r0, r1 } +func (m queryMetricsStore) ClaimPrebuild(ctx context.Context, newOwnerID database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { + start := time.Now() + r0, r1 := m.s.ClaimPrebuild(ctx, newOwnerID) + m.queryLatencies.WithLabelValues("ClaimPrebuild").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) CleanTailnetCoordinators(ctx context.Context) error { start := time.Now() err := m.s.CleanTailnetCoordinators(ctx) @@ -1033,6 +1040,20 @@ func (m queryMetricsStore) GetParameterSchemasByJobID(ctx context.Context, jobID return schemas, err } +func (m queryMetricsStore) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuildMetricsRow, error) { + start := time.Now() + r0, r1 := m.s.GetPrebuildMetrics(ctx) + m.queryLatencies.WithLabelValues("GetPrebuildMetrics").Observe(time.Since(start).Seconds()) + return r0, r1 +} + +func (m queryMetricsStore) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { + start := time.Now() + r0, r1 := m.s.GetPrebuildsInProgress(ctx) + m.queryLatencies.WithLabelValues("GetPrebuildsInProgress").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { start := time.Now() r0, r1 := m.s.GetPresetByWorkspaceBuildID(ctx, workspaceBuildID) @@ -1047,6 +1068,13 @@ func (m queryMetricsStore) GetPresetParametersByTemplateVersionID(ctx context.Co return r0, r1 } +func (m queryMetricsStore) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) { + start := time.Now() + r0, r1 := m.s.GetPresetsBackoff(ctx, lookback) + m.queryLatencies.WithLabelValues("GetPresetsBackoff").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]database.TemplateVersionPreset, error) { start := time.Now() r0, r1 := m.s.GetPresetsByTemplateVersionID(ctx, templateVersionID) @@ -1180,6 +1208,13 @@ func (m queryMetricsStore) GetReplicasUpdatedAfter(ctx context.Context, updatedA return replicas, err } +func (m queryMetricsStore) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { + start := time.Now() + r0, r1 := m.s.GetRunningPrebuilds(ctx) + m.queryLatencies.WithLabelValues("GetRunningPrebuilds").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetRuntimeConfig(ctx context.Context, key string) (string, error) { start := time.Now() r0, r1 := m.s.GetRuntimeConfig(ctx, key) @@ -1306,6 +1341,13 @@ func (m queryMetricsStore) GetTemplateParameterInsights(ctx context.Context, arg return r0, r1 } +func (m queryMetricsStore) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]database.GetTemplatePresetsWithPrebuildsRow, error) { + start := time.Now() + r0, r1 := m.s.GetTemplatePresetsWithPrebuilds(ctx, templateID) + m.queryLatencies.WithLabelValues("GetTemplatePresetsWithPrebuilds").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetTemplateUsageStats(ctx context.Context, arg database.GetTemplateUsageStatsParams) ([]database.TemplateUsageStat, error) { start := time.Now() r0, r1 := m.s.GetTemplateUsageStats(ctx, arg) @@ -2013,6 +2055,13 @@ func (m queryMetricsStore) InsertPresetParameters(ctx context.Context, arg datab return r0, r1 } +func (m queryMetricsStore) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { + start := time.Now() + r0, r1 := m.s.InsertPresetPrebuild(ctx, arg) + m.queryLatencies.WithLabelValues("InsertPresetPrebuild").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { start := time.Now() job, err := m.s.InsertProvisionerJob(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index b2e8eabe4594e..803be16000abf 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -190,6 +190,21 @@ func (mr *MockStoreMockRecorder) BulkMarkNotificationMessagesSent(ctx, arg any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BulkMarkNotificationMessagesSent", reflect.TypeOf((*MockStore)(nil).BulkMarkNotificationMessagesSent), ctx, arg) } +// ClaimPrebuild mocks base method. +func (m *MockStore) ClaimPrebuild(ctx context.Context, arg database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClaimPrebuild", ctx, arg) + ret0, _ := ret[0].(database.ClaimPrebuildRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ClaimPrebuild indicates an expected call of ClaimPrebuild. +func (mr *MockStoreMockRecorder) ClaimPrebuild(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimPrebuild", reflect.TypeOf((*MockStore)(nil).ClaimPrebuild), ctx, arg) +} + // CleanTailnetCoordinators mocks base method. func (m *MockStore) CleanTailnetCoordinators(ctx context.Context) error { m.ctrl.T.Helper() @@ -2107,6 +2122,36 @@ func (mr *MockStoreMockRecorder) GetParameterSchemasByJobID(ctx, jobID any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetParameterSchemasByJobID", reflect.TypeOf((*MockStore)(nil).GetParameterSchemasByJobID), ctx, jobID) } +// GetPrebuildMetrics mocks base method. +func (m *MockStore) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuildMetricsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrebuildMetrics", ctx) + ret0, _ := ret[0].([]database.GetPrebuildMetricsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrebuildMetrics indicates an expected call of GetPrebuildMetrics. +func (mr *MockStoreMockRecorder) GetPrebuildMetrics(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrebuildMetrics", reflect.TypeOf((*MockStore)(nil).GetPrebuildMetrics), ctx) +} + +// GetPrebuildsInProgress mocks base method. +func (m *MockStore) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPrebuildsInProgress", ctx) + ret0, _ := ret[0].([]database.GetPrebuildsInProgressRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPrebuildsInProgress indicates an expected call of GetPrebuildsInProgress. +func (mr *MockStoreMockRecorder) GetPrebuildsInProgress(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrebuildsInProgress", reflect.TypeOf((*MockStore)(nil).GetPrebuildsInProgress), ctx) +} + // GetPresetByWorkspaceBuildID mocks base method. func (m *MockStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { m.ctrl.T.Helper() @@ -2137,6 +2182,21 @@ func (mr *MockStoreMockRecorder) GetPresetParametersByTemplateVersionID(ctx, tem return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPresetParametersByTemplateVersionID", reflect.TypeOf((*MockStore)(nil).GetPresetParametersByTemplateVersionID), ctx, templateVersionID) } +// GetPresetsBackoff mocks base method. +func (m *MockStore) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPresetsBackoff", ctx, lookback) + ret0, _ := ret[0].([]database.GetPresetsBackoffRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPresetsBackoff indicates an expected call of GetPresetsBackoff. +func (mr *MockStoreMockRecorder) GetPresetsBackoff(ctx, lookback any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPresetsBackoff", reflect.TypeOf((*MockStore)(nil).GetPresetsBackoff), ctx, lookback) +} + // GetPresetsByTemplateVersionID mocks base method. func (m *MockStore) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]database.TemplateVersionPreset, error) { m.ctrl.T.Helper() @@ -2422,6 +2482,21 @@ func (mr *MockStoreMockRecorder) GetReplicasUpdatedAfter(ctx, updatedAt any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReplicasUpdatedAfter", reflect.TypeOf((*MockStore)(nil).GetReplicasUpdatedAfter), ctx, updatedAt) } +// GetRunningPrebuilds mocks base method. +func (m *MockStore) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRunningPrebuilds", ctx) + ret0, _ := ret[0].([]database.GetRunningPrebuildsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRunningPrebuilds indicates an expected call of GetRunningPrebuilds. +func (mr *MockStoreMockRecorder) GetRunningPrebuilds(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPrebuilds", reflect.TypeOf((*MockStore)(nil).GetRunningPrebuilds), ctx) +} + // GetRuntimeConfig mocks base method. func (m *MockStore) GetRuntimeConfig(ctx context.Context, key string) (string, error) { m.ctrl.T.Helper() @@ -2707,6 +2782,21 @@ func (mr *MockStoreMockRecorder) GetTemplateParameterInsights(ctx, arg any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplateParameterInsights", reflect.TypeOf((*MockStore)(nil).GetTemplateParameterInsights), ctx, arg) } +// GetTemplatePresetsWithPrebuilds mocks base method. +func (m *MockStore) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]database.GetTemplatePresetsWithPrebuildsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTemplatePresetsWithPrebuilds", ctx, templateID) + ret0, _ := ret[0].([]database.GetTemplatePresetsWithPrebuildsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTemplatePresetsWithPrebuilds indicates an expected call of GetTemplatePresetsWithPrebuilds. +func (mr *MockStoreMockRecorder) GetTemplatePresetsWithPrebuilds(ctx, templateID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTemplatePresetsWithPrebuilds", reflect.TypeOf((*MockStore)(nil).GetTemplatePresetsWithPrebuilds), ctx, templateID) +} + // GetTemplateUsageStats mocks base method. func (m *MockStore) GetTemplateUsageStats(ctx context.Context, arg database.GetTemplateUsageStatsParams) ([]database.TemplateUsageStat, error) { m.ctrl.T.Helper() @@ -4247,6 +4337,21 @@ func (mr *MockStoreMockRecorder) InsertPresetParameters(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertPresetParameters", reflect.TypeOf((*MockStore)(nil).InsertPresetParameters), ctx, arg) } +// InsertPresetPrebuild mocks base method. +func (m *MockStore) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertPresetPrebuild", ctx, arg) + ret0, _ := ret[0].(database.TemplateVersionPresetPrebuild) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InsertPresetPrebuild indicates an expected call of InsertPresetPrebuild. +func (mr *MockStoreMockRecorder) InsertPresetPrebuild(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertPresetPrebuild", reflect.TypeOf((*MockStore)(nil).InsertPresetPrebuild), ctx, arg) +} + // InsertProvisionerJob mocks base method. func (m *MockStore) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 359bf5472b119..b5936f0c49168 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1382,6 +1382,13 @@ CREATE TABLE template_version_preset_parameters ( value text NOT NULL ); +CREATE TABLE template_version_preset_prebuilds ( + id uuid NOT NULL, + preset_id uuid NOT NULL, + desired_instances integer NOT NULL, + invalidate_after_secs integer DEFAULT 0 +); + CREATE TABLE template_version_presets ( id uuid DEFAULT gen_random_uuid() NOT NULL, template_version_id uuid NOT NULL, @@ -1892,6 +1899,30 @@ CREATE VIEW workspace_build_with_user AS COMMENT ON VIEW workspace_build_with_user IS 'Joins in the username + avatar url of the initiated by user.'; +CREATE VIEW workspace_latest_build AS + SELECT wb.id, + wb.created_at, + wb.updated_at, + wb.workspace_id, + wb.template_version_id, + wb.build_number, + wb.transition, + wb.initiator_id, + wb.provisioner_state, + wb.job_id, + wb.deadline, + wb.reason, + wb.daily_cost, + wb.max_deadline, + wb.template_version_preset_id + FROM (( SELECT tv.template_id, + wbmax_1.workspace_id, + max(wbmax_1.build_number) AS max_build_number + FROM (workspace_builds wbmax_1 + JOIN template_versions tv ON ((tv.id = wbmax_1.template_version_id))) + GROUP BY tv.template_id, wbmax_1.workspace_id) wbmax + JOIN workspace_builds wb ON (((wb.workspace_id = wbmax.workspace_id) AND (wb.build_number = wbmax.max_build_number)))); + CREATE TABLE workspace_modules ( id uuid NOT NULL, job_id uuid NOT NULL, @@ -1902,6 +1933,48 @@ CREATE TABLE workspace_modules ( created_at timestamp with time zone NOT NULL ); +CREATE VIEW workspace_prebuild_builds AS + SELECT workspace_builds.id, + workspace_builds.created_at, + workspace_builds.updated_at, + workspace_builds.workspace_id, + workspace_builds.template_version_id, + workspace_builds.build_number, + workspace_builds.transition, + workspace_builds.initiator_id, + workspace_builds.provisioner_state, + workspace_builds.job_id, + workspace_builds.deadline, + workspace_builds.reason, + workspace_builds.daily_cost, + workspace_builds.max_deadline, + workspace_builds.template_version_preset_id + FROM workspace_builds + WHERE (workspace_builds.initiator_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid); + +CREATE VIEW workspace_prebuilds AS +SELECT + NULL::uuid AS id, + NULL::timestamp with time zone AS created_at, + NULL::timestamp with time zone AS updated_at, + NULL::uuid AS owner_id, + NULL::uuid AS organization_id, + NULL::uuid AS template_id, + NULL::boolean AS deleted, + NULL::character varying(64) AS name, + NULL::text AS autostart_schedule, + NULL::bigint AS ttl, + NULL::timestamp with time zone AS last_used_at, + NULL::timestamp with time zone AS dormant_at, + NULL::timestamp with time zone AS deleting_at, + NULL::automatic_updates AS automatic_updates, + NULL::boolean AS favorite, + NULL::timestamp with time zone AS next_start_at, + NULL::uuid AS agent_id, + NULL::workspace_agent_lifecycle_state AS lifecycle_state, + NULL::timestamp with time zone AS ready_at, + NULL::uuid AS current_preset_id; + CREATE TABLE workspace_proxies ( id uuid NOT NULL, name text NOT NULL, @@ -2198,6 +2271,9 @@ ALTER TABLE ONLY template_version_parameters ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id); +ALTER TABLE ONLY template_version_preset_prebuilds + ADD CONSTRAINT template_version_preset_prebuilds_pkey PRIMARY KEY (id); + ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id); @@ -2348,6 +2424,8 @@ CREATE INDEX idx_tailnet_tunnels_dst_id ON tailnet_tunnels USING hash (dst_id); CREATE INDEX idx_tailnet_tunnels_src_id ON tailnet_tunnels USING hash (src_id); +CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets USING btree (name, template_version_id); + CREATE INDEX idx_user_deleted_deleted_at ON user_deleted USING btree (deleted_at); CREATE INDEX idx_user_status_changes_changed_at ON user_status_changes USING btree (changed_at); @@ -2464,6 +2542,90 @@ CREATE OR REPLACE VIEW provisioner_job_stats AS LEFT JOIN provisioner_job_timings pjt ON ((pjt.job_id = pj.id))) GROUP BY pj.id, wb.workspace_id; +CREATE OR REPLACE VIEW workspace_prebuilds AS + WITH all_prebuilds AS ( + SELECT w.id, + w.created_at, + w.updated_at, + w.owner_id, + w.organization_id, + w.template_id, + w.deleted, + w.name, + w.autostart_schedule, + w.ttl, + w.last_used_at, + w.dormant_at, + w.deleting_at, + w.automatic_updates, + w.favorite, + w.next_start_at + FROM workspaces w + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + ), workspace_agents AS ( + SELECT w.id AS workspace_id, + wa.id AS agent_id, + wa.lifecycle_state, + wa.ready_at + FROM (((workspaces w + JOIN workspace_latest_build wlb ON ((wlb.workspace_id = w.id))) + JOIN workspace_resources wr ON ((wr.job_id = wlb.job_id))) + JOIN workspace_agents wa ON ((wa.resource_id = wr.id))) + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + GROUP BY w.id, wa.id + ), current_presets AS ( + SELECT w.id AS prebuild_id, + lps.template_version_preset_id + FROM (workspaces w + JOIN ( SELECT wb.id, + wb.created_at, + wb.updated_at, + wb.workspace_id, + wb.template_version_id, + wb.build_number, + wb.transition, + wb.initiator_id, + wb.provisioner_state, + wb.job_id, + wb.deadline, + wb.reason, + wb.daily_cost, + wb.max_deadline, + wb.template_version_preset_id + FROM (( SELECT tv.template_id, + wbmax_1.workspace_id, + max(wbmax_1.build_number) AS max_build_number + FROM (workspace_builds wbmax_1 + JOIN template_versions tv ON ((tv.id = wbmax_1.template_version_id))) + WHERE (wbmax_1.template_version_preset_id IS NOT NULL) + GROUP BY tv.template_id, wbmax_1.workspace_id) wbmax + JOIN workspace_builds wb ON (((wb.workspace_id = wbmax.workspace_id) AND (wb.build_number = wbmax.max_build_number))))) lps ON ((lps.workspace_id = w.id))) + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + ) + SELECT p.id, + p.created_at, + p.updated_at, + p.owner_id, + p.organization_id, + p.template_id, + p.deleted, + p.name, + p.autostart_schedule, + p.ttl, + p.last_used_at, + p.dormant_at, + p.deleting_at, + p.automatic_updates, + p.favorite, + p.next_start_at, + a.agent_id, + a.lifecycle_state, + a.ready_at, + cp.template_version_preset_id AS current_preset_id + FROM ((all_prebuilds p + LEFT JOIN workspace_agents a ON ((a.workspace_id = p.id))) + JOIN current_presets cp ON ((cp.prebuild_id = p.id))); + CREATE TRIGGER inhibit_enqueue_if_disabled BEFORE INSERT ON notification_messages FOR EACH ROW EXECUTE FUNCTION inhibit_enqueue_if_disabled(); CREATE TRIGGER prevent_system_user_deletions BEFORE DELETE ON users FOR EACH ROW WHEN ((old.is_system = true)) EXECUTE FUNCTION prevent_system_user_changes(); @@ -2615,6 +2777,9 @@ ALTER TABLE ONLY template_version_parameters ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; +ALTER TABLE ONLY template_version_preset_prebuilds + ADD CONSTRAINT template_version_preset_prebuilds_preset_id_fkey FOREIGN KEY (preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; + ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; diff --git a/coderd/database/foreign_key_constraint.go b/coderd/database/foreign_key_constraint.go index f7044815852cd..f762141505b4d 100644 --- a/coderd/database/foreign_key_constraint.go +++ b/coderd/database/foreign_key_constraint.go @@ -43,6 +43,7 @@ const ( ForeignKeyTailnetTunnelsCoordinatorID ForeignKeyConstraint = "tailnet_tunnels_coordinator_id_fkey" // ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_coordinator_id_fkey FOREIGN KEY (coordinator_id) REFERENCES tailnet_coordinators(id) ON DELETE CASCADE; ForeignKeyTemplateVersionParametersTemplateVersionID ForeignKeyConstraint = "template_version_parameters_template_version_id_fkey" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetParametTemplateVersionPresetID ForeignKeyConstraint = "template_version_preset_paramet_template_version_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; + ForeignKeyTemplateVersionPresetPrebuildsPresetID ForeignKeyConstraint = "template_version_preset_prebuilds_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_prebuilds ADD CONSTRAINT template_version_preset_prebuilds_preset_id_fkey FOREIGN KEY (preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetsTemplateVersionID ForeignKeyConstraint = "template_version_presets_template_version_id_fkey" // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionVariablesTemplateVersionID ForeignKeyConstraint = "template_version_variables_template_version_id_fkey" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionWorkspaceTagsTemplateVersionID ForeignKeyConstraint = "template_version_workspace_tags_template_version_id_fkey" // ALTER TABLE ONLY template_version_workspace_tags ADD CONSTRAINT template_version_workspace_tags_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; diff --git a/coderd/database/lock.go b/coderd/database/lock.go index 0bc8b2a75d001..a0d0a92be5325 100644 --- a/coderd/database/lock.go +++ b/coderd/database/lock.go @@ -12,6 +12,8 @@ const ( LockIDDBPurge LockIDNotificationsReportGenerator LockIDCryptoKeyRotation + LockIDReconcileTemplatePrebuilds + LockIDDeterminePrebuildsState ) // GenLockID generates a unique and consistent lock ID from a given string. diff --git a/coderd/database/migrations/000302_prebuilds.down.sql b/coderd/database/migrations/000302_prebuilds.down.sql new file mode 100644 index 0000000000000..251cb657a0d0d --- /dev/null +++ b/coderd/database/migrations/000302_prebuilds.down.sql @@ -0,0 +1,4 @@ +-- Revert prebuild views +DROP VIEW IF EXISTS workspace_prebuild_builds; +DROP VIEW IF EXISTS workspace_prebuilds; +DROP VIEW IF EXISTS workspace_latest_build; diff --git a/coderd/database/migrations/000302_prebuilds.up.sql b/coderd/database/migrations/000302_prebuilds.up.sql new file mode 100644 index 0000000000000..ed673b511efe1 --- /dev/null +++ b/coderd/database/migrations/000302_prebuilds.up.sql @@ -0,0 +1,58 @@ +CREATE VIEW workspace_latest_build AS +SELECT wb.* +FROM (SELECT tv.template_id, + wbmax.workspace_id, + MAX(wbmax.build_number) as max_build_number + FROM workspace_builds wbmax + JOIN template_versions tv ON (tv.id = wbmax.template_version_id) + GROUP BY tv.template_id, wbmax.workspace_id) wbmax + JOIN workspace_builds wb ON ( + wb.workspace_id = wbmax.workspace_id + AND wb.build_number = wbmax.max_build_number + ); + +CREATE VIEW workspace_prebuilds AS +WITH + -- All workspaces owned by the "prebuilds" user. + all_prebuilds AS (SELECT w.* + FROM workspaces w + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'), -- The system user responsible for prebuilds. + -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. + workspace_agents AS (SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at + FROM workspaces w + INNER JOIN workspace_latest_build wlb ON wlb.workspace_id = w.id + INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id + INNER JOIN workspace_agents wa ON wa.resource_id = wr.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. + GROUP BY w.id, wa.id), + -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the + -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, + -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. + -- + -- See https://github.com/coder/internal/issues/398 + current_presets AS (SELECT w.id AS prebuild_id, lps.template_version_preset_id + FROM workspaces w + INNER JOIN ( + -- The latest workspace build which had a preset explicitly selected + SELECT wb.* + FROM (SELECT tv.template_id, + wbmax.workspace_id, + MAX(wbmax.build_number) as max_build_number + FROM workspace_builds wbmax + JOIN template_versions tv ON (tv.id = wbmax.template_version_id) + WHERE wbmax.template_version_preset_id IS NOT NULL + GROUP BY tv.template_id, wbmax.workspace_id) wbmax + JOIN workspace_builds wb ON ( + wb.workspace_id = wbmax.workspace_id + AND wb.build_number = wbmax.max_build_number + )) lps ON lps.workspace_id = w.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. +SELECT p.*, a.agent_id, a.lifecycle_state, a.ready_at, cp.template_version_preset_id AS current_preset_id +FROM all_prebuilds p + LEFT JOIN workspace_agents a ON a.workspace_id = p.id + INNER JOIN current_presets cp ON cp.prebuild_id = p.id; + +CREATE VIEW workspace_prebuild_builds AS +SELECT * +FROM workspace_builds +WHERE initiator_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; -- The system user responsible for prebuilds. diff --git a/coderd/database/migrations/000303_preset_prebuilds.down.sql b/coderd/database/migrations/000303_preset_prebuilds.down.sql new file mode 100644 index 0000000000000..5e949be505020 --- /dev/null +++ b/coderd/database/migrations/000303_preset_prebuilds.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS template_version_preset_prebuilds; +DROP INDEX IF EXISTS idx_unique_preset_name; diff --git a/coderd/database/migrations/000303_preset_prebuilds.up.sql b/coderd/database/migrations/000303_preset_prebuilds.up.sql new file mode 100644 index 0000000000000..f28607bbaf3a7 --- /dev/null +++ b/coderd/database/migrations/000303_preset_prebuilds.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE template_version_preset_prebuilds +( + id UUID PRIMARY KEY, + preset_id UUID NOT NULL, + desired_instances INT NOT NULL, + invalidate_after_secs INT NULL DEFAULT 0, + + -- Deletion should never occur, but if we allow it we should check no prebuilds exist attached to this preset first. + FOREIGN KEY (preset_id) REFERENCES template_version_presets (id) ON DELETE CASCADE +); + +-- We should not be able to have presets with the same name for a particular template version. +CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets (name, template_version_id); diff --git a/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql new file mode 100644 index 0000000000000..1bceed871dbdc --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql @@ -0,0 +1,2 @@ +INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances) +VALUES (gen_random_uuid(), '28b42cc0-c4fe-4907-a0fe-e4d20f1e9bfe', 1); diff --git a/coderd/database/models.go b/coderd/database/models.go index d384b988bd1d7..ef3209ca2707c 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3121,6 +3121,13 @@ type TemplateVersionPresetParameter struct { Value string `db:"value" json:"value"` } +type TemplateVersionPresetPrebuild struct { + ID uuid.UUID `db:"id" json:"id"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` + InvalidateAfterSecs sql.NullInt32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` +} + type TemplateVersionTable struct { ID uuid.UUID `db:"id" json:"id"` TemplateID uuid.NullUUID `db:"template_id" json:"template_id"` @@ -3507,6 +3514,24 @@ type WorkspaceBuildTable struct { TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` } +type WorkspaceLatestBuild struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + BuildNumber int32 `db:"build_number" json:"build_number"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + InitiatorID uuid.UUID `db:"initiator_id" json:"initiator_id"` + ProvisionerState []byte `db:"provisioner_state" json:"provisioner_state"` + JobID uuid.UUID `db:"job_id" json:"job_id"` + Deadline time.Time `db:"deadline" json:"deadline"` + Reason BuildReason `db:"reason" json:"reason"` + DailyCost int32 `db:"daily_cost" json:"daily_cost"` + MaxDeadline time.Time `db:"max_deadline" json:"max_deadline"` + TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` +} + type WorkspaceModule struct { ID uuid.UUID `db:"id" json:"id"` JobID uuid.UUID `db:"job_id" json:"job_id"` @@ -3517,6 +3542,47 @@ type WorkspaceModule struct { CreatedAt time.Time `db:"created_at" json:"created_at"` } +type WorkspacePrebuild struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + OwnerID uuid.UUID `db:"owner_id" json:"owner_id"` + OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + Deleted bool `db:"deleted" json:"deleted"` + Name string `db:"name" json:"name"` + AutostartSchedule sql.NullString `db:"autostart_schedule" json:"autostart_schedule"` + Ttl sql.NullInt64 `db:"ttl" json:"ttl"` + LastUsedAt time.Time `db:"last_used_at" json:"last_used_at"` + DormantAt sql.NullTime `db:"dormant_at" json:"dormant_at"` + DeletingAt sql.NullTime `db:"deleting_at" json:"deleting_at"` + AutomaticUpdates AutomaticUpdates `db:"automatic_updates" json:"automatic_updates"` + Favorite bool `db:"favorite" json:"favorite"` + NextStartAt sql.NullTime `db:"next_start_at" json:"next_start_at"` + AgentID uuid.NullUUID `db:"agent_id" json:"agent_id"` + LifecycleState NullWorkspaceAgentLifecycleState `db:"lifecycle_state" json:"lifecycle_state"` + ReadyAt sql.NullTime `db:"ready_at" json:"ready_at"` + CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` +} + +type WorkspacePrebuildBuild struct { + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + BuildNumber int32 `db:"build_number" json:"build_number"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + InitiatorID uuid.UUID `db:"initiator_id" json:"initiator_id"` + ProvisionerState []byte `db:"provisioner_state" json:"provisioner_state"` + JobID uuid.UUID `db:"job_id" json:"job_id"` + Deadline time.Time `db:"deadline" json:"deadline"` + Reason BuildReason `db:"reason" json:"reason"` + DailyCost int32 `db:"daily_cost" json:"daily_cost"` + MaxDeadline time.Time `db:"max_deadline" json:"max_deadline"` + TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` +} + type WorkspaceProxy struct { ID uuid.UUID `db:"id" json:"id"` Name string `db:"name" json:"name"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 0de928932652b..91a64f6f6f1ef 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -60,6 +60,7 @@ type sqlcQuerier interface { BatchUpdateWorkspaceNextStartAt(ctx context.Context, arg BatchUpdateWorkspaceNextStartAtParams) error BulkMarkNotificationMessagesFailed(ctx context.Context, arg BulkMarkNotificationMessagesFailedParams) (int64, error) BulkMarkNotificationMessagesSent(ctx context.Context, arg BulkMarkNotificationMessagesSentParams) (int64, error) + ClaimPrebuild(ctx context.Context, arg ClaimPrebuildParams) (ClaimPrebuildRow, error) CleanTailnetCoordinators(ctx context.Context) error CleanTailnetLostPeers(ctx context.Context) error CleanTailnetTunnels(ctx context.Context) error @@ -220,8 +221,11 @@ type sqlcQuerier interface { GetOrganizations(ctx context.Context, arg GetOrganizationsParams) ([]Organization, error) GetOrganizationsByUserID(ctx context.Context, arg GetOrganizationsByUserIDParams) ([]Organization, error) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) + GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) + GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuildsInProgressRow, error) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) + GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPreset, error) GetPreviousTemplateVersion(ctx context.Context, arg GetPreviousTemplateVersionParams) (TemplateVersion, error) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDaemon, error) @@ -243,6 +247,7 @@ type sqlcQuerier interface { GetQuotaConsumedForUser(ctx context.Context, arg GetQuotaConsumedForUserParams) (int64, error) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) + GetRunningPrebuilds(ctx context.Context) ([]GetRunningPrebuildsRow, error) GetRuntimeConfig(ctx context.Context, key string) (string, error) GetTailnetAgents(ctx context.Context, id uuid.UUID) ([]TailnetAgent, error) GetTailnetClientsForAgent(ctx context.Context, agentID uuid.UUID) ([]TailnetClient, error) @@ -285,6 +290,7 @@ type sqlcQuerier interface { // created in the timeframe and return the aggregate usage counts of parameter // values. GetTemplateParameterInsights(ctx context.Context, arg GetTemplateParameterInsightsParams) ([]GetTemplateParameterInsightsRow, error) + GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) GetTemplateUsageStats(ctx context.Context, arg GetTemplateUsageStatsParams) ([]TemplateUsageStat, error) GetTemplateVersionByID(ctx context.Context, id uuid.UUID) (TemplateVersion, error) GetTemplateVersionByJobID(ctx context.Context, jobID uuid.UUID) (TemplateVersion, error) @@ -431,6 +437,7 @@ type sqlcQuerier interface { InsertOrganizationMember(ctx context.Context, arg InsertOrganizationMemberParams) (OrganizationMember, error) InsertPreset(ctx context.Context, arg InsertPresetParams) (TemplateVersionPreset, error) InsertPresetParameters(ctx context.Context, arg InsertPresetParametersParams) ([]TemplateVersionPresetParameter, error) + InsertPresetPrebuild(ctx context.Context, arg InsertPresetPrebuildParams) (TemplateVersionPresetPrebuild, error) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) InsertProvisionerJobTimings(ctx context.Context, arg InsertProvisionerJobTimingsParams) ([]ProvisionerJobTiming, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 8c8288f45f7f2..4cec4e1e93ad6 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5750,6 +5750,373 @@ func (q *sqlQuerier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid. return items, nil } +const claimPrebuild = `-- name: ClaimPrebuild :one +UPDATE workspaces w +SET owner_id = $1::uuid, + name = $2::text, + updated_at = NOW() +WHERE w.id IN (SELECT p.id + FROM workspace_prebuilds p + INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status IN ('succeeded'::provisioner_job_status)) + AND b.template_version_id = t.active_version_id + AND b.template_version_preset_id = $3::uuid + AND p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state + ORDER BY random() + LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. +RETURNING w.id, w.name +` + +type ClaimPrebuildParams struct { + NewUserID uuid.UUID `db:"new_user_id" json:"new_user_id"` + NewName string `db:"new_name" json:"new_name"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` +} + +type ClaimPrebuildRow struct { + ID uuid.UUID `db:"id" json:"id"` + Name string `db:"name" json:"name"` +} + +func (q *sqlQuerier) ClaimPrebuild(ctx context.Context, arg ClaimPrebuildParams) (ClaimPrebuildRow, error) { + row := q.db.QueryRowContext(ctx, claimPrebuild, arg.NewUserID, arg.NewName, arg.PresetID) + var i ClaimPrebuildRow + err := row.Scan(&i.ID, &i.Name) + return i, err +} + +const getPrebuildMetrics = `-- name: GetPrebuildMetrics :many +SELECT + t.name as template_name, + tvp.name as preset_name, + COUNT(*) as created_count, + COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, + COUNT(*) FILTER ( + WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. + ) as claimed_count +FROM workspaces w +INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id +INNER JOIN templates t ON t.id = w.template_id +INNER JOIN template_version_presets tvp ON tvp.id = wpb.template_version_preset_id +INNER JOIN provisioner_jobs pj ON pj.id = wpb.job_id +WHERE wpb.build_number = 1 +GROUP BY t.name, tvp.name +ORDER BY t.name, tvp.name +` + +type GetPrebuildMetricsRow struct { + TemplateName string `db:"template_name" json:"template_name"` + PresetName string `db:"preset_name" json:"preset_name"` + CreatedCount int64 `db:"created_count" json:"created_count"` + FailedCount int64 `db:"failed_count" json:"failed_count"` + ClaimedCount int64 `db:"claimed_count" json:"claimed_count"` +} + +func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) { + rows, err := q.db.QueryContext(ctx, getPrebuildMetrics) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetPrebuildMetricsRow + for rows.Next() { + var i GetPrebuildMetricsRow + if err := rows.Scan( + &i.TemplateName, + &i.PresetName, + &i.CreatedCount, + &i.FailedCount, + &i.ClaimedCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getPrebuildsInProgress = `-- name: GetPrebuildsInProgress :many +SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count +FROM workspace_latest_build wlb + INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id +WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) +GROUP BY t.id, wpb.template_version_id, wpb.transition +` + +type GetPrebuildsInProgressRow struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + Count int32 `db:"count" json:"count"` +} + +func (q *sqlQuerier) GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuildsInProgressRow, error) { + rows, err := q.db.QueryContext(ctx, getPrebuildsInProgress) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetPrebuildsInProgressRow + for rows.Next() { + var i GetPrebuildsInProgressRow + if err := rows.Scan( + &i.TemplateID, + &i.TemplateVersionID, + &i.Transition, + &i.Count, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getPresetsBackoff = `-- name: GetPresetsBackoff :many +WITH filtered_builds AS ( + -- Only select builds which are for prebuild creations + SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances + FROM template_version_presets tvp + JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id + JOIN provisioner_jobs pj ON wlb.job_id = pj.id + JOIN template_versions tv ON wlb.template_version_id = tv.id + JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id + WHERE wlb.transition = 'start'::workspace_transition), + latest_builds AS ( + -- Select only the latest build per template_version AND preset + SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, + ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + FROM filtered_builds fb), + failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= $1::timestamptz + GROUP BY preset_id) +SELECT lb.template_version_id, + lb.preset_id, + MAX(lb.job_status)::provisioner_job_status AS latest_build_status, + MAX(COALESCE(fc.num_failed, 0))::int AS num_failed, + MAX(lb.created_at)::timestamptz AS last_build_at +FROM latest_builds lb + LEFT JOIN failed_count fc ON fc.preset_id = lb.preset_id +WHERE lb.rn <= lb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff + AND lb.job_status = 'failed'::provisioner_job_status +GROUP BY lb.template_version_id, lb.preset_id, lb.job_status +` + +type GetPresetsBackoffRow struct { + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + LatestBuildStatus ProvisionerJobStatus `db:"latest_build_status" json:"latest_build_status"` + NumFailed int32 `db:"num_failed" json:"num_failed"` + LastBuildAt time.Time `db:"last_build_at" json:"last_build_at"` +} + +func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) { + rows, err := q.db.QueryContext(ctx, getPresetsBackoff, lookback) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetPresetsBackoffRow + for rows.Next() { + var i GetPresetsBackoffRow + if err := rows.Scan( + &i.TemplateVersionID, + &i.PresetID, + &i.LatestBuildStatus, + &i.NumFailed, + &i.LastBuildAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getRunningPrebuilds = `-- name: GetRunningPrebuilds :many +SELECT p.id AS workspace_id, + p.name AS workspace_name, + p.template_id, + b.template_version_id, + tvp_curr.id AS current_preset_id, + CASE + WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE + ELSE FALSE END AS ready, + p.created_at +FROM workspace_prebuilds p + INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + LEFT JOIN template_version_presets tvp_curr + ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. +WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status = 'succeeded'::provisioner_job_status) +` + +type GetRunningPrebuildsRow struct { + WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` + WorkspaceName string `db:"workspace_name" json:"workspace_name"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` + Ready bool `db:"ready" json:"ready"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (q *sqlQuerier) GetRunningPrebuilds(ctx context.Context) ([]GetRunningPrebuildsRow, error) { + rows, err := q.db.QueryContext(ctx, getRunningPrebuilds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetRunningPrebuildsRow + for rows.Next() { + var i GetRunningPrebuildsRow + if err := rows.Scan( + &i.WorkspaceID, + &i.WorkspaceName, + &i.TemplateID, + &i.TemplateVersionID, + &i.CurrentPresetID, + &i.Ready, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getTemplatePresetsWithPrebuilds = `-- name: GetTemplatePresetsWithPrebuilds :many +SELECT t.id AS template_id, + t.name AS template_name, + tv.id AS template_version_id, + tv.name AS template_version_name, + tv.id = t.active_version_id AS using_active_version, + tvpp.preset_id, + tvp.name, + tvpp.desired_instances AS desired_instances, + t.deleted, + t.deprecated != '' AS deprecated +FROM templates t + INNER JOIN template_versions tv ON tv.template_id = t.id + INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id +WHERE (t.id = $1::uuid OR $1 IS NULL) +` + +type GetTemplatePresetsWithPrebuildsRow struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateName string `db:"template_name" json:"template_name"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + TemplateVersionName string `db:"template_version_name" json:"template_version_name"` + UsingActiveVersion bool `db:"using_active_version" json:"using_active_version"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + Name string `db:"name" json:"name"` + DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` + Deleted bool `db:"deleted" json:"deleted"` + Deprecated bool `db:"deprecated" json:"deprecated"` +} + +func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) { + rows, err := q.db.QueryContext(ctx, getTemplatePresetsWithPrebuilds, templateID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetTemplatePresetsWithPrebuildsRow + for rows.Next() { + var i GetTemplatePresetsWithPrebuildsRow + if err := rows.Scan( + &i.TemplateID, + &i.TemplateName, + &i.TemplateVersionID, + &i.TemplateVersionName, + &i.UsingActiveVersion, + &i.PresetID, + &i.Name, + &i.DesiredInstances, + &i.Deleted, + &i.Deprecated, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertPresetPrebuild = `-- name: InsertPresetPrebuild :one +INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances, invalidate_after_secs) +VALUES ($1::uuid, $2::uuid, $3::int, $4::int) +RETURNING id, preset_id, desired_instances, invalidate_after_secs +` + +type InsertPresetPrebuildParams struct { + ID uuid.UUID `db:"id" json:"id"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` + InvalidateAfterSecs int32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` +} + +func (q *sqlQuerier) InsertPresetPrebuild(ctx context.Context, arg InsertPresetPrebuildParams) (TemplateVersionPresetPrebuild, error) { + row := q.db.QueryRowContext(ctx, insertPresetPrebuild, + arg.ID, + arg.PresetID, + arg.DesiredInstances, + arg.InvalidateAfterSecs, + ) + var i TemplateVersionPresetPrebuild + err := row.Scan( + &i.ID, + &i.PresetID, + &i.DesiredInstances, + &i.InvalidateAfterSecs, + ) + return i, err +} + const getPresetByWorkspaceBuildID = `-- name: GetPresetByWorkspaceBuildID :one SELECT template_version_presets.id, template_version_presets.template_version_id, template_version_presets.name, template_version_presets.created_at diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql new file mode 100644 index 0000000000000..f5d1d1a674050 --- /dev/null +++ b/coderd/database/queries/prebuilds.sql @@ -0,0 +1,120 @@ +-- name: GetTemplatePresetsWithPrebuilds :many +SELECT t.id AS template_id, + t.name AS template_name, + tv.id AS template_version_id, + tv.name AS template_version_name, + tv.id = t.active_version_id AS using_active_version, + tvpp.preset_id, + tvp.name, + tvpp.desired_instances AS desired_instances, + t.deleted, + t.deprecated != '' AS deprecated +FROM templates t + INNER JOIN template_versions tv ON tv.template_id = t.id + INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id +WHERE (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); + +-- name: GetRunningPrebuilds :many +SELECT p.id AS workspace_id, + p.name AS workspace_name, + p.template_id, + b.template_version_id, + tvp_curr.id AS current_preset_id, + CASE + WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE + ELSE FALSE END AS ready, + p.created_at +FROM workspace_prebuilds p + INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + LEFT JOIN template_version_presets tvp_curr + ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. +WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status = 'succeeded'::provisioner_job_status); + +-- name: GetPrebuildsInProgress :many +SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count +FROM workspace_latest_build wlb + INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id +WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) +GROUP BY t.id, wpb.template_version_id, wpb.transition; + +-- name: GetPresetsBackoff :many +WITH filtered_builds AS ( + -- Only select builds which are for prebuild creations + SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances + FROM template_version_presets tvp + JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id + JOIN provisioner_jobs pj ON wlb.job_id = pj.id + JOIN template_versions tv ON wlb.template_version_id = tv.id + JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id + WHERE wlb.transition = 'start'::workspace_transition), + latest_builds AS ( + -- Select only the latest build per template_version AND preset + SELECT fb.*, + ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + FROM filtered_builds fb), + failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= @lookback::timestamptz + GROUP BY preset_id) +SELECT lb.template_version_id, + lb.preset_id, + MAX(lb.job_status)::provisioner_job_status AS latest_build_status, + MAX(COALESCE(fc.num_failed, 0))::int AS num_failed, + MAX(lb.created_at)::timestamptz AS last_build_at +FROM latest_builds lb + LEFT JOIN failed_count fc ON fc.preset_id = lb.preset_id +WHERE lb.rn <= lb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff + AND lb.job_status = 'failed'::provisioner_job_status +GROUP BY lb.template_version_id, lb.preset_id, lb.job_status; + +-- name: ClaimPrebuild :one +UPDATE workspaces w +SET owner_id = @new_user_id::uuid, + name = @new_name::text, + updated_at = NOW() +WHERE w.id IN (SELECT p.id + FROM workspace_prebuilds p + INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status IN ('succeeded'::provisioner_job_status)) + AND b.template_version_id = t.active_version_id + AND b.template_version_preset_id = @preset_id::uuid + AND p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state + ORDER BY random() + LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. +RETURNING w.id, w.name; + +-- name: InsertPresetPrebuild :one +INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances, invalidate_after_secs) +VALUES (@id::uuid, @preset_id::uuid, @desired_instances::int, @invalidate_after_secs::int) +RETURNING *; + +-- name: GetPrebuildMetrics :many +SELECT + t.name as template_name, + tvp.name as preset_name, + COUNT(*) as created_count, + COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, + COUNT(*) FILTER ( + WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. + ) as claimed_count +FROM workspaces w +INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id +INNER JOIN templates t ON t.id = w.template_id +INNER JOIN template_version_presets tvp ON tvp.id = wpb.template_version_preset_id +INNER JOIN provisioner_jobs pj ON pj.id = wpb.job_id +WHERE wpb.build_number = 1 +GROUP BY t.name, tvp.name +ORDER BY t.name, tvp.name; diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index b2c814241d55a..648508e957e47 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -59,6 +59,7 @@ const ( UniqueTemplateUsageStatsPkey UniqueConstraint = "template_usage_stats_pkey" // ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); UniqueTemplateVersionParametersTemplateVersionIDNameKey UniqueConstraint = "template_version_parameters_template_version_id_name_key" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionPresetParametersPkey UniqueConstraint = "template_version_preset_parameters_pkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id); + UniqueTemplateVersionPresetPrebuildsPkey UniqueConstraint = "template_version_preset_prebuilds_pkey" // ALTER TABLE ONLY template_version_preset_prebuilds ADD CONSTRAINT template_version_preset_prebuilds_pkey PRIMARY KEY (id); UniqueTemplateVersionPresetsPkey UniqueConstraint = "template_version_presets_pkey" // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id); UniqueTemplateVersionVariablesTemplateVersionIDNameKey UniqueConstraint = "template_version_variables_template_version_id_name_key" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionWorkspaceTagsTemplateVersionIDKeyKey UniqueConstraint = "template_version_workspace_tags_template_version_id_key_key" // ALTER TABLE ONLY template_version_workspace_tags ADD CONSTRAINT template_version_workspace_tags_template_version_id_key_key UNIQUE (template_version_id, key); @@ -97,6 +98,7 @@ const ( UniqueIndexCustomRolesNameLower UniqueConstraint = "idx_custom_roles_name_lower" // CREATE UNIQUE INDEX idx_custom_roles_name_lower ON custom_roles USING btree (lower(name)); UniqueIndexOrganizationNameLower UniqueConstraint = "idx_organization_name_lower" // CREATE UNIQUE INDEX idx_organization_name_lower ON organizations USING btree (lower(name)) WHERE (deleted = false); UniqueIndexProvisionerDaemonsOrgNameOwnerKey UniqueConstraint = "idx_provisioner_daemons_org_name_owner_key" // CREATE UNIQUE INDEX idx_provisioner_daemons_org_name_owner_key ON provisioner_daemons USING btree (organization_id, name, lower(COALESCE((tags ->> 'owner'::text), ''::text))); + UniqueIndexUniquePresetName UniqueConstraint = "idx_unique_preset_name" // CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets USING btree (name, template_version_id); UniqueIndexUsersEmail UniqueConstraint = "idx_users_email" // CREATE UNIQUE INDEX idx_users_email ON users USING btree (email) WHERE (deleted = false); UniqueIndexUsersUsername UniqueConstraint = "idx_users_username" // CREATE UNIQUE INDEX idx_users_username ON users USING btree (username) WHERE (deleted = false); UniqueNotificationMessagesDedupeHashIndex UniqueConstraint = "notification_messages_dedupe_hash_idx" // CREATE UNIQUE INDEX notification_messages_dedupe_hash_idx ON notification_messages USING btree (dedupe_hash); From 51773ecaa7a6ce81f3cee93d8bc6005b33e076a4 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 14 Mar 2025 16:34:05 +0000 Subject: [PATCH 11/72] Simplify workspace_latest_build view Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz.go | 2 +- coderd/database/dump.sql | 39 +++++----- .../migrations/000302_prebuilds.up.sql | 76 +++++++------------ 3 files changed, 47 insertions(+), 70 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 72f7b489401a7..0824ff621d692 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -362,7 +362,7 @@ var ( subjectPrebuildsOrchestrator = rbac.Subject{ FriendlyName: "Prebuilds Orchestrator", - ID: prebuilds.OwnerID.String(), + ID: prebuilds.SystemUserID.String(), Roles: rbac.Roles([]rbac.Role{ { Identifier: rbac.RoleIdentifier{Name: "prebuilds-orchestrator"}, diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index b5936f0c49168..f698a01330d45 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1900,28 +1900,23 @@ CREATE VIEW workspace_build_with_user AS COMMENT ON VIEW workspace_build_with_user IS 'Joins in the username + avatar url of the initiated by user.'; CREATE VIEW workspace_latest_build AS - SELECT wb.id, - wb.created_at, - wb.updated_at, - wb.workspace_id, - wb.template_version_id, - wb.build_number, - wb.transition, - wb.initiator_id, - wb.provisioner_state, - wb.job_id, - wb.deadline, - wb.reason, - wb.daily_cost, - wb.max_deadline, - wb.template_version_preset_id - FROM (( SELECT tv.template_id, - wbmax_1.workspace_id, - max(wbmax_1.build_number) AS max_build_number - FROM (workspace_builds wbmax_1 - JOIN template_versions tv ON ((tv.id = wbmax_1.template_version_id))) - GROUP BY tv.template_id, wbmax_1.workspace_id) wbmax - JOIN workspace_builds wb ON (((wb.workspace_id = wbmax.workspace_id) AND (wb.build_number = wbmax.max_build_number)))); + SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.id, + workspace_builds.created_at, + workspace_builds.updated_at, + workspace_builds.workspace_id, + workspace_builds.template_version_id, + workspace_builds.build_number, + workspace_builds.transition, + workspace_builds.initiator_id, + workspace_builds.provisioner_state, + workspace_builds.job_id, + workspace_builds.deadline, + workspace_builds.reason, + workspace_builds.daily_cost, + workspace_builds.max_deadline, + workspace_builds.template_version_preset_id + FROM workspace_builds + ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC; CREATE TABLE workspace_modules ( id uuid NOT NULL, diff --git a/coderd/database/migrations/000302_prebuilds.up.sql b/coderd/database/migrations/000302_prebuilds.up.sql index ed673b511efe1..18abef57a554a 100644 --- a/coderd/database/migrations/000302_prebuilds.up.sql +++ b/coderd/database/migrations/000302_prebuilds.up.sql @@ -1,56 +1,38 @@ CREATE VIEW workspace_latest_build AS -SELECT wb.* -FROM (SELECT tv.template_id, - wbmax.workspace_id, - MAX(wbmax.build_number) as max_build_number - FROM workspace_builds wbmax - JOIN template_versions tv ON (tv.id = wbmax.template_version_id) - GROUP BY tv.template_id, wbmax.workspace_id) wbmax - JOIN workspace_builds wb ON ( - wb.workspace_id = wbmax.workspace_id - AND wb.build_number = wbmax.max_build_number - ); +SELECT DISTINCT ON (workspace_id) * +FROM workspace_builds +ORDER BY workspace_id, build_number DESC; CREATE VIEW workspace_prebuilds AS WITH - -- All workspaces owned by the "prebuilds" user. - all_prebuilds AS (SELECT w.* - FROM workspaces w - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'), -- The system user responsible for prebuilds. - -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. - workspace_agents AS (SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at - FROM workspaces w - INNER JOIN workspace_latest_build wlb ON wlb.workspace_id = w.id - INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id - INNER JOIN workspace_agents wa ON wa.resource_id = wr.id - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. - GROUP BY w.id, wa.id), - -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the - -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, - -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. - -- - -- See https://github.com/coder/internal/issues/398 - current_presets AS (SELECT w.id AS prebuild_id, lps.template_version_preset_id - FROM workspaces w - INNER JOIN ( - -- The latest workspace build which had a preset explicitly selected - SELECT wb.* - FROM (SELECT tv.template_id, - wbmax.workspace_id, - MAX(wbmax.build_number) as max_build_number - FROM workspace_builds wbmax - JOIN template_versions tv ON (tv.id = wbmax.template_version_id) - WHERE wbmax.template_version_preset_id IS NOT NULL - GROUP BY tv.template_id, wbmax.workspace_id) wbmax - JOIN workspace_builds wb ON ( - wb.workspace_id = wbmax.workspace_id - AND wb.build_number = wbmax.max_build_number - )) lps ON lps.workspace_id = w.id - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. + -- All workspaces owned by the "prebuilds" user. + all_prebuilds AS (SELECT w.* + FROM workspaces w + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'), -- The system user responsible for prebuilds. + -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the + -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, + -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. + -- + -- See https://github.com/coder/internal/issues/398 + latest_prebuild_builds AS (SELECT * + FROM workspace_latest_build + WHERE template_version_preset_id IS NOT NULL), + -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. + workspace_agents AS (SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at + FROM workspaces w + INNER JOIN workspace_latest_build wlb ON wlb.workspace_id = w.id + INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id + INNER JOIN workspace_agents wa ON wa.resource_id = wr.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. + GROUP BY w.id, wa.id), + current_presets AS (SELECT w.id AS prebuild_id, lpb.template_version_preset_id + FROM workspaces w + INNER JOIN latest_prebuild_builds lpb ON lpb.workspace_id = w.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. SELECT p.*, a.agent_id, a.lifecycle_state, a.ready_at, cp.template_version_preset_id AS current_preset_id FROM all_prebuilds p - LEFT JOIN workspace_agents a ON a.workspace_id = p.id - INNER JOIN current_presets cp ON cp.prebuild_id = p.id; + LEFT JOIN workspace_agents a ON a.workspace_id = p.id + INNER JOIN current_presets cp ON cp.prebuild_id = p.id; CREATE VIEW workspace_prebuild_builds AS SELECT * From 23773c2d4cb88edc2bb3aa82db0abf5575b294ed Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Mon, 17 Mar 2025 13:02:16 +0000 Subject: [PATCH 12/72] Revert test change Signed-off-by: Danny Kopping --- coderd/database/dbauthz/dbauthz_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index b02c0cbae32c4..4693437ed17c7 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -979,8 +979,8 @@ func (s *MethodTestSuite) TestOrganization() { }) check.Args(database.OrganizationMembersParams{ - OrganizationID: o.ID, - UserID: u.ID, + OrganizationID: uuid.UUID{}, + UserID: uuid.UUID{}, }).Asserts( mem, policy.ActionRead, ) From bc3ff444a606f961df4c7083bf6ed4a7c8b91fd9 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Mon, 17 Mar 2025 13:11:58 +0000 Subject: [PATCH 13/72] make gen Signed-off-by: Danny Kopping --- coderd/database/dump.sql | 44 ++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index f698a01330d45..da723f4c0650c 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -2557,6 +2557,24 @@ CREATE OR REPLACE VIEW workspace_prebuilds AS w.next_start_at FROM workspaces w WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + ), latest_prebuild_builds AS ( + SELECT workspace_latest_build.id, + workspace_latest_build.created_at, + workspace_latest_build.updated_at, + workspace_latest_build.workspace_id, + workspace_latest_build.template_version_id, + workspace_latest_build.build_number, + workspace_latest_build.transition, + workspace_latest_build.initiator_id, + workspace_latest_build.provisioner_state, + workspace_latest_build.job_id, + workspace_latest_build.deadline, + workspace_latest_build.reason, + workspace_latest_build.daily_cost, + workspace_latest_build.max_deadline, + workspace_latest_build.template_version_preset_id + FROM workspace_latest_build + WHERE (workspace_latest_build.template_version_preset_id IS NOT NULL) ), workspace_agents AS ( SELECT w.id AS workspace_id, wa.id AS agent_id, @@ -2570,31 +2588,9 @@ CREATE OR REPLACE VIEW workspace_prebuilds AS GROUP BY w.id, wa.id ), current_presets AS ( SELECT w.id AS prebuild_id, - lps.template_version_preset_id + lpb.template_version_preset_id FROM (workspaces w - JOIN ( SELECT wb.id, - wb.created_at, - wb.updated_at, - wb.workspace_id, - wb.template_version_id, - wb.build_number, - wb.transition, - wb.initiator_id, - wb.provisioner_state, - wb.job_id, - wb.deadline, - wb.reason, - wb.daily_cost, - wb.max_deadline, - wb.template_version_preset_id - FROM (( SELECT tv.template_id, - wbmax_1.workspace_id, - max(wbmax_1.build_number) AS max_build_number - FROM (workspace_builds wbmax_1 - JOIN template_versions tv ON ((tv.id = wbmax_1.template_version_id))) - WHERE (wbmax_1.template_version_preset_id IS NOT NULL) - GROUP BY tv.template_id, wbmax_1.workspace_id) wbmax - JOIN workspace_builds wb ON (((wb.workspace_id = wbmax.workspace_id) AND (wb.build_number = wbmax.max_build_number))))) lps ON ((lps.workspace_id = w.id))) + JOIN latest_prebuild_builds lpb ON ((lpb.workspace_id = w.id))) WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) ) SELECT p.id, From baa30769439186c9fca8755f2770d4c815104a84 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 19 Mar 2025 16:43:15 -0400 Subject: [PATCH 14/72] refactor: add comments to SQL queries --- coderd/database/migrations/000302_prebuilds.up.sql | 3 +++ coderd/database/querier.go | 3 +++ coderd/database/queries.sql.go | 3 +++ coderd/database/queries/prebuilds.sql | 3 +++ 4 files changed, 12 insertions(+) diff --git a/coderd/database/migrations/000302_prebuilds.up.sql b/coderd/database/migrations/000302_prebuilds.up.sql index 18abef57a554a..b4e6cba939864 100644 --- a/coderd/database/migrations/000302_prebuilds.up.sql +++ b/coderd/database/migrations/000302_prebuilds.up.sql @@ -1,8 +1,11 @@ +-- workspace_latest_build contains latest build for every workspace CREATE VIEW workspace_latest_build AS SELECT DISTINCT ON (workspace_id) * FROM workspace_builds ORDER BY workspace_id, build_number DESC; +-- workspace_prebuilds contains all prebuilt workspaces with corresponding agent information +-- (including lifecycle_state which indicates is agent ready or not) and corresponding preset_id for prebuild CREATE VIEW workspace_prebuilds AS WITH -- All workspaces owned by the "prebuilds" user. diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 91a64f6f6f1ef..3616e36f65850 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -290,6 +290,9 @@ type sqlcQuerier interface { // created in the timeframe and return the aggregate usage counts of parameter // values. GetTemplateParameterInsights(ctx context.Context, arg GetTemplateParameterInsightsParams) ([]GetTemplateParameterInsightsRow, error) + // GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. + // It also returns the number of desired instances for each preset. + // If template_id is specified, only template versions associated with that template will be returned. GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) GetTemplateUsageStats(ctx context.Context, arg GetTemplateUsageStatsParams) ([]TemplateUsageStat, error) GetTemplateVersionByID(ctx context.Context, id uuid.UUID) (TemplateVersion, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 4cec4e1e93ad6..7ed0dfce68f71 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6053,6 +6053,9 @@ type GetTemplatePresetsWithPrebuildsRow struct { Deprecated bool `db:"deprecated" json:"deprecated"` } +// GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. +// It also returns the number of desired instances for each preset. +// If template_id is specified, only template versions associated with that template will be returned. func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) { rows, err := q.db.QueryContext(ctx, getTemplatePresetsWithPrebuilds, templateID) if err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index f5d1d1a674050..8e78b485ea578 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -1,4 +1,7 @@ -- name: GetTemplatePresetsWithPrebuilds :many +-- GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. +-- It also returns the number of desired instances for each preset. +-- If template_id is specified, only template versions associated with that template will be returned. SELECT t.id AS template_id, t.name AS template_name, tv.id AS template_version_id, From ed14fb3eef2fc5195d966de7c98d6913f89e625a Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 20 Mar 2025 16:04:26 -0400 Subject: [PATCH 15/72] test: added get-presets-backoff test --- coderd/database/querier_test.go | 463 ++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 7606e90b5107c..b93970ca09be2 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3474,6 +3474,469 @@ func TestOrganizationDeleteTrigger(t *testing.T) { }) } +func TestGetPresetsBackoff(t *testing.T) { + t.Parallel() + type extTmplVersion struct { + database.TemplateVersion + preset database.TemplateVersionPreset + } + + now := dbtime.Now() + orgID := uuid.New() + userID := uuid.New() + + createTemplate := func(db database.Store) database.Template { + // create template + tmpl := dbgen.Template(t, db, database.Template{ + OrganizationID: orgID, + CreatedBy: userID, + ActiveVersionID: uuid.New(), + }) + + return tmpl + } + type tmplVersionOpts struct { + DesiredInstances int + } + createTmplVersion := func(db database.Store, tmpl database.Template, versionId uuid.UUID, opts *tmplVersionOpts) extTmplVersion { + // Create template version with corresponding preset and preset prebuild + tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + ID: versionId, + TemplateID: uuid.NullUUID{ + UUID: tmpl.ID, + Valid: true, + }, + OrganizationID: tmpl.OrganizationID, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: tmpl.CreatedBy, + }) + preset := dbgen.Preset(t, db, database.InsertPresetParams{ + TemplateVersionID: tmplVersion.ID, + Name: "preset", + }) + desiredInstances := 1 + if opts != nil { + desiredInstances = opts.DesiredInstances + } + dbgen.PresetPrebuild(t, db, database.InsertPresetPrebuildParams{ + PresetID: preset.ID, + DesiredInstances: int32(desiredInstances), + }) + + return extTmplVersion{ + TemplateVersion: tmplVersion, + preset: preset, + } + } + type workspaceBuildOpts struct { + successfulJob bool + createdAt time.Time + } + createWorkspaceBuild := func( + db database.Store, + tmpl database.Template, + extTmplVersion extTmplVersion, + opts *workspaceBuildOpts, + ) { + // Create job with corresponding resource and agent + jobError := sql.NullString{String: "failed", Valid: true} + if opts != nil && opts.successfulJob { + jobError = sql.NullString{} + } + job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + Type: database.ProvisionerJobTypeWorkspaceBuild, + OrganizationID: orgID, + + CreatedAt: now.Add(-1 * time.Minute), + Error: jobError, + }) + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ + JobID: job.ID, + }) + dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ + ResourceID: resource.ID, + }) + + // Create corresponding workspace and workspace build + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: tmpl.CreatedBy, + OrganizationID: tmpl.OrganizationID, + TemplateID: tmpl.ID, + }) + createdAt := now + if opts != nil { + createdAt = opts.createdAt + } + dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ + CreatedAt: createdAt, + WorkspaceID: workspace.ID, + TemplateVersionID: extTmplVersion.ID, + BuildNumber: 1, + Transition: database.WorkspaceTransitionStart, + InitiatorID: tmpl.CreatedBy, + JobID: job.ID, + TemplateVersionPresetID: uuid.NullUUID{ + UUID: extTmplVersion.preset.ID, + Valid: true, + }, + }) + } + findBackoffByTmplVersionId := func(backoffs []database.GetPresetsBackoffRow, tmplVersionID uuid.UUID) *database.GetPresetsBackoffRow { + for _, backoff := range backoffs { + if backoff.TemplateVersionID == tmplVersionID { + return &backoff + } + } + + return nil + } + + t.Run("Single Workspace Build", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl := createTemplate(db) + tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl, tmplV1, nil) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmplV1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(1), backoff.NumFailed) + }) + + t.Run("Multiple Workspace Builds", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl := createTemplate(db) + tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl, tmplV1, nil) + createWorkspaceBuild(db, tmpl, tmplV1, nil) + createWorkspaceBuild(db, tmpl, tmplV1, nil) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmplV1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(3), backoff.NumFailed) + }) + + t.Run("Ignore Inactive Version", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl := createTemplate(db) + tmplV1 := createTmplVersion(db, tmpl, uuid.New(), nil) + createWorkspaceBuild(db, tmpl, tmplV1, nil) + + // Active Version + tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl, tmplV2, nil) + createWorkspaceBuild(db, tmpl, tmplV2, nil) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmplV2.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(2), backoff.NumFailed) + }) + + t.Run("Multiple Templates", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) + + tmpl2 := createTemplate(db) + tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 2) + { + backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) + require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(1), backoff.NumFailed) + } + { + backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) + require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(1), backoff.NumFailed) + } + }) + + t.Run("Multiple Templates, Versions and Workspace Builds", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) + + tmpl2 := createTemplate(db) + tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) + createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) + + tmpl3 := createTemplate(db) + tmpl3V1 := createTmplVersion(db, tmpl3, uuid.New(), nil) + createWorkspaceBuild(db, tmpl3, tmpl3V1, nil) + + tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, nil) + createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) + createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) + createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 3) + { + backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) + require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(1), backoff.NumFailed) + } + { + backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) + require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(2), backoff.NumFailed) + } + { + backoff := findBackoffByTmplVersionId(backoffs, tmpl3.ActiveVersionID) + require.Equal(t, backoff.TemplateVersionID, tmpl3.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl3V2.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(3), backoff.NumFailed) + } + }) + + t.Run("No Workspace Builds", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + _ = tmpl1V1 + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + require.Nil(t, backoffs) + }) + + t.Run("No Failed Workspace Builds", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + successfulJobOpts := workspaceBuildOpts{ + successfulJob: true, + } + createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + require.Nil(t, backoffs) + }) + + t.Run("Last job is successful - no backoff", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 1, + }) + failedJobOpts := workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-2 * time.Minute), + } + successfulJobOpts := workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-1 * time.Minute), + } + createWorkspaceBuild(db, tmpl1, tmpl1V1, &failedJobOpts) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + require.Nil(t, backoffs) + }) + + t.Run("Last 3 jobs are successful - no backoff", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 3, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-4 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-3 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-2 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-1 * time.Minute), + }) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + require.Nil(t, backoffs) + }) + + t.Run("1 job failed out of 3 - backoff", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 3, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-3 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-2 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-1 * time.Minute), + }) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + { + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(1), backoff.NumFailed) + } + }) +} + func requireUsersMatch(t testing.TB, expected []database.User, found []database.GetUsersRow, msg string) { t.Helper() require.ElementsMatch(t, expected, database.ConvertUserRows(found), msg) From 3cc74fb42db5c2799232b692f6246af0bc9109e4 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 20 Mar 2025 16:27:02 -0400 Subject: [PATCH 16/72] refactor: add comment to SQL query --- coderd/database/querier.go | 5 +++++ coderd/database/queries.sql.go | 5 +++++ coderd/database/queries/prebuilds.sql | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 3616e36f65850..5672ecca5cec2 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -225,6 +225,11 @@ type sqlcQuerier interface { GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuildsInProgressRow, error) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) + // GetPresetsBackoff groups workspace builds by template version ID. + // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. + // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. + // Query returns a list of template version IDs for which we should backoff. + // Only template versions with configured presets are considered. GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPreset, error) GetPreviousTemplateVersion(ctx context.Context, arg GetPreviousTemplateVersionParams) (TemplateVersion, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 7ed0dfce68f71..c4e9f57f911d8 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5932,6 +5932,11 @@ type GetPresetsBackoffRow struct { LastBuildAt time.Time `db:"last_build_at" json:"last_build_at"` } +// GetPresetsBackoff groups workspace builds by template version ID. +// For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. +// If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. +// Query returns a list of template version IDs for which we should backoff. +// Only template versions with configured presets are considered. func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) { rows, err := q.db.QueryContext(ctx, getPresetsBackoff, lookback) if err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 8e78b485ea578..5b146b879ed79 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -47,6 +47,11 @@ WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisione GROUP BY t.id, wpb.template_version_id, wpb.transition; -- name: GetPresetsBackoff :many +-- GetPresetsBackoff groups workspace builds by template version ID. +-- For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. +-- If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. +-- Query returns a list of template version IDs for which we should backoff. +-- Only template versions with configured presets are considered. WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances From fc3215477f9bdc8360488958ed762a43bf1357e9 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 08:50:01 -0400 Subject: [PATCH 17/72] refactor: add comments + improve tests --- coderd/database/querier.go | 12 ++++++- coderd/database/querier_test.go | 51 +++++++++++++++++++++++++++ coderd/database/queries.sql.go | 36 ++++++++++++------- coderd/database/queries/prebuilds.sql | 36 ++++++++++++------- 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 5672ecca5cec2..40477ffb8a2ff 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -229,7 +229,17 @@ type sqlcQuerier interface { // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. - // Only template versions with configured presets are considered. + // Only active template versions with configured presets are considered. + // + // NOTE: + // We back off on the template version ID if at least one of the N latest workspace builds has failed. + // However, we also return the number of failed workspace builds that occurred during the lookback period. + // + // In other words: + // - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). + // - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. + // + // The number of failed builds is used downstream to determine the backoff duration. GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPreset, error) GetPreviousTemplateVersion(ctx context.Context, arg GetPreviousTemplateVersionParams) (TemplateVersion, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index b93970ca09be2..28b21a58302f8 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3935,6 +3935,57 @@ func TestGetPresetsBackoff(t *testing.T) { require.Equal(t, int32(1), backoff.NumFailed) } }) + + t.Run("3 job failed out of 5 - backoff", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + lookbackPeriod := time.Hour + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 3, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-2 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: true, + createdAt: now.Add(-1 * time.Minute), + }) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + { + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(2), backoff.NumFailed) + } + }) } func requireUsersMatch(t testing.TB, expected []database.User, found []database.GetUsersRow, msg string) { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index c4e9f57f911d8..9da07de29a44a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5900,8 +5900,8 @@ WITH filtered_builds AS ( JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id WHERE wlb.transition = 'start'::workspace_transition), - latest_builds AS ( - -- Select only the latest build per template_version AND preset + time_sorted_builds AS ( + -- Group builds by template version, then sort each group by created_at. SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb), @@ -5912,16 +5912,16 @@ WITH filtered_builds AS ( WHERE job_status = 'failed'::provisioner_job_status AND created_at >= $1::timestamptz GROUP BY preset_id) -SELECT lb.template_version_id, - lb.preset_id, - MAX(lb.job_status)::provisioner_job_status AS latest_build_status, - MAX(COALESCE(fc.num_failed, 0))::int AS num_failed, - MAX(lb.created_at)::timestamptz AS last_build_at -FROM latest_builds lb - LEFT JOIN failed_count fc ON fc.preset_id = lb.preset_id -WHERE lb.rn <= lb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff - AND lb.job_status = 'failed'::provisioner_job_status -GROUP BY lb.template_version_id, lb.preset_id, lb.job_status +SELECT tsb.template_version_id, + tsb.preset_id, + tsb.job_status::provisioner_job_status AS latest_build_status, + COALESCE(fc.num_failed, 0)::int AS num_failed, + tsb.created_at::timestamptz AS last_build_at +FROM time_sorted_builds tsb + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id +WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff + AND tsb.job_status = 'failed'::provisioner_job_status +GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, tsb.created_at, fc.num_failed ` type GetPresetsBackoffRow struct { @@ -5936,7 +5936,17 @@ type GetPresetsBackoffRow struct { // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. -// Only template versions with configured presets are considered. +// Only active template versions with configured presets are considered. +// +// NOTE: +// We back off on the template version ID if at least one of the N latest workspace builds has failed. +// However, we also return the number of failed workspace builds that occurred during the lookback period. +// +// In other words: +// - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +// - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. +// +// The number of failed builds is used downstream to determine the backoff duration. func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) { rows, err := q.db.QueryContext(ctx, getPresetsBackoff, lookback) if err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 5b146b879ed79..365dbb5f29f51 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -51,7 +51,17 @@ GROUP BY t.id, wpb.template_version_id, wpb.transition; -- For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. -- If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. -- Query returns a list of template version IDs for which we should backoff. --- Only template versions with configured presets are considered. +-- Only active template versions with configured presets are considered. +-- +-- NOTE: +-- We back off on the template version ID if at least one of the N latest workspace builds has failed. +-- However, we also return the number of failed workspace builds that occurred during the lookback period. +-- +-- In other words: +-- - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +-- - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. +-- +-- The number of failed builds is used downstream to determine the backoff duration. WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances @@ -62,8 +72,8 @@ WITH filtered_builds AS ( JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id WHERE wlb.transition = 'start'::workspace_transition), - latest_builds AS ( - -- Select only the latest build per template_version AND preset + time_sorted_builds AS ( + -- Group builds by template version, then sort each group by created_at. SELECT fb.*, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb), @@ -74,16 +84,16 @@ WITH filtered_builds AS ( WHERE job_status = 'failed'::provisioner_job_status AND created_at >= @lookback::timestamptz GROUP BY preset_id) -SELECT lb.template_version_id, - lb.preset_id, - MAX(lb.job_status)::provisioner_job_status AS latest_build_status, - MAX(COALESCE(fc.num_failed, 0))::int AS num_failed, - MAX(lb.created_at)::timestamptz AS last_build_at -FROM latest_builds lb - LEFT JOIN failed_count fc ON fc.preset_id = lb.preset_id -WHERE lb.rn <= lb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff - AND lb.job_status = 'failed'::provisioner_job_status -GROUP BY lb.template_version_id, lb.preset_id, lb.job_status; +SELECT tsb.template_version_id, + tsb.preset_id, + tsb.job_status::provisioner_job_status AS latest_build_status, + COALESCE(fc.num_failed, 0)::int AS num_failed, + tsb.created_at::timestamptz AS last_build_at +FROM time_sorted_builds tsb + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id +WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff + AND tsb.job_status = 'failed'::provisioner_job_status +GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, tsb.created_at, fc.num_failed; -- name: ClaimPrebuild :one UPDATE workspaces w From d7b4ec41dc71535d9c082efe27b21001587e7541 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 09:52:14 -0400 Subject: [PATCH 18/72] fix: bug in SQL --- coderd/database/querier_test.go | 53 +++++++++++++++++++++++++++ coderd/database/queries.sql.go | 6 +-- coderd/database/queries/prebuilds.sql | 4 +- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 28b21a58302f8..1db4e39213ff9 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3986,6 +3986,59 @@ func TestGetPresetsBackoff(t *testing.T) { require.Equal(t, int32(2), backoff.NumFailed) } }) + + t.Run("check LastBuildAt timestamp", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + lookbackPeriod := time.Hour + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 5, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-4 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-0 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-3 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-1 * time.Minute), + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-2 * time.Minute), + }) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) + require.NoError(t, err) + + require.Len(t, backoffs, 1) + { + backoff := backoffs[0] + require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) + require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) + require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) + require.Equal(t, int32(5), backoff.NumFailed) + // make sure LastBuildAt is equal to latest failed build timestamp + require.Equal(t, 0, now.Compare(backoff.LastBuildAt.(time.Time))) + } + }) } func requireUsersMatch(t testing.TB, expected []database.User, found []database.GetUsersRow, msg string) { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 9da07de29a44a..62540ce5465ba 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5916,12 +5916,12 @@ SELECT tsb.template_version_id, tsb.preset_id, tsb.job_status::provisioner_job_status AS latest_build_status, COALESCE(fc.num_failed, 0)::int AS num_failed, - tsb.created_at::timestamptz AS last_build_at + MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, tsb.created_at, fc.num_failed +GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, fc.num_failed ` type GetPresetsBackoffRow struct { @@ -5929,7 +5929,7 @@ type GetPresetsBackoffRow struct { PresetID uuid.UUID `db:"preset_id" json:"preset_id"` LatestBuildStatus ProvisionerJobStatus `db:"latest_build_status" json:"latest_build_status"` NumFailed int32 `db:"num_failed" json:"num_failed"` - LastBuildAt time.Time `db:"last_build_at" json:"last_build_at"` + LastBuildAt interface{} `db:"last_build_at" json:"last_build_at"` } // GetPresetsBackoff groups workspace builds by template version ID. diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 365dbb5f29f51..3a1c0a5df4493 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -88,12 +88,12 @@ SELECT tsb.template_version_id, tsb.preset_id, tsb.job_status::provisioner_job_status AS latest_build_status, COALESCE(fc.num_failed, 0)::int AS num_failed, - tsb.created_at::timestamptz AS last_build_at + MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, tsb.created_at, fc.num_failed; +GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, fc.num_failed; -- name: ClaimPrebuild :one UPDATE workspaces w From e8b53f757f7a799ddd4701fedfd96f8207b9be71 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 10:04:58 -0400 Subject: [PATCH 19/72] test: minor changes to the test --- coderd/database/querier_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 1db4e39213ff9..5220fe15522fc 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -4002,7 +4002,11 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(db) tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 5, + DesiredInstances: 6, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ successfulJob: false, From 9df655403f70d984f37238344490973ac164ef01 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 10:52:12 -0400 Subject: [PATCH 20/72] refactor: remove job_status from SQL query --- coderd/database/querier_test.go | 11 ----------- coderd/database/queries.sql.go | 17 +++++++---------- coderd/database/queries/prebuilds.sql | 7 +++---- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 5220fe15522fc..73ae0d124988d 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3615,7 +3615,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) require.Equal(t, backoff.PresetID, tmplV1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(1), backoff.NumFailed) }) @@ -3644,7 +3643,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) require.Equal(t, backoff.PresetID, tmplV1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(3), backoff.NumFailed) }) @@ -3676,7 +3674,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl.ActiveVersionID) require.Equal(t, backoff.PresetID, tmplV2.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(2), backoff.NumFailed) }) @@ -3708,14 +3705,12 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(1), backoff.NumFailed) } { backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(1), backoff.NumFailed) } }) @@ -3758,21 +3753,18 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(1), backoff.NumFailed) } { backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(2), backoff.NumFailed) } { backoff := findBackoffByTmplVersionId(backoffs, tmpl3.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl3.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl3V2.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(3), backoff.NumFailed) } }) @@ -3931,7 +3923,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(1), backoff.NumFailed) } }) @@ -3982,7 +3973,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(2), backoff.NumFailed) } }) @@ -4037,7 +4027,6 @@ func TestGetPresetsBackoff(t *testing.T) { backoff := backoffs[0] require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) - require.Equal(t, database.ProvisionerJobStatusFailed, backoff.LatestBuildStatus) require.Equal(t, int32(5), backoff.NumFailed) // make sure LastBuildAt is equal to latest failed build timestamp require.Equal(t, 0, now.Compare(backoff.LastBuildAt.(time.Time))) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 62540ce5465ba..b7f99c7d63679 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5914,22 +5914,20 @@ WITH filtered_builds AS ( GROUP BY preset_id) SELECT tsb.template_version_id, tsb.preset_id, - tsb.job_status::provisioner_job_status AS latest_build_status, - COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at::timestamptz) AS last_build_at + COALESCE(fc.num_failed, 0)::int AS num_failed, + MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, fc.num_failed +GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed ` type GetPresetsBackoffRow struct { - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - PresetID uuid.UUID `db:"preset_id" json:"preset_id"` - LatestBuildStatus ProvisionerJobStatus `db:"latest_build_status" json:"latest_build_status"` - NumFailed int32 `db:"num_failed" json:"num_failed"` - LastBuildAt interface{} `db:"last_build_at" json:"last_build_at"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + NumFailed int32 `db:"num_failed" json:"num_failed"` + LastBuildAt interface{} `db:"last_build_at" json:"last_build_at"` } // GetPresetsBackoff groups workspace builds by template version ID. @@ -5959,7 +5957,6 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) if err := rows.Scan( &i.TemplateVersionID, &i.PresetID, - &i.LatestBuildStatus, &i.NumFailed, &i.LastBuildAt, ); err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 3a1c0a5df4493..edb9c51dc5fd6 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -86,14 +86,13 @@ WITH filtered_builds AS ( GROUP BY preset_id) SELECT tsb.template_version_id, tsb.preset_id, - tsb.job_status::provisioner_job_status AS latest_build_status, - COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at::timestamptz) AS last_build_at + COALESCE(fc.num_failed, 0)::int AS num_failed, + MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, tsb.job_status, fc.num_failed; +GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; -- name: ClaimPrebuild :one UPDATE workspaces w From ccc309e8c341aef9c8e3483a4b9d8fd693f94364 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 15:00:43 -0400 Subject: [PATCH 21/72] refactor: embed preset_prebuilds table into presets table --- coderd/database/dbauthz/dbauthz.go | 7 - coderd/database/dbgen/dbgen.go | 19 +-- coderd/database/dbmem/dbmem.go | 9 -- coderd/database/dbmetrics/querymetrics.go | 7 - coderd/database/dbmock/dbmock.go | 15 -- coderd/database/dump.sql | 17 +-- coderd/database/foreign_key_constraint.go | 1 - .../migrations/000303_preset_prebuilds.up.sql | 13 +- coderd/database/models.go | 17 +-- coderd/database/querier.go | 1 - coderd/database/querier_test.go | 14 +- coderd/database/queries.sql.go | 137 +++++++++--------- coderd/database/queries/prebuilds.sql | 46 +++--- coderd/database/queries/presets.sql | 18 ++- coderd/database/unique_constraint.go | 1 - 15 files changed, 127 insertions(+), 195 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 0824ff621d692..4f49661bca5cf 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -3310,13 +3310,6 @@ func (q *querier) InsertPresetParameters(ctx context.Context, arg database.Inser return q.db.InsertPresetParameters(ctx, arg) } -func (q *querier) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { - if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { - return database.TemplateVersionPresetPrebuild{}, err - } - return q.db.InsertPresetPrebuild(ctx, arg) -} - // TODO: We need to create a ProvisionerJob resource type func (q *querier) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { // if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index c7df7fe3171ee..d5bcc69e2ae09 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -1160,9 +1160,11 @@ func TelemetryItem(t testing.TB, db database.Store, seed database.TelemetryItem) func Preset(t testing.TB, db database.Store, seed database.InsertPresetParams) database.TemplateVersionPreset { preset, err := db.InsertPreset(genCtx, database.InsertPresetParams{ - TemplateVersionID: takeFirst(seed.TemplateVersionID, uuid.New()), - Name: takeFirst(seed.Name, testutil.GetRandomName(t)), - CreatedAt: takeFirst(seed.CreatedAt, dbtime.Now()), + TemplateVersionID: takeFirst(seed.TemplateVersionID, uuid.New()), + Name: takeFirst(seed.Name, testutil.GetRandomName(t)), + CreatedAt: takeFirst(seed.CreatedAt, dbtime.Now()), + DesiredInstances: seed.DesiredInstances, + InvalidateAfterSecs: seed.InvalidateAfterSecs, }) require.NoError(t, err, "insert preset") return preset @@ -1179,17 +1181,6 @@ func PresetParameter(t testing.TB, db database.Store, seed database.InsertPreset return parameters } -func PresetPrebuild(t testing.TB, db database.Store, seed database.InsertPresetPrebuildParams) database.TemplateVersionPresetPrebuild { - prebuild, err := db.InsertPresetPrebuild(genCtx, database.InsertPresetPrebuildParams{ - ID: takeFirst(seed.ID, uuid.New()), - PresetID: takeFirst(seed.PresetID, uuid.New()), - DesiredInstances: takeFirst(seed.DesiredInstances, 1), - InvalidateAfterSecs: 0, - }) - require.NoError(t, err, "insert preset prebuild") - return prebuild -} - func provisionerJobTiming(t testing.TB, db database.Store, seed database.ProvisionerJobTiming) database.ProvisionerJobTiming { timing, err := db.InsertProvisionerJobTimings(genCtx, database.InsertProvisionerJobTimingsParams{ JobID: takeFirst(seed.JobID, uuid.New()), diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 5b05951fc264a..e6e1edab6226a 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -8566,15 +8566,6 @@ func (q *FakeQuerier) InsertPresetParameters(_ context.Context, arg database.Ins return presetParameters, nil } -func (*FakeQuerier) InsertPresetPrebuild(_ context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { - err := validateDatabaseType(arg) - if err != nil { - return database.TemplateVersionPresetPrebuild{}, err - } - - return database.TemplateVersionPresetPrebuild{}, ErrUnimplemented -} - func (q *FakeQuerier) InsertProvisionerJob(_ context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { if err := validateDatabaseType(arg); err != nil { return database.ProvisionerJob{}, err diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index bd3480a2ff31c..c0b083ff5bad0 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -2055,13 +2055,6 @@ func (m queryMetricsStore) InsertPresetParameters(ctx context.Context, arg datab return r0, r1 } -func (m queryMetricsStore) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { - start := time.Now() - r0, r1 := m.s.InsertPresetPrebuild(ctx, arg) - m.queryLatencies.WithLabelValues("InsertPresetPrebuild").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m queryMetricsStore) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { start := time.Now() job, err := m.s.InsertProvisionerJob(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 803be16000abf..9f7b8ae7fe4c9 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -4337,21 +4337,6 @@ func (mr *MockStoreMockRecorder) InsertPresetParameters(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertPresetParameters", reflect.TypeOf((*MockStore)(nil).InsertPresetParameters), ctx, arg) } -// InsertPresetPrebuild mocks base method. -func (m *MockStore) InsertPresetPrebuild(ctx context.Context, arg database.InsertPresetPrebuildParams) (database.TemplateVersionPresetPrebuild, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InsertPresetPrebuild", ctx, arg) - ret0, _ := ret[0].(database.TemplateVersionPresetPrebuild) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// InsertPresetPrebuild indicates an expected call of InsertPresetPrebuild. -func (mr *MockStoreMockRecorder) InsertPresetPrebuild(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertPresetPrebuild", reflect.TypeOf((*MockStore)(nil).InsertPresetPrebuild), ctx, arg) -} - // InsertProvisionerJob mocks base method. func (m *MockStore) InsertProvisionerJob(ctx context.Context, arg database.InsertProvisionerJobParams) (database.ProvisionerJob, error) { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index da723f4c0650c..fa011eaf68181 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1382,18 +1382,13 @@ CREATE TABLE template_version_preset_parameters ( value text NOT NULL ); -CREATE TABLE template_version_preset_prebuilds ( - id uuid NOT NULL, - preset_id uuid NOT NULL, - desired_instances integer NOT NULL, - invalidate_after_secs integer DEFAULT 0 -); - CREATE TABLE template_version_presets ( id uuid DEFAULT gen_random_uuid() NOT NULL, template_version_id uuid NOT NULL, name text NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL + created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + desired_instances integer, + invalidate_after_secs integer DEFAULT 0 ); CREATE TABLE template_version_variables ( @@ -2266,9 +2261,6 @@ ALTER TABLE ONLY template_version_parameters ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id); -ALTER TABLE ONLY template_version_preset_prebuilds - ADD CONSTRAINT template_version_preset_prebuilds_pkey PRIMARY KEY (id); - ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id); @@ -2768,9 +2760,6 @@ ALTER TABLE ONLY template_version_parameters ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; -ALTER TABLE ONLY template_version_preset_prebuilds - ADD CONSTRAINT template_version_preset_prebuilds_preset_id_fkey FOREIGN KEY (preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; - ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; diff --git a/coderd/database/foreign_key_constraint.go b/coderd/database/foreign_key_constraint.go index f762141505b4d..f7044815852cd 100644 --- a/coderd/database/foreign_key_constraint.go +++ b/coderd/database/foreign_key_constraint.go @@ -43,7 +43,6 @@ const ( ForeignKeyTailnetTunnelsCoordinatorID ForeignKeyConstraint = "tailnet_tunnels_coordinator_id_fkey" // ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_coordinator_id_fkey FOREIGN KEY (coordinator_id) REFERENCES tailnet_coordinators(id) ON DELETE CASCADE; ForeignKeyTemplateVersionParametersTemplateVersionID ForeignKeyConstraint = "template_version_parameters_template_version_id_fkey" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetParametTemplateVersionPresetID ForeignKeyConstraint = "template_version_preset_paramet_template_version_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; - ForeignKeyTemplateVersionPresetPrebuildsPresetID ForeignKeyConstraint = "template_version_preset_prebuilds_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_prebuilds ADD CONSTRAINT template_version_preset_prebuilds_preset_id_fkey FOREIGN KEY (preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetsTemplateVersionID ForeignKeyConstraint = "template_version_presets_template_version_id_fkey" // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionVariablesTemplateVersionID ForeignKeyConstraint = "template_version_variables_template_version_id_fkey" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionWorkspaceTagsTemplateVersionID ForeignKeyConstraint = "template_version_workspace_tags_template_version_id_fkey" // ALTER TABLE ONLY template_version_workspace_tags ADD CONSTRAINT template_version_workspace_tags_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; diff --git a/coderd/database/migrations/000303_preset_prebuilds.up.sql b/coderd/database/migrations/000303_preset_prebuilds.up.sql index f28607bbaf3a7..03ee6515e741c 100644 --- a/coderd/database/migrations/000303_preset_prebuilds.up.sql +++ b/coderd/database/migrations/000303_preset_prebuilds.up.sql @@ -1,13 +1,6 @@ -CREATE TABLE template_version_preset_prebuilds -( - id UUID PRIMARY KEY, - preset_id UUID NOT NULL, - desired_instances INT NOT NULL, - invalidate_after_secs INT NULL DEFAULT 0, - - -- Deletion should never occur, but if we allow it we should check no prebuilds exist attached to this preset first. - FOREIGN KEY (preset_id) REFERENCES template_version_presets (id) ON DELETE CASCADE -); +ALTER TABLE template_version_presets + ADD COLUMN desired_instances INT NULL, + ADD COLUMN invalidate_after_secs INT NULL DEFAULT 0; -- We should not be able to have presets with the same name for a particular template version. CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets (name, template_version_id); diff --git a/coderd/database/models.go b/coderd/database/models.go index ef3209ca2707c..9bf2923fd264e 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3108,10 +3108,12 @@ type TemplateVersionParameter struct { } type TemplateVersionPreset struct { - ID uuid.UUID `db:"id" json:"id"` - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - Name string `db:"name" json:"name"` - CreatedAt time.Time `db:"created_at" json:"created_at"` + ID uuid.UUID `db:"id" json:"id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + Name string `db:"name" json:"name"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"` + InvalidateAfterSecs sql.NullInt32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` } type TemplateVersionPresetParameter struct { @@ -3121,13 +3123,6 @@ type TemplateVersionPresetParameter struct { Value string `db:"value" json:"value"` } -type TemplateVersionPresetPrebuild struct { - ID uuid.UUID `db:"id" json:"id"` - PresetID uuid.UUID `db:"preset_id" json:"preset_id"` - DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` - InvalidateAfterSecs sql.NullInt32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` -} - type TemplateVersionTable struct { ID uuid.UUID `db:"id" json:"id"` TemplateID uuid.NullUUID `db:"template_id" json:"template_id"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 40477ffb8a2ff..43ab7b4266f86 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -455,7 +455,6 @@ type sqlcQuerier interface { InsertOrganizationMember(ctx context.Context, arg InsertOrganizationMemberParams) (OrganizationMember, error) InsertPreset(ctx context.Context, arg InsertPresetParams) (TemplateVersionPreset, error) InsertPresetParameters(ctx context.Context, arg InsertPresetParametersParams) ([]TemplateVersionPresetParameter, error) - InsertPresetPrebuild(ctx context.Context, arg InsertPresetPrebuildParams) (TemplateVersionPresetPrebuild, error) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) InsertProvisionerJobTimings(ctx context.Context, arg InsertProvisionerJobTimingsParams) ([]ProvisionerJobTiming, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 73ae0d124988d..95690e6e0dec6 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3511,17 +3511,17 @@ func TestGetPresetsBackoff(t *testing.T) { UpdatedAt: now, CreatedBy: tmpl.CreatedBy, }) - preset := dbgen.Preset(t, db, database.InsertPresetParams{ - TemplateVersionID: tmplVersion.ID, - Name: "preset", - }) desiredInstances := 1 if opts != nil { desiredInstances = opts.DesiredInstances } - dbgen.PresetPrebuild(t, db, database.InsertPresetPrebuildParams{ - PresetID: preset.ID, - DesiredInstances: int32(desiredInstances), + preset := dbgen.Preset(t, db, database.InsertPresetParams{ + TemplateVersionID: tmplVersion.ID, + Name: "preset", + DesiredInstances: sql.NullInt32{ + Int32: int32(desiredInstances), + Valid: true, + }, }) return extTmplVersion{ diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b7f99c7d63679..f068cd9c8fc3c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5892,26 +5892,29 @@ func (q *sqlQuerier) GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuilds const getPresetsBackoff = `-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances + SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id JOIN provisioner_jobs pj ON wlb.job_id = pj.id JOIN template_versions tv ON wlb.template_version_id = tv.id JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id - JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id - WHERE wlb.transition = 'start'::workspace_transition), - time_sorted_builds AS ( - -- Group builds by template version, then sort each group by created_at. - SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, - ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn - FROM filtered_builds fb), - failed_count AS ( - -- Count failed builds per template version/preset in the given period - SELECT preset_id, COUNT(*) AS num_failed - FROM filtered_builds - WHERE job_status = 'failed'::provisioner_job_status - AND created_at >= $1::timestamptz - GROUP BY preset_id) + WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. + AND wlb.transition = 'start'::workspace_transition +), +time_sorted_builds AS ( + -- Group builds by template version, then sort each group by created_at. + SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, + ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + FROM filtered_builds fb +), +failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= $1::timestamptz + GROUP BY preset_id +) SELECT tsb.template_version_id, tsb.preset_id, COALESCE(fc.num_failed, 0)::int AS num_failed, @@ -6040,29 +6043,29 @@ SELECT t.id AS template_id, tv.id AS template_version_id, tv.name AS template_version_name, tv.id = t.active_version_id AS using_active_version, - tvpp.preset_id, + tvp.id, tvp.name, - tvpp.desired_instances AS desired_instances, + tvp.desired_instances AS desired_instances, t.deleted, t.deprecated != '' AS deprecated FROM templates t INNER JOIN template_versions tv ON tv.template_id = t.id INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id - INNER JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id -WHERE (t.id = $1::uuid OR $1 IS NULL) +WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. + AND (t.id = $1::uuid OR $1 IS NULL) ` type GetTemplatePresetsWithPrebuildsRow struct { - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - TemplateName string `db:"template_name" json:"template_name"` - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - TemplateVersionName string `db:"template_version_name" json:"template_version_name"` - UsingActiveVersion bool `db:"using_active_version" json:"using_active_version"` - PresetID uuid.UUID `db:"preset_id" json:"preset_id"` - Name string `db:"name" json:"name"` - DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` - Deleted bool `db:"deleted" json:"deleted"` - Deprecated bool `db:"deprecated" json:"deprecated"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateName string `db:"template_name" json:"template_name"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + TemplateVersionName string `db:"template_version_name" json:"template_version_name"` + UsingActiveVersion bool `db:"using_active_version" json:"using_active_version"` + ID uuid.UUID `db:"id" json:"id"` + Name string `db:"name" json:"name"` + DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"` + Deleted bool `db:"deleted" json:"deleted"` + Deprecated bool `db:"deprecated" json:"deprecated"` } // GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. @@ -6083,7 +6086,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa &i.TemplateVersionID, &i.TemplateVersionName, &i.UsingActiveVersion, - &i.PresetID, + &i.ID, &i.Name, &i.DesiredInstances, &i.Deleted, @@ -6102,39 +6105,9 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa return items, nil } -const insertPresetPrebuild = `-- name: InsertPresetPrebuild :one -INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances, invalidate_after_secs) -VALUES ($1::uuid, $2::uuid, $3::int, $4::int) -RETURNING id, preset_id, desired_instances, invalidate_after_secs -` - -type InsertPresetPrebuildParams struct { - ID uuid.UUID `db:"id" json:"id"` - PresetID uuid.UUID `db:"preset_id" json:"preset_id"` - DesiredInstances int32 `db:"desired_instances" json:"desired_instances"` - InvalidateAfterSecs int32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` -} - -func (q *sqlQuerier) InsertPresetPrebuild(ctx context.Context, arg InsertPresetPrebuildParams) (TemplateVersionPresetPrebuild, error) { - row := q.db.QueryRowContext(ctx, insertPresetPrebuild, - arg.ID, - arg.PresetID, - arg.DesiredInstances, - arg.InvalidateAfterSecs, - ) - var i TemplateVersionPresetPrebuild - err := row.Scan( - &i.ID, - &i.PresetID, - &i.DesiredInstances, - &i.InvalidateAfterSecs, - ) - return i, err -} - const getPresetByWorkspaceBuildID = `-- name: GetPresetByWorkspaceBuildID :one SELECT - template_version_presets.id, template_version_presets.template_version_id, template_version_presets.name, template_version_presets.created_at + template_version_presets.id, template_version_presets.template_version_id, template_version_presets.name, template_version_presets.created_at, template_version_presets.desired_instances, template_version_presets.invalidate_after_secs FROM template_version_presets INNER JOIN workspace_builds ON workspace_builds.template_version_preset_id = template_version_presets.id @@ -6150,6 +6123,8 @@ func (q *sqlQuerier) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceB &i.TemplateVersionID, &i.Name, &i.CreatedAt, + &i.DesiredInstances, + &i.InvalidateAfterSecs, ) return i, err } @@ -6194,7 +6169,7 @@ func (q *sqlQuerier) GetPresetParametersByTemplateVersionID(ctx context.Context, const getPresetsByTemplateVersionID = `-- name: GetPresetsByTemplateVersionID :many SELECT - id, template_version_id, name, created_at + id, template_version_id, name, created_at, desired_instances, invalidate_after_secs FROM template_version_presets WHERE @@ -6215,6 +6190,8 @@ func (q *sqlQuerier) GetPresetsByTemplateVersionID(ctx context.Context, template &i.TemplateVersionID, &i.Name, &i.CreatedAt, + &i.DesiredInstances, + &i.InvalidateAfterSecs, ); err != nil { return nil, err } @@ -6230,26 +6207,46 @@ func (q *sqlQuerier) GetPresetsByTemplateVersionID(ctx context.Context, template } const insertPreset = `-- name: InsertPreset :one -INSERT INTO - template_version_presets (template_version_id, name, created_at) -VALUES - ($1, $2, $3) RETURNING id, template_version_id, name, created_at +INSERT INTO template_version_presets ( + template_version_id, + name, + created_at, + desired_instances, + invalidate_after_secs +) +VALUES ( + $1, + $2, + $3, + $4, + $5 +) RETURNING id, template_version_id, name, created_at, desired_instances, invalidate_after_secs ` type InsertPresetParams struct { - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - Name string `db:"name" json:"name"` - CreatedAt time.Time `db:"created_at" json:"created_at"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + Name string `db:"name" json:"name"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"` + InvalidateAfterSecs sql.NullInt32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` } func (q *sqlQuerier) InsertPreset(ctx context.Context, arg InsertPresetParams) (TemplateVersionPreset, error) { - row := q.db.QueryRowContext(ctx, insertPreset, arg.TemplateVersionID, arg.Name, arg.CreatedAt) + row := q.db.QueryRowContext(ctx, insertPreset, + arg.TemplateVersionID, + arg.Name, + arg.CreatedAt, + arg.DesiredInstances, + arg.InvalidateAfterSecs, + ) var i TemplateVersionPreset err := row.Scan( &i.ID, &i.TemplateVersionID, &i.Name, &i.CreatedAt, + &i.DesiredInstances, + &i.InvalidateAfterSecs, ) return i, err } diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index edb9c51dc5fd6..f72d6102251b9 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -7,16 +7,16 @@ SELECT t.id AS template_id, tv.id AS template_version_id, tv.name AS template_version_name, tv.id = t.active_version_id AS using_active_version, - tvpp.preset_id, + tvp.id, tvp.name, - tvpp.desired_instances AS desired_instances, + tvp.desired_instances AS desired_instances, t.deleted, t.deprecated != '' AS deprecated FROM templates t INNER JOIN template_versions tv ON tv.template_id = t.id INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id - INNER JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id -WHERE (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); +WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. + AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); -- name: GetRunningPrebuilds :many SELECT p.id AS workspace_id, @@ -64,26 +64,29 @@ GROUP BY t.id, wpb.template_version_id, wpb.transition; -- The number of failed builds is used downstream to determine the backoff duration. WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvpp.desired_instances + SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id JOIN provisioner_jobs pj ON wlb.job_id = pj.id JOIN template_versions tv ON wlb.template_version_id = tv.id JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id - JOIN template_version_preset_prebuilds tvpp ON tvpp.preset_id = tvp.id - WHERE wlb.transition = 'start'::workspace_transition), - time_sorted_builds AS ( - -- Group builds by template version, then sort each group by created_at. - SELECT fb.*, - ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn - FROM filtered_builds fb), - failed_count AS ( - -- Count failed builds per template version/preset in the given period - SELECT preset_id, COUNT(*) AS num_failed - FROM filtered_builds - WHERE job_status = 'failed'::provisioner_job_status - AND created_at >= @lookback::timestamptz - GROUP BY preset_id) + WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. + AND wlb.transition = 'start'::workspace_transition +), +time_sorted_builds AS ( + -- Group builds by template version, then sort each group by created_at. + SELECT fb.*, + ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + FROM filtered_builds fb +), +failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= @lookback::timestamptz + GROUP BY preset_id +) SELECT tsb.template_version_id, tsb.preset_id, COALESCE(fc.num_failed, 0)::int AS num_failed, @@ -113,11 +116,6 @@ WHERE w.id IN (SELECT p.id LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. RETURNING w.id, w.name; --- name: InsertPresetPrebuild :one -INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances, invalidate_after_secs) -VALUES (@id::uuid, @preset_id::uuid, @desired_instances::int, @invalidate_after_secs::int) -RETURNING *; - -- name: GetPrebuildMetrics :many SELECT t.name as template_name, diff --git a/coderd/database/queries/presets.sql b/coderd/database/queries/presets.sql index 8e648fce6ca88..93c44fbc5898a 100644 --- a/coderd/database/queries/presets.sql +++ b/coderd/database/queries/presets.sql @@ -1,8 +1,18 @@ -- name: InsertPreset :one -INSERT INTO - template_version_presets (template_version_id, name, created_at) -VALUES - (@template_version_id, @name, @created_at) RETURNING *; +INSERT INTO template_version_presets ( + template_version_id, + name, + created_at, + desired_instances, + invalidate_after_secs +) +VALUES ( + @template_version_id, + @name, + @created_at, + @desired_instances, + @invalidate_after_secs +) RETURNING *; -- name: InsertPresetParameters :many INSERT INTO diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index 648508e957e47..8fcd3cb3c84bc 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -59,7 +59,6 @@ const ( UniqueTemplateUsageStatsPkey UniqueConstraint = "template_usage_stats_pkey" // ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); UniqueTemplateVersionParametersTemplateVersionIDNameKey UniqueConstraint = "template_version_parameters_template_version_id_name_key" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionPresetParametersPkey UniqueConstraint = "template_version_preset_parameters_pkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id); - UniqueTemplateVersionPresetPrebuildsPkey UniqueConstraint = "template_version_preset_prebuilds_pkey" // ALTER TABLE ONLY template_version_preset_prebuilds ADD CONSTRAINT template_version_preset_prebuilds_pkey PRIMARY KEY (id); UniqueTemplateVersionPresetsPkey UniqueConstraint = "template_version_presets_pkey" // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id); UniqueTemplateVersionVariablesTemplateVersionIDNameKey UniqueConstraint = "template_version_variables_template_version_id_name_key" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionWorkspaceTagsTemplateVersionIDKeyKey UniqueConstraint = "template_version_workspace_tags_template_version_id_key_key" // ALTER TABLE ONLY template_version_workspace_tags ADD CONSTRAINT template_version_workspace_tags_template_version_id_key_key UNIQUE (template_version_id, key); From ee1f16a024d95de2363a4434dc507fddd503da9e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 21 Mar 2025 16:37:52 -0400 Subject: [PATCH 22/72] refactor: rename sql table --- coderd/database/dump.sql | 38 +++++++++---------- .../migrations/000302_prebuilds.up.sql | 8 ++-- coderd/database/queries.sql.go | 8 ++-- coderd/database/queries/prebuilds.sql | 8 ++-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index fa011eaf68181..57e86a331e313 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1894,7 +1894,7 @@ CREATE VIEW workspace_build_with_user AS COMMENT ON VIEW workspace_build_with_user IS 'Joins in the username + avatar url of the initiated by user.'; -CREATE VIEW workspace_latest_build AS +CREATE VIEW workspace_latest_builds AS SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.id, workspace_builds.created_at, workspace_builds.updated_at, @@ -2550,30 +2550,30 @@ CREATE OR REPLACE VIEW workspace_prebuilds AS FROM workspaces w WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) ), latest_prebuild_builds AS ( - SELECT workspace_latest_build.id, - workspace_latest_build.created_at, - workspace_latest_build.updated_at, - workspace_latest_build.workspace_id, - workspace_latest_build.template_version_id, - workspace_latest_build.build_number, - workspace_latest_build.transition, - workspace_latest_build.initiator_id, - workspace_latest_build.provisioner_state, - workspace_latest_build.job_id, - workspace_latest_build.deadline, - workspace_latest_build.reason, - workspace_latest_build.daily_cost, - workspace_latest_build.max_deadline, - workspace_latest_build.template_version_preset_id - FROM workspace_latest_build - WHERE (workspace_latest_build.template_version_preset_id IS NOT NULL) + SELECT workspace_latest_builds.id, + workspace_latest_builds.created_at, + workspace_latest_builds.updated_at, + workspace_latest_builds.workspace_id, + workspace_latest_builds.template_version_id, + workspace_latest_builds.build_number, + workspace_latest_builds.transition, + workspace_latest_builds.initiator_id, + workspace_latest_builds.provisioner_state, + workspace_latest_builds.job_id, + workspace_latest_builds.deadline, + workspace_latest_builds.reason, + workspace_latest_builds.daily_cost, + workspace_latest_builds.max_deadline, + workspace_latest_builds.template_version_preset_id + FROM workspace_latest_builds + WHERE (workspace_latest_builds.template_version_preset_id IS NOT NULL) ), workspace_agents AS ( SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at FROM (((workspaces w - JOIN workspace_latest_build wlb ON ((wlb.workspace_id = w.id))) + JOIN workspace_latest_builds wlb ON ((wlb.workspace_id = w.id))) JOIN workspace_resources wr ON ((wr.job_id = wlb.job_id))) JOIN workspace_agents wa ON ((wa.resource_id = wr.id))) WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) diff --git a/coderd/database/migrations/000302_prebuilds.up.sql b/coderd/database/migrations/000302_prebuilds.up.sql index b4e6cba939864..f9578a28cc7cb 100644 --- a/coderd/database/migrations/000302_prebuilds.up.sql +++ b/coderd/database/migrations/000302_prebuilds.up.sql @@ -1,5 +1,5 @@ --- workspace_latest_build contains latest build for every workspace -CREATE VIEW workspace_latest_build AS +-- workspace_latest_builds contains latest build for every workspace +CREATE VIEW workspace_latest_builds AS SELECT DISTINCT ON (workspace_id) * FROM workspace_builds ORDER BY workspace_id, build_number DESC; @@ -18,12 +18,12 @@ WITH -- -- See https://github.com/coder/internal/issues/398 latest_prebuild_builds AS (SELECT * - FROM workspace_latest_build + FROM workspace_latest_builds WHERE template_version_preset_id IS NOT NULL), -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. workspace_agents AS (SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at FROM workspaces w - INNER JOIN workspace_latest_build wlb ON wlb.workspace_id = w.id + INNER JOIN workspace_latest_builds wlb ON wlb.workspace_id = w.id INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id INNER JOIN workspace_agents wa ON wa.resource_id = wr.id WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index f068cd9c8fc3c..228ba3c83fc50 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5757,7 +5757,7 @@ SET owner_id = $1::uuid, updated_at = NOW() WHERE w.id IN (SELECT p.id FROM workspace_prebuilds p - INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition @@ -5846,7 +5846,7 @@ func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetri const getPrebuildsInProgress = `-- name: GetPrebuildsInProgress :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count -FROM workspace_latest_build wlb +FROM workspace_latest_builds wlb INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id INNER JOIN templates t ON t.active_version_id = wlb.template_version_id @@ -5894,7 +5894,7 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp - JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id + JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id JOIN provisioner_jobs pj ON wlb.job_id = pj.id JOIN template_versions tv ON wlb.template_version_id = tv.id JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id @@ -5987,7 +5987,7 @@ SELECT p.id AS workspace_id, ELSE FALSE END AS ready, p.created_at FROM workspace_prebuilds p - INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id LEFT JOIN template_version_presets tvp_curr diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index f72d6102251b9..517d478728820 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -29,7 +29,7 @@ SELECT p.id AS workspace_id, ELSE FALSE END AS ready, p.created_at FROM workspace_prebuilds p - INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id LEFT JOIN template_version_presets tvp_curr @@ -39,7 +39,7 @@ WHERE (b.transition = 'start'::workspace_transition -- name: GetPrebuildsInProgress :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count -FROM workspace_latest_build wlb +FROM workspace_latest_builds wlb INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id INNER JOIN templates t ON t.active_version_id = wlb.template_version_id @@ -66,7 +66,7 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp - JOIN workspace_latest_build wlb ON wlb.template_version_preset_id = tvp.id + JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id JOIN provisioner_jobs pj ON wlb.job_id = pj.id JOIN template_versions tv ON wlb.template_version_id = tv.id JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id @@ -104,7 +104,7 @@ SET owner_id = @new_user_id::uuid, updated_at = NOW() WHERE w.id IN (SELECT p.id FROM workspace_prebuilds p - INNER JOIN workspace_latest_build b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition From d040ddd93afbcd197390a5349b9fd8ceea24446f Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Sun, 23 Mar 2025 17:03:59 -0400 Subject: [PATCH 23/72] refactor: remove unnecessary JOIN --- coderd/database/dbauthz/dbauthz_test.go | 71 +++++++++++++------------ coderd/database/queries.sql.go | 1 - coderd/database/queries/prebuilds.sql | 1 - 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 4693437ed17c7..901d23ad609c0 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1692,7 +1692,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1723,7 +1723,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1776,7 +1776,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3757,7 +3757,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -3934,7 +3934,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -3979,14 +3979,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -3994,7 +3994,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4005,21 +4005,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4675,30 +4675,31 @@ func (s *MethodTestSuite) TestPrebuilds() { Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) - s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { - org := dbgen.Organization(s.T(), db, database.Organization{}) - user := dbgen.User(s.T(), db, database.User{}) - template := dbgen.Template(s.T(), db, database.Template{ - CreatedBy: user.ID, - OrganizationID: org.ID, - }) - templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ - TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true}, - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ - Name: coderdtest.RandomName(s.T()), - TemplateVersionID: templateVersion.ID, - }) - check.Args(database.InsertPresetPrebuildParams{ - ID: uuid.New(), - PresetID: preset.ID, - DesiredInstances: 1, - }). - Asserts(rbac.ResourceSystem, policy.ActionCreate). - ErrorsWithInMemDB(dbmem.ErrUnimplemented) - })) + // TODO: remove? + //s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { + // org := dbgen.Organization(s.T(), db, database.Organization{}) + // user := dbgen.User(s.T(), db, database.User{}) + // template := dbgen.Template(s.T(), db, database.Template{ + // CreatedBy: user.ID, + // OrganizationID: org.ID, + // }) + // templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ + // TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true}, + // OrganizationID: org.ID, + // CreatedBy: user.ID, + // }) + // preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ + // Name: coderdtest.RandomName(s.T()), + // TemplateVersionID: templateVersion.ID, + // }) + // check.Args(database.InsertPresetPrebuildParams{ + // ID: uuid.New(), + // PresetID: preset.ID, + // DesiredInstances: 1, + // }). + // Asserts(rbac.ResourceSystem, policy.ActionCreate). + // ErrorsWithInMemDB(dbmem.ErrUnimplemented) + //})) } func (s *MethodTestSuite) TestOAuth2ProviderApps() { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 228ba3c83fc50..3811d507f0490 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5989,7 +5989,6 @@ SELECT p.id AS workspace_id, FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - INNER JOIN templates t ON p.template_id = t.id LEFT JOIN template_version_presets tvp_curr ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 517d478728820..8f47aa34022b0 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -31,7 +31,6 @@ SELECT p.id AS workspace_id, FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - INNER JOIN templates t ON p.template_id = t.id LEFT JOIN template_version_presets tvp_curr ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition From 83a67223b46e2064c6a310bcc68f82fe44f4796e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Sun, 23 Mar 2025 17:20:32 -0400 Subject: [PATCH 24/72] refactor: remove unnecessary JOIN --- coderd/database/queries.sql.go | 10 ++++------ coderd/database/queries/prebuilds.sql | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 3811d507f0490..ed7459ab2dc68 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5977,20 +5977,18 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) } const getRunningPrebuilds = `-- name: GetRunningPrebuilds :many -SELECT p.id AS workspace_id, - p.name AS workspace_name, +SELECT p.id AS workspace_id, + p.name AS workspace_name, p.template_id, b.template_version_id, - tvp_curr.id AS current_preset_id, + p.current_preset_id AS current_preset_id, CASE WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE ELSE FALSE END AS ready, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - LEFT JOIN template_version_presets tvp_curr - ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition AND pj.job_status = 'succeeded'::provisioner_job_status) ` diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 8f47aa34022b0..f8fa60e57642d 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -19,20 +19,18 @@ WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a pre AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); -- name: GetRunningPrebuilds :many -SELECT p.id AS workspace_id, - p.name AS workspace_name, +SELECT p.id AS workspace_id, + p.name AS workspace_name, p.template_id, b.template_version_id, - tvp_curr.id AS current_preset_id, + p.current_preset_id AS current_preset_id, CASE WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE ELSE FALSE END AS ready, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - LEFT JOIN template_version_presets tvp_curr - ON tvp_curr.id = p.current_preset_id -- See https://github.com/coder/internal/issues/398. + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition AND pj.job_status = 'succeeded'::provisioner_job_status); From cd70710f82fb82fe5ba10f7afeec92a40b6d33a1 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Sun, 23 Mar 2025 17:26:28 -0400 Subject: [PATCH 25/72] refactor: use INNER JOIN for consistency --- coderd/database/queries.sql.go | 8 ++++---- coderd/database/queries/prebuilds.sql | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ed7459ab2dc68..887dda88d5810 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5894,10 +5894,10 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp - JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - JOIN provisioner_jobs pj ON wlb.job_id = pj.id - JOIN template_versions tv ON wlb.template_version_id = tv.id - JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id + INNER JOIN template_versions tv ON wlb.template_version_id = tv.id + INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND wlb.transition = 'start'::workspace_transition ), diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index f8fa60e57642d..878cb0992135a 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -63,10 +63,10 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp - JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - JOIN provisioner_jobs pj ON wlb.job_id = pj.id - JOIN template_versions tv ON wlb.template_version_id = tv.id - JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id + INNER JOIN template_versions tv ON wlb.template_version_id = tv.id + INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND wlb.transition = 'start'::workspace_transition ), From 97cc4ffc4399b1a6594ac93e94c42625d9464842 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 24 Mar 2025 11:55:55 -0400 Subject: [PATCH 26/72] refactor: simplify GetPresetsBackoff SQL Query --- coderd/database/dbauthz/dbauthz_test.go | 22 +++++++++--------- coderd/database/querier.go | 13 ++++------- coderd/database/querier_test.go | 18 ++++++++++---- coderd/database/queries.sql.go | 29 +++++++---------------- coderd/database/queries/prebuilds.sql | 31 +++++++------------------ 5 files changed, 45 insertions(+), 68 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 901d23ad609c0..f36d737d15c6c 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1692,7 +1692,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1723,7 +1723,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1776,7 +1776,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3757,7 +3757,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -3934,7 +3934,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -3979,14 +3979,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -3994,7 +3994,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4005,21 +4005,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 43ab7b4266f86..0b428de0836a2 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -226,20 +226,15 @@ type sqlcQuerier interface { GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by template version ID. - // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. + // For each group, the query checks up to N of the most recent jobs that occurred within the + // lookback period, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. // // NOTE: - // We back off on the template version ID if at least one of the N latest workspace builds has failed. - // However, we also return the number of failed workspace builds that occurred during the lookback period. - // - // In other words: - // - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). - // - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. - // - // The number of failed builds is used downstream to determine the backoff duration. + // We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. + // We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPreset, error) GetPreviousTemplateVersion(ctx context.Context, arg GetPreviousTemplateVersionParams) (TemplateVersion, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 95690e6e0dec6..1f7e68a0da47f 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3631,7 +3631,9 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 3, + }) createWorkspaceBuild(db, tmpl, tmplV1, nil) createWorkspaceBuild(db, tmpl, tmplV1, nil) createWorkspaceBuild(db, tmpl, tmplV1, nil) @@ -3663,7 +3665,9 @@ func TestGetPresetsBackoff(t *testing.T) { createWorkspaceBuild(db, tmpl, tmplV1, nil) // Active Version - tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 2, + }) createWorkspaceBuild(db, tmpl, tmplV2, nil) createWorkspaceBuild(db, tmpl, tmplV2, nil) @@ -3732,7 +3736,9 @@ func TestGetPresetsBackoff(t *testing.T) { createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) tmpl2 := createTemplate(db) - tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) + tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 2, + }) createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) @@ -3740,7 +3746,9 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl3V1 := createTmplVersion(db, tmpl3, uuid.New(), nil) createWorkspaceBuild(db, tmpl3, tmpl3V1, nil) - tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, nil) + tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 3, + }) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) @@ -3942,7 +3950,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(db) tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 3, + DesiredInstances: 5, }) createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ successfulJob: false, diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 887dda88d5810..f0f9f42166f7b 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5899,31 +5899,23 @@ WITH filtered_builds AS ( INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition + AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb -), -failed_count AS ( - -- Count failed builds per template version/preset in the given period - SELECT preset_id, COUNT(*) AS num_failed - FROM filtered_builds - WHERE job_status = 'failed'::provisioner_job_status - AND created_at >= $1::timestamptz - GROUP BY preset_id ) SELECT tsb.template_version_id, tsb.preset_id, - COALESCE(fc.num_failed, 0)::int AS num_failed, + COUNT(*)::int AS num_failed, -- Count failed builds per template version/preset in the given period MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb - LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed + AND created_at >= $1::timestamptz +GROUP BY tsb.template_version_id, tsb.preset_id ` type GetPresetsBackoffRow struct { @@ -5934,20 +5926,15 @@ type GetPresetsBackoffRow struct { } // GetPresetsBackoff groups workspace builds by template version ID. -// For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. +// For each group, the query checks up to N of the most recent jobs that occurred within the +// lookback period, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. // // NOTE: -// We back off on the template version ID if at least one of the N latest workspace builds has failed. -// However, we also return the number of failed workspace builds that occurred during the lookback period. -// -// In other words: -// - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). -// - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. -// -// The number of failed builds is used downstream to determine the backoff duration. +// We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. +// We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) { rows, err := q.db.QueryContext(ctx, getPresetsBackoff, lookback) if err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 878cb0992135a..5dde2b21865a0 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -43,22 +43,17 @@ FROM workspace_latest_builds wlb WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; --- name: GetPresetsBackoff :many -- GetPresetsBackoff groups workspace builds by template version ID. --- For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. +-- For each group, the query checks up to N of the most recent jobs that occurred within the +-- lookback period, where N equals the number of desired instances for the corresponding preset. -- If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. -- Query returns a list of template version IDs for which we should backoff. -- Only active template versions with configured presets are considered. -- -- NOTE: --- We back off on the template version ID if at least one of the N latest workspace builds has failed. --- However, we also return the number of failed workspace builds that occurred during the lookback period. --- --- In other words: --- - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). --- - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. --- --- The number of failed builds is used downstream to determine the backoff duration. +-- We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. +-- We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. +-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances @@ -68,31 +63,23 @@ WITH filtered_builds AS ( INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition + AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. SELECT fb.*, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb -), -failed_count AS ( - -- Count failed builds per template version/preset in the given period - SELECT preset_id, COUNT(*) AS num_failed - FROM filtered_builds - WHERE job_status = 'failed'::provisioner_job_status - AND created_at >= @lookback::timestamptz - GROUP BY preset_id ) SELECT tsb.template_version_id, tsb.preset_id, - COALESCE(fc.num_failed, 0)::int AS num_failed, + COUNT(*)::int AS num_failed, -- Count failed builds per template version/preset in the given period MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb - LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status -GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; + AND created_at >= @lookback::timestamptz +GROUP BY tsb.template_version_id, tsb.preset_id; -- name: ClaimPrebuild :one UPDATE workspaces w From 4d59039e62421921fee83f460fcd53f8650c30a8 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 24 Mar 2025 13:36:08 -0400 Subject: [PATCH 27/72] Revert "refactor: simplify GetPresetsBackoff SQL Query" This reverts commit 97cc4ffc4399b1a6594ac93e94c42625d9464842. --- coderd/database/dbauthz/dbauthz_test.go | 22 +++++++++--------- coderd/database/querier.go | 13 +++++++---- coderd/database/querier_test.go | 18 ++++---------- coderd/database/queries.sql.go | 29 ++++++++++++++++------- coderd/database/queries/prebuilds.sql | 31 ++++++++++++++++++------- 5 files changed, 68 insertions(+), 45 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index f36d737d15c6c..901d23ad609c0 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1692,7 +1692,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1723,7 +1723,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1776,7 +1776,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3757,7 +3757,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -3934,7 +3934,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -3979,14 +3979,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -3994,7 +3994,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4005,21 +4005,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 0b428de0836a2..43ab7b4266f86 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -226,15 +226,20 @@ type sqlcQuerier interface { GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by template version ID. - // For each group, the query checks up to N of the most recent jobs that occurred within the - // lookback period, where N equals the number of desired instances for the corresponding preset. + // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. // // NOTE: - // We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. - // We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. + // We back off on the template version ID if at least one of the N latest workspace builds has failed. + // However, we also return the number of failed workspace builds that occurred during the lookback period. + // + // In other words: + // - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). + // - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. + // + // The number of failed builds is used downstream to determine the backoff duration. GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) GetPresetsByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPreset, error) GetPreviousTemplateVersion(ctx context.Context, arg GetPreviousTemplateVersionParams) (TemplateVersion, error) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 1f7e68a0da47f..95690e6e0dec6 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3631,9 +3631,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 3, - }) + tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) createWorkspaceBuild(db, tmpl, tmplV1, nil) createWorkspaceBuild(db, tmpl, tmplV1, nil) createWorkspaceBuild(db, tmpl, tmplV1, nil) @@ -3665,9 +3663,7 @@ func TestGetPresetsBackoff(t *testing.T) { createWorkspaceBuild(db, tmpl, tmplV1, nil) // Active Version - tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 2, - }) + tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) createWorkspaceBuild(db, tmpl, tmplV2, nil) createWorkspaceBuild(db, tmpl, tmplV2, nil) @@ -3736,9 +3732,7 @@ func TestGetPresetsBackoff(t *testing.T) { createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) tmpl2 := createTemplate(db) - tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 2, - }) + tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) @@ -3746,9 +3740,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl3V1 := createTmplVersion(db, tmpl3, uuid.New(), nil) createWorkspaceBuild(db, tmpl3, tmpl3V1, nil) - tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 3, - }) + tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, nil) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) @@ -3950,7 +3942,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(db) tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ - DesiredInstances: 5, + DesiredInstances: 3, }) createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ successfulJob: false, diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index f0f9f42166f7b..887dda88d5810 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5899,23 +5899,31 @@ WITH filtered_builds AS ( INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition + AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb +), +failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= $1::timestamptz + GROUP BY preset_id ) SELECT tsb.template_version_id, tsb.preset_id, - COUNT(*)::int AS num_failed, -- Count failed builds per template version/preset in the given period + COALESCE(fc.num_failed, 0)::int AS num_failed, MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status - AND created_at >= $1::timestamptz -GROUP BY tsb.template_version_id, tsb.preset_id +GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed ` type GetPresetsBackoffRow struct { @@ -5926,15 +5934,20 @@ type GetPresetsBackoffRow struct { } // GetPresetsBackoff groups workspace builds by template version ID. -// For each group, the query checks up to N of the most recent jobs that occurred within the -// lookback period, where N equals the number of desired instances for the corresponding preset. +// For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. // // NOTE: -// We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. -// We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. +// We back off on the template version ID if at least one of the N latest workspace builds has failed. +// However, we also return the number of failed workspace builds that occurred during the lookback period. +// +// In other words: +// - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +// - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. +// +// The number of failed builds is used downstream to determine the backoff duration. func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]GetPresetsBackoffRow, error) { rows, err := q.db.QueryContext(ctx, getPresetsBackoff, lookback) if err != nil { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 5dde2b21865a0..878cb0992135a 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -43,17 +43,22 @@ FROM workspace_latest_builds wlb WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; +-- name: GetPresetsBackoff :many -- GetPresetsBackoff groups workspace builds by template version ID. --- For each group, the query checks up to N of the most recent jobs that occurred within the --- lookback period, where N equals the number of desired instances for the corresponding preset. +-- For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. -- If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. -- Query returns a list of template version IDs for which we should backoff. -- Only active template versions with configured presets are considered. -- -- NOTE: --- We only consider jobs that occurred within the lookback period; any failures that happened before this period are ignored. --- We also return the number of failed workspace builds, which is used downstream to determine the backoff duration. --- name: GetPresetsBackoff :many +-- We back off on the template version ID if at least one of the N latest workspace builds has failed. +-- However, we also return the number of failed workspace builds that occurred during the lookback period. +-- +-- In other words: +-- - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +-- - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. +-- +-- The number of failed builds is used downstream to determine the backoff duration. WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances @@ -63,23 +68,31 @@ WITH filtered_builds AS ( INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition + AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. SELECT fb.*, ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb +), +failed_count AS ( + -- Count failed builds per template version/preset in the given period + SELECT preset_id, COUNT(*) AS num_failed + FROM filtered_builds + WHERE job_status = 'failed'::provisioner_job_status + AND created_at >= @lookback::timestamptz + GROUP BY preset_id ) SELECT tsb.template_version_id, tsb.preset_id, - COUNT(*)::int AS num_failed, -- Count failed builds per template version/preset in the given period + COALESCE(fc.num_failed, 0)::int AS num_failed, MAX(tsb.created_at::timestamptz) AS last_build_at FROM time_sorted_builds tsb + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status - AND created_at >= @lookback::timestamptz -GROUP BY tsb.template_version_id, tsb.preset_id; +GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; -- name: ClaimPrebuild :one UPDATE workspaces w From 205d6af30ac51ad7299dadd16dedeac222ae637f Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 24 Mar 2025 15:10:28 -0400 Subject: [PATCH 28/72] refactor: improve GetPresetsBackoff query --- coderd/database/querier.go | 12 +++++------- coderd/database/querier_test.go | 27 +++++++++++++++++++++++++++ coderd/database/queries.sql.go | 13 ++++++------- coderd/database/queries/prebuilds.sql | 15 +++++++-------- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 43ab7b4266f86..24c7d831b7e5f 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -226,17 +226,15 @@ type sqlcQuerier interface { GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by template version ID. - // For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. - // If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. + // For each group, the query checks up to N of the most recent jobs that occurred within the + // lookback period, where N equals the number of desired instances for the corresponding preset. + // If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. + // We also return the number of failed workspace builds that occurred during the lookback period. // // NOTE: - // We back off on the template version ID if at least one of the N latest workspace builds has failed. - // However, we also return the number of failed workspace builds that occurred during the lookback period. - // - // In other words: - // - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). + // - To **decide whether to back off**, we look at up to the N most recent builds (within the defined lookback period). // - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. // // The number of failed builds is used downstream to determine the backoff duration. diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 95690e6e0dec6..6c1844f13721d 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -4032,6 +4032,33 @@ func TestGetPresetsBackoff(t *testing.T) { require.Equal(t, 0, now.Compare(backoff.LastBuildAt.(time.Time))) } }) + + t.Run("failed job outside lookback period", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + lookbackPeriod := time.Hour + + tmpl1 := createTemplate(db) + tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + DesiredInstances: 1, + }) + createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + successfulJob: false, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped + }) + + backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) + require.NoError(t, err) + require.Len(t, backoffs, 0) + }) } func requireUsersMatch(t testing.TB, expected []database.User, found []database.GetUsersRow, msg string) { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 887dda88d5810..ef432e440e1f1 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5923,6 +5923,7 @@ FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status + AND created_at >= $1::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed ` @@ -5934,17 +5935,15 @@ type GetPresetsBackoffRow struct { } // GetPresetsBackoff groups workspace builds by template version ID. -// For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. -// If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. +// For each group, the query checks up to N of the most recent jobs that occurred within the +// lookback period, where N equals the number of desired instances for the corresponding preset. +// If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. // Query returns a list of template version IDs for which we should backoff. // Only active template versions with configured presets are considered. +// We also return the number of failed workspace builds that occurred during the lookback period. // // NOTE: -// We back off on the template version ID if at least one of the N latest workspace builds has failed. -// However, we also return the number of failed workspace builds that occurred during the lookback period. -// -// In other words: -// - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +// - To **decide whether to back off**, we look at up to the N most recent builds (within the defined lookback period). // - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. // // The number of failed builds is used downstream to determine the backoff duration. diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 878cb0992135a..b9e29689e82ad 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -43,22 +43,20 @@ FROM workspace_latest_builds wlb WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; --- name: GetPresetsBackoff :many -- GetPresetsBackoff groups workspace builds by template version ID. --- For each group, the query checks the last N jobs, where N equals the number of desired instances for the corresponding preset. --- If at least one of the last N jobs has failed, we should backoff on the corresponding template version ID. +-- For each group, the query checks up to N of the most recent jobs that occurred within the +-- lookback period, where N equals the number of desired instances for the corresponding preset. +-- If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. -- Query returns a list of template version IDs for which we should backoff. -- Only active template versions with configured presets are considered. +-- We also return the number of failed workspace builds that occurred during the lookback period. -- -- NOTE: --- We back off on the template version ID if at least one of the N latest workspace builds has failed. --- However, we also return the number of failed workspace builds that occurred during the lookback period. --- --- In other words: --- - To **decide whether to back off**, we look at the N most recent builds (regardless of when they happened). +-- - To **decide whether to back off**, we look at up to the N most recent builds (within the defined lookback period). -- - To **calculate the number of failed builds**, we consider all builds within the defined lookback period. -- -- The number of failed builds is used downstream to determine the backoff duration. +-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances @@ -92,6 +90,7 @@ FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff AND tsb.job_status = 'failed'::provisioner_job_status + AND created_at >= @lookback::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; -- name: ClaimPrebuild :one From 20470e4534fa47aa53f5d50b36f324ea6f241fd3 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 09:34:41 -0400 Subject: [PATCH 29/72] fix: bump migration numbers --- .../{000309_prebuilds.down.sql => 000310_prebuilds.down.sql} | 0 .../{000309_prebuilds.up.sql => 000310_prebuilds.up.sql} | 0 ...preset_prebuilds.down.sql => 000311_preset_prebuilds.down.sql} | 0 ...310_preset_prebuilds.up.sql => 000311_preset_prebuilds.up.sql} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000309_prebuilds.down.sql => 000310_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000309_prebuilds.up.sql => 000310_prebuilds.up.sql} (100%) rename coderd/database/migrations/{000310_preset_prebuilds.down.sql => 000311_preset_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000310_preset_prebuilds.up.sql => 000311_preset_prebuilds.up.sql} (100%) diff --git a/coderd/database/migrations/000309_prebuilds.down.sql b/coderd/database/migrations/000310_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000309_prebuilds.down.sql rename to coderd/database/migrations/000310_prebuilds.down.sql diff --git a/coderd/database/migrations/000309_prebuilds.up.sql b/coderd/database/migrations/000310_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000309_prebuilds.up.sql rename to coderd/database/migrations/000310_prebuilds.up.sql diff --git a/coderd/database/migrations/000310_preset_prebuilds.down.sql b/coderd/database/migrations/000311_preset_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000310_preset_prebuilds.down.sql rename to coderd/database/migrations/000311_preset_prebuilds.down.sql diff --git a/coderd/database/migrations/000310_preset_prebuilds.up.sql b/coderd/database/migrations/000311_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000310_preset_prebuilds.up.sql rename to coderd/database/migrations/000311_preset_prebuilds.up.sql From 7b9c8cec6ed4c1df5a1edaecd89637e48260bcd7 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 09:59:54 -0400 Subject: [PATCH 30/72] test: remove deprecated test --- coderd/users_test.go | 96 -------------------------------------------- 1 file changed, 96 deletions(-) diff --git a/coderd/users_test.go b/coderd/users_test.go index 10febb480e5b9..c21eca85a5ee7 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "net/http" - "runtime" "slices" "strings" "testing" @@ -15,10 +14,8 @@ import ( "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" - "github.com/coder/coder/v2/coderd/database/migrations" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/notifications/notificationstest" - "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/golang-jwt/jwt/v4" @@ -2446,96 +2443,3 @@ func BenchmarkUsersMe(b *testing.B) { require.NoError(b, err) } } - -func TestSystemUserBehaviour(t *testing.T) { - // Setup. - t.Parallel() - - if runtime.GOOS != "linux" { - t.Skip("skipping because non-linux platforms are tricky to run the mock db container on, and there's no platform-dependence on these tests") - } - - ctx := testutil.Context(t, testutil.WaitLong) - - sqlDB := testSQLDB(t) - err := migrations.Up(sqlDB) // coderd/database/migrations/00030*_system_user.up.sql will create a system user. - require.NoError(t, err, "migrations") - - db := database.New(sqlDB) - - // ================================================================================================================= - - // When: retrieving users with the include_system flag enabled. - other := dbgen.User(t, db, database.User{}) - users, err := db.GetUsers(ctx, database.GetUsersParams{ - IncludeSystem: true, - }) - - // Then: system users are returned, alongside other users. - require.NoError(t, err) - require.Len(t, users, 2) - - var systemUser, regularUser database.GetUsersRow - for _, u := range users { - if u.IsSystem { - systemUser = u - } else { - regularUser = u - } - } - require.NotNil(t, systemUser) - require.NotNil(t, regularUser) - - require.True(t, systemUser.IsSystem) - require.Equal(t, systemUser.ID, prebuilds.SystemUserID) - require.False(t, regularUser.IsSystem) - require.Equal(t, regularUser.ID, other.ID) - - // ================================================================================================================= - - // When: retrieving users with the include_system flag disabled. - users, err = db.GetUsers(ctx, database.GetUsersParams{ - IncludeSystem: false, - }) - - // Then: only regular users are returned. - require.NoError(t, err) - require.Len(t, users, 1) - require.False(t, users[0].IsSystem) - - // ================================================================================================================= - - // When: attempting to update a system user's name. - _, err = db.UpdateUserProfile(ctx, database.UpdateUserProfileParams{ - ID: systemUser.ID, - Name: "not prebuilds", - }) - // Then: the attempt is rejected by a postgres trigger. - require.ErrorContains(t, err, "Cannot modify or delete system users") - - // When: attempting to delete a system user. - err = db.UpdateUserDeletedByID(ctx, systemUser.ID) - // Then: the attempt is rejected by a postgres trigger. - require.ErrorContains(t, err, "Cannot modify or delete system users") - - // When: attempting to update a user's roles. - _, err = db.UpdateUserRoles(ctx, database.UpdateUserRolesParams{ - ID: systemUser.ID, - GrantedRoles: []string{rbac.RoleAuditor().String()}, - }) - // Then: the attempt is rejected by a postgres trigger. - require.ErrorContains(t, err, "Cannot modify or delete system users") -} - -func testSQLDB(t testing.TB) *sql.DB { - t.Helper() - - connection, err := dbtestutil.Open(t) - require.NoError(t, err) - - db, err := sql.Open("postgres", connection) - require.NoError(t, err) - t.Cleanup(func() { _ = db.Close() }) - - return db -} From e189a0bfb6e4dff3502e97d75c96c1dfd8a322a8 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 10:05:58 -0400 Subject: [PATCH 31/72] fix: fix linter --- coderd/database/dbauthz/dbauthz_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 93a2925375948..738ef9d20a23c 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1715,7 +1715,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1746,7 +1746,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1799,7 +1799,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3810,7 +3810,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -3987,7 +3987,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -4032,14 +4032,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -4047,7 +4047,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4058,21 +4058,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) From 692c0e52e2fc3d3257778384aaec88342be65b72 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 10:35:28 -0400 Subject: [PATCH 32/72] fix: fix 000310_prebuilds.down migration --- coderd/database/migrations/000310_prebuilds.down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/database/migrations/000310_prebuilds.down.sql b/coderd/database/migrations/000310_prebuilds.down.sql index 251cb657a0d0d..bc8bc52e92da0 100644 --- a/coderd/database/migrations/000310_prebuilds.down.sql +++ b/coderd/database/migrations/000310_prebuilds.down.sql @@ -1,4 +1,4 @@ -- Revert prebuild views DROP VIEW IF EXISTS workspace_prebuild_builds; DROP VIEW IF EXISTS workspace_prebuilds; -DROP VIEW IF EXISTS workspace_latest_build; +DROP VIEW IF EXISTS workspace_latest_builds; From f747db09f1f420bb1857ac8dbac5eb332dc73e15 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 11:19:52 -0400 Subject: [PATCH 33/72] fix: fix fixture migration --- .../testdata/fixtures/000303_preset_prebuilds.up.sql | 2 -- .../testdata/fixtures/000311_preset_prebuilds.up.sql | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql create mode 100644 coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql deleted file mode 100644 index 1bceed871dbdc..0000000000000 --- a/coderd/database/migrations/testdata/fixtures/000303_preset_prebuilds.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO template_version_preset_prebuilds (id, preset_id, desired_instances) -VALUES (gen_random_uuid(), '28b42cc0-c4fe-4907-a0fe-e4d20f1e9bfe', 1); diff --git a/coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql new file mode 100644 index 0000000000000..c1f284b3e43c9 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql @@ -0,0 +1,3 @@ +UPDATE template_version_presets +SET desired_instances = 1 +WHERE id = '28b42cc0-c4fe-4907-a0fe-e4d20f1e9bfe'; From 3166a42091bf3db0290ce2c8c714b435f7b860ff Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 12:03:26 -0400 Subject: [PATCH 34/72] fix: fix get-presets-backoff test --- coderd/database/querier_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 9c1650c9a5ed4..b661d76f5b7db 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3587,6 +3587,10 @@ func TestOrganizationDeleteTrigger(t *testing.T) { func TestGetPresetsBackoff(t *testing.T) { t.Parallel() + if !dbtestutil.WillUsePostgres() { + t.SkipNow() + } + type extTmplVersion struct { database.TemplateVersion preset database.TemplateVersionPreset From aa6b490051b2db408b1f6118f843ac5ff6f84b9a Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 12:54:59 -0400 Subject: [PATCH 35/72] fix: fix linter --- coderd/database/dbauthz/dbauthz_test.go | 2 +- coderd/database/querier_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 738ef9d20a23c..1596663aa5342 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4751,7 +4751,7 @@ func (s *MethodTestSuite) TestPrebuilds() { ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) // TODO: remove? - //s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { + // s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { // org := dbgen.Organization(s.T(), db, database.Organization{}) // user := dbgen.User(s.T(), db, database.User{}) // template := dbgen.Template(s.T(), db, database.Template{ diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index b661d76f5b7db..ba14d648b2b3d 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3697,7 +3697,7 @@ func TestGetPresetsBackoff(t *testing.T) { }, }) } - findBackoffByTmplVersionId := func(backoffs []database.GetPresetsBackoffRow, tmplVersionID uuid.UUID) *database.GetPresetsBackoffRow { + findBackoffByTmplVersionID := func(backoffs []database.GetPresetsBackoffRow, tmplVersionID uuid.UUID) *database.GetPresetsBackoffRow { for _, backoff := range backoffs { if backoff.TemplateVersionID == tmplVersionID { return &backoff @@ -3817,13 +3817,13 @@ func TestGetPresetsBackoff(t *testing.T) { require.Len(t, backoffs, 2) { - backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) + backoff := findBackoffByTmplVersionID(backoffs, tmpl1.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) require.Equal(t, int32(1), backoff.NumFailed) } { - backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) + backoff := findBackoffByTmplVersionID(backoffs, tmpl2.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) require.Equal(t, int32(1), backoff.NumFailed) @@ -3865,19 +3865,19 @@ func TestGetPresetsBackoff(t *testing.T) { require.Len(t, backoffs, 3) { - backoff := findBackoffByTmplVersionId(backoffs, tmpl1.ActiveVersionID) + backoff := findBackoffByTmplVersionID(backoffs, tmpl1.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl1.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) require.Equal(t, int32(1), backoff.NumFailed) } { - backoff := findBackoffByTmplVersionId(backoffs, tmpl2.ActiveVersionID) + backoff := findBackoffByTmplVersionID(backoffs, tmpl2.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl2.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl2V1.preset.ID) require.Equal(t, int32(2), backoff.NumFailed) } { - backoff := findBackoffByTmplVersionId(backoffs, tmpl3.ActiveVersionID) + backoff := findBackoffByTmplVersionID(backoffs, tmpl3.ActiveVersionID) require.Equal(t, backoff.TemplateVersionID, tmpl3.ActiveVersionID) require.Equal(t, backoff.PresetID, tmpl3V2.preset.ID) require.Equal(t, int32(3), backoff.NumFailed) From bc4e7d2dbd99c9ea3ff6b9be01914d2e21d5df92 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 25 Mar 2025 13:02:22 -0400 Subject: [PATCH 36/72] fix: fix linter --- coderd/database/dbauthz/dbauthz_test.go | 2 +- coderd/provisionerdserver/provisionerdserver.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 1596663aa5342..b5ef5a12e4c24 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4774,7 +4774,7 @@ func (s *MethodTestSuite) TestPrebuilds() { // }). // Asserts(rbac.ResourceSystem, policy.ActionCreate). // ErrorsWithInMemDB(dbmem.ErrUnimplemented) - //})) + // })) } func (s *MethodTestSuite) TestOAuth2ProviderApps() { diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 05cadb5875e5a..5d6a88197cd6d 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -1856,9 +1856,11 @@ func InsertWorkspacePresetsAndParameters(ctx context.Context, logger slog.Logger func InsertWorkspacePresetAndParameters(ctx context.Context, db database.Store, templateVersionID uuid.UUID, protoPreset *sdkproto.Preset, t time.Time) error { err := db.InTx(func(tx database.Store) error { dbPreset, err := tx.InsertPreset(ctx, database.InsertPresetParams{ - TemplateVersionID: templateVersionID, - Name: protoPreset.Name, - CreatedAt: t, + TemplateVersionID: templateVersionID, + Name: protoPreset.Name, + CreatedAt: t, + DesiredInstances: sql.NullInt32{}, + InvalidateAfterSecs: sql.NullInt32{}, }) if err != nil { return xerrors.Errorf("insert preset: %w", err) From f167b92189661e412c718ef9e24219b2096be69d Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Wed, 26 Mar 2025 08:40:28 +0000 Subject: [PATCH 37/72] correctly select for the latest built with a preset in latest_prebuild_builds --- coderd/database/dump.sql | 70 +++---------------- .../migrations/000310_prebuilds.up.sql | 35 ++++++---- coderd/database/models.go | 28 +++----- 3 files changed, 40 insertions(+), 93 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c64f642e138d8..ed6652f16f207 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1997,21 +1997,9 @@ CREATE VIEW workspace_prebuild_builds AS CREATE VIEW workspace_prebuilds AS SELECT NULL::uuid AS id, - NULL::timestamp with time zone AS created_at, - NULL::timestamp with time zone AS updated_at, - NULL::uuid AS owner_id, - NULL::uuid AS organization_id, - NULL::uuid AS template_id, - NULL::boolean AS deleted, NULL::character varying(64) AS name, - NULL::text AS autostart_schedule, - NULL::bigint AS ttl, - NULL::timestamp with time zone AS last_used_at, - NULL::timestamp with time zone AS dormant_at, - NULL::timestamp with time zone AS deleting_at, - NULL::automatic_updates AS automatic_updates, - NULL::boolean AS favorite, - NULL::timestamp with time zone AS next_start_at, + NULL::uuid AS template_id, + NULL::timestamp with time zone AS created_at, NULL::uuid AS agent_id, NULL::workspace_agent_lifecycle_state AS lifecycle_state, NULL::timestamp with time zone AS ready_at, @@ -2602,41 +2590,17 @@ CREATE OR REPLACE VIEW provisioner_job_stats AS CREATE OR REPLACE VIEW workspace_prebuilds AS WITH all_prebuilds AS ( SELECT w.id, - w.created_at, - w.updated_at, - w.owner_id, - w.organization_id, - w.template_id, - w.deleted, w.name, - w.autostart_schedule, - w.ttl, - w.last_used_at, - w.dormant_at, - w.deleting_at, - w.automatic_updates, - w.favorite, - w.next_start_at + w.template_id, + w.created_at FROM workspaces w WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) ), latest_prebuild_builds AS ( - SELECT workspace_latest_builds.id, - workspace_latest_builds.created_at, - workspace_latest_builds.updated_at, - workspace_latest_builds.workspace_id, - workspace_latest_builds.template_version_id, - workspace_latest_builds.build_number, - workspace_latest_builds.transition, - workspace_latest_builds.initiator_id, - workspace_latest_builds.provisioner_state, - workspace_latest_builds.job_id, - workspace_latest_builds.deadline, - workspace_latest_builds.reason, - workspace_latest_builds.daily_cost, - workspace_latest_builds.max_deadline, - workspace_latest_builds.template_version_preset_id - FROM workspace_latest_builds - WHERE (workspace_latest_builds.template_version_preset_id IS NOT NULL) + SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.workspace_id, + workspace_builds.template_version_preset_id + FROM workspace_builds + WHERE (workspace_builds.template_version_preset_id IS NOT NULL) + ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC ), workspace_agents AS ( SELECT w.id AS workspace_id, wa.id AS agent_id, @@ -2656,21 +2620,9 @@ CREATE OR REPLACE VIEW workspace_prebuilds AS WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) ) SELECT p.id, - p.created_at, - p.updated_at, - p.owner_id, - p.organization_id, - p.template_id, - p.deleted, p.name, - p.autostart_schedule, - p.ttl, - p.last_used_at, - p.dormant_at, - p.deleting_at, - p.automatic_updates, - p.favorite, - p.next_start_at, + p.template_id, + p.created_at, a.agent_id, a.lifecycle_state, a.ready_at, diff --git a/coderd/database/migrations/000310_prebuilds.up.sql b/coderd/database/migrations/000310_prebuilds.up.sql index f9578a28cc7cb..e4ebd53f2a879 100644 --- a/coderd/database/migrations/000310_prebuilds.up.sql +++ b/coderd/database/migrations/000310_prebuilds.up.sql @@ -9,30 +9,37 @@ ORDER BY workspace_id, build_number DESC; CREATE VIEW workspace_prebuilds AS WITH -- All workspaces owned by the "prebuilds" user. - all_prebuilds AS (SELECT w.* - FROM workspaces w - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'), -- The system user responsible for prebuilds. + all_prebuilds AS ( + SELECT w.id, w.name, w.template_id, w.created_at + FROM workspaces w + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. + ), -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. -- -- See https://github.com/coder/internal/issues/398 - latest_prebuild_builds AS (SELECT * - FROM workspace_latest_builds - WHERE template_version_preset_id IS NOT NULL), + latest_prebuild_builds AS ( + SELECT DISTINCT ON (workspace_id) workspace_id, template_version_preset_id + FROM workspace_builds + WHERE template_version_preset_id IS NOT NULL + ORDER BY workspace_id, build_number DESC + ), -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. - workspace_agents AS (SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at - FROM workspaces w - INNER JOIN workspace_latest_builds wlb ON wlb.workspace_id = w.id - INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id - INNER JOIN workspace_agents wa ON wa.resource_id = wr.id - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. - GROUP BY w.id, wa.id), + workspace_agents AS ( + SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at + FROM workspaces w + INNER JOIN workspace_latest_builds wlb ON wlb.workspace_id = w.id + INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id + INNER JOIN workspace_agents wa ON wa.resource_id = wr.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. + GROUP BY w.id, wa.id + ), current_presets AS (SELECT w.id AS prebuild_id, lpb.template_version_preset_id FROM workspaces w INNER JOIN latest_prebuild_builds lpb ON lpb.workspace_id = w.id WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. -SELECT p.*, a.agent_id, a.lifecycle_state, a.ready_at, cp.template_version_preset_id AS current_preset_id +SELECT p.id, p.name, p.template_id, p.created_at, a.agent_id, a.lifecycle_state, a.ready_at, cp.template_version_preset_id AS current_preset_id FROM all_prebuilds p LEFT JOIN workspace_agents a ON a.workspace_id = p.id INNER JOIN current_presets cp ON cp.prebuild_id = p.id; diff --git a/coderd/database/models.go b/coderd/database/models.go index d4411d2f31579..5f4a355d9db7e 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3584,26 +3584,14 @@ type WorkspaceModule struct { } type WorkspacePrebuild struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - OwnerID uuid.UUID `db:"owner_id" json:"owner_id"` - OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - Deleted bool `db:"deleted" json:"deleted"` - Name string `db:"name" json:"name"` - AutostartSchedule sql.NullString `db:"autostart_schedule" json:"autostart_schedule"` - Ttl sql.NullInt64 `db:"ttl" json:"ttl"` - LastUsedAt time.Time `db:"last_used_at" json:"last_used_at"` - DormantAt sql.NullTime `db:"dormant_at" json:"dormant_at"` - DeletingAt sql.NullTime `db:"deleting_at" json:"deleting_at"` - AutomaticUpdates AutomaticUpdates `db:"automatic_updates" json:"automatic_updates"` - Favorite bool `db:"favorite" json:"favorite"` - NextStartAt sql.NullTime `db:"next_start_at" json:"next_start_at"` - AgentID uuid.NullUUID `db:"agent_id" json:"agent_id"` - LifecycleState NullWorkspaceAgentLifecycleState `db:"lifecycle_state" json:"lifecycle_state"` - ReadyAt sql.NullTime `db:"ready_at" json:"ready_at"` - CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` + ID uuid.UUID `db:"id" json:"id"` + Name string `db:"name" json:"name"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + AgentID uuid.NullUUID `db:"agent_id" json:"agent_id"` + LifecycleState NullWorkspaceAgentLifecycleState `db:"lifecycle_state" json:"lifecycle_state"` + ReadyAt sql.NullTime `db:"ready_at" json:"ready_at"` + CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` } type WorkspacePrebuildBuild struct { From 7a8ec4973388618a227e4487bca0bfb8d9c5b0ff Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Wed, 26 Mar 2025 21:11:49 +0000 Subject: [PATCH 38/72] Properly label and filter metrics for prebuilds --- coderd/database/dump.sql | 161 ++++++++---------- ...lds.down.sql => 000312_prebuilds.down.sql} | 0 ...ebuilds.up.sql => 000312_prebuilds.up.sql} | 21 ++- ...n.sql => 000313_preset_prebuilds.down.sql} | 0 ....up.sql => 000313_preset_prebuilds.up.sql} | 0 ....up.sql => 000313_preset_prebuilds.up.sql} | 0 coderd/database/models.go | 14 +- coderd/database/queries.sql.go | 61 ++++--- coderd/database/queries/prebuilds.sql | 39 +++-- 9 files changed, 146 insertions(+), 150 deletions(-) rename coderd/database/migrations/{000310_prebuilds.down.sql => 000312_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000310_prebuilds.up.sql => 000312_prebuilds.up.sql} (72%) rename coderd/database/migrations/{000311_preset_prebuilds.down.sql => 000313_preset_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000311_preset_prebuilds.up.sql => 000313_preset_prebuilds.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000311_preset_prebuilds.up.sql => 000313_preset_prebuilds.up.sql} (100%) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index b1f28426bfd08..7f5fa103f8468 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -2013,16 +2013,80 @@ CREATE VIEW workspace_prebuild_builds AS FROM workspace_builds WHERE (workspace_builds.initiator_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid); +CREATE TABLE workspace_resources ( + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + job_id uuid NOT NULL, + transition workspace_transition NOT NULL, + type character varying(192) NOT NULL, + name character varying(64) NOT NULL, + hide boolean DEFAULT false NOT NULL, + icon character varying(256) DEFAULT ''::character varying NOT NULL, + instance_type character varying(256), + daily_cost integer DEFAULT 0 NOT NULL, + module_path text +); + +CREATE TABLE workspaces ( + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + owner_id uuid NOT NULL, + organization_id uuid NOT NULL, + template_id uuid NOT NULL, + deleted boolean DEFAULT false NOT NULL, + name character varying(64) NOT NULL, + autostart_schedule text, + ttl bigint, + last_used_at timestamp with time zone DEFAULT '0001-01-01 00:00:00+00'::timestamp with time zone NOT NULL, + dormant_at timestamp with time zone, + deleting_at timestamp with time zone, + automatic_updates automatic_updates DEFAULT 'never'::automatic_updates NOT NULL, + favorite boolean DEFAULT false NOT NULL, + next_start_at timestamp with time zone +); + +COMMENT ON COLUMN workspaces.favorite IS 'Favorite is true if the workspace owner has favorited the workspace.'; + CREATE VIEW workspace_prebuilds AS -SELECT - NULL::uuid AS id, - NULL::character varying(64) AS name, - NULL::uuid AS template_id, - NULL::timestamp with time zone AS created_at, - NULL::uuid AS agent_id, - NULL::workspace_agent_lifecycle_state AS lifecycle_state, - NULL::timestamp with time zone AS ready_at, - NULL::uuid AS current_preset_id; + WITH all_prebuilds AS ( + SELECT w.id, + w.name, + w.template_id, + w.created_at + FROM workspaces w + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + ), workspaces_with_latest_presets AS ( + SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.workspace_id, + workspace_builds.template_version_preset_id + FROM workspace_builds + WHERE (workspace_builds.template_version_preset_id IS NOT NULL) + ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC + ), workspaces_with_agents_status AS ( + SELECT w.id AS workspace_id, + bool_and((wa.lifecycle_state = 'ready'::workspace_agent_lifecycle_state)) AS ready + FROM (((workspaces w + JOIN workspace_latest_builds wlb ON ((wlb.workspace_id = w.id))) + JOIN workspace_resources wr ON ((wr.job_id = wlb.job_id))) + JOIN workspace_agents wa ON ((wa.resource_id = wr.id))) + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + GROUP BY w.id + ), current_presets AS ( + SELECT w.id AS prebuild_id, + wlp.template_version_preset_id + FROM (workspaces w + JOIN workspaces_with_latest_presets wlp ON ((wlp.workspace_id = w.id))) + WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) + ) + SELECT p.id, + p.name, + p.template_id, + p.created_at, + COALESCE(a.ready, false) AS ready, + cp.template_version_preset_id AS current_preset_id + FROM ((all_prebuilds p + LEFT JOIN workspaces_with_agents_status a ON ((a.workspace_id = p.id))) + JOIN current_presets cp ON ((cp.prebuild_id = p.id))); CREATE TABLE workspace_proxies ( id uuid NOT NULL, @@ -2080,41 +2144,6 @@ CREATE SEQUENCE workspace_resource_metadata_id_seq ALTER SEQUENCE workspace_resource_metadata_id_seq OWNED BY workspace_resource_metadata.id; -CREATE TABLE workspace_resources ( - id uuid NOT NULL, - created_at timestamp with time zone NOT NULL, - job_id uuid NOT NULL, - transition workspace_transition NOT NULL, - type character varying(192) NOT NULL, - name character varying(64) NOT NULL, - hide boolean DEFAULT false NOT NULL, - icon character varying(256) DEFAULT ''::character varying NOT NULL, - instance_type character varying(256), - daily_cost integer DEFAULT 0 NOT NULL, - module_path text -); - -CREATE TABLE workspaces ( - id uuid NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - owner_id uuid NOT NULL, - organization_id uuid NOT NULL, - template_id uuid NOT NULL, - deleted boolean DEFAULT false NOT NULL, - name character varying(64) NOT NULL, - autostart_schedule text, - ttl bigint, - last_used_at timestamp with time zone DEFAULT '0001-01-01 00:00:00+00'::timestamp with time zone NOT NULL, - dormant_at timestamp with time zone, - deleting_at timestamp with time zone, - automatic_updates automatic_updates DEFAULT 'never'::automatic_updates NOT NULL, - favorite boolean DEFAULT false NOT NULL, - next_start_at timestamp with time zone -); - -COMMENT ON COLUMN workspaces.favorite IS 'Favorite is true if the workspace owner has favorited the workspace.'; - CREATE VIEW workspaces_expanded AS SELECT workspaces.id, workspaces.created_at, @@ -2606,50 +2635,6 @@ CREATE OR REPLACE VIEW provisioner_job_stats AS LEFT JOIN provisioner_job_timings pjt ON ((pjt.job_id = pj.id))) GROUP BY pj.id, wb.workspace_id; -CREATE OR REPLACE VIEW workspace_prebuilds AS - WITH all_prebuilds AS ( - SELECT w.id, - w.name, - w.template_id, - w.created_at - FROM workspaces w - WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) - ), latest_prebuild_builds AS ( - SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.workspace_id, - workspace_builds.template_version_preset_id - FROM workspace_builds - WHERE (workspace_builds.template_version_preset_id IS NOT NULL) - ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC - ), workspace_agents AS ( - SELECT w.id AS workspace_id, - wa.id AS agent_id, - wa.lifecycle_state, - wa.ready_at - FROM (((workspaces w - JOIN workspace_latest_builds wlb ON ((wlb.workspace_id = w.id))) - JOIN workspace_resources wr ON ((wr.job_id = wlb.job_id))) - JOIN workspace_agents wa ON ((wa.resource_id = wr.id))) - WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) - GROUP BY w.id, wa.id - ), current_presets AS ( - SELECT w.id AS prebuild_id, - lpb.template_version_preset_id - FROM (workspaces w - JOIN latest_prebuild_builds lpb ON ((lpb.workspace_id = w.id))) - WHERE (w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid) - ) - SELECT p.id, - p.name, - p.template_id, - p.created_at, - a.agent_id, - a.lifecycle_state, - a.ready_at, - cp.template_version_preset_id AS current_preset_id - FROM ((all_prebuilds p - LEFT JOIN workspace_agents a ON ((a.workspace_id = p.id))) - JOIN current_presets cp ON ((cp.prebuild_id = p.id))); - CREATE TRIGGER inhibit_enqueue_if_disabled BEFORE INSERT ON notification_messages FOR EACH ROW EXECUTE FUNCTION inhibit_enqueue_if_disabled(); CREATE TRIGGER protect_deleting_organizations BEFORE UPDATE ON organizations FOR EACH ROW WHEN (((new.deleted = true) AND (old.deleted = false))) EXECUTE FUNCTION protect_deleting_organizations(); diff --git a/coderd/database/migrations/000310_prebuilds.down.sql b/coderd/database/migrations/000312_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000310_prebuilds.down.sql rename to coderd/database/migrations/000312_prebuilds.down.sql diff --git a/coderd/database/migrations/000310_prebuilds.up.sql b/coderd/database/migrations/000312_prebuilds.up.sql similarity index 72% rename from coderd/database/migrations/000310_prebuilds.up.sql rename to coderd/database/migrations/000312_prebuilds.up.sql index e4ebd53f2a879..57be23d4af34d 100644 --- a/coderd/database/migrations/000310_prebuilds.up.sql +++ b/coderd/database/migrations/000312_prebuilds.up.sql @@ -19,29 +19,32 @@ WITH -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. -- -- See https://github.com/coder/internal/issues/398 - latest_prebuild_builds AS ( + workspaces_with_latest_presets AS ( SELECT DISTINCT ON (workspace_id) workspace_id, template_version_preset_id FROM workspace_builds WHERE template_version_preset_id IS NOT NULL ORDER BY workspace_id, build_number DESC ), - -- All workspace agents belonging to the workspaces owned by the "prebuilds" user. - workspace_agents AS ( - SELECT w.id AS workspace_id, wa.id AS agent_id, wa.lifecycle_state, wa.ready_at + -- workspaces_with_agents_status contains workspaces owned by the "prebuilds" user, + -- along with the readiness status of their agents. + -- A workspace is marked as 'ready' only if ALL of its agents are ready. + workspaces_with_agents_status AS ( + SELECT w.id AS workspace_id, + BOOL_AND(wa.lifecycle_state = 'ready'::workspace_agent_lifecycle_state) AS ready FROM workspaces w INNER JOIN workspace_latest_builds wlb ON wlb.workspace_id = w.id INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id INNER JOIN workspace_agents wa ON wa.resource_id = wr.id WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. - GROUP BY w.id, wa.id + GROUP BY w.id ), - current_presets AS (SELECT w.id AS prebuild_id, lpb.template_version_preset_id + current_presets AS (SELECT w.id AS prebuild_id, wlp.template_version_preset_id FROM workspaces w - INNER JOIN latest_prebuild_builds lpb ON lpb.workspace_id = w.id + INNER JOIN workspaces_with_latest_presets wlp ON wlp.workspace_id = w.id WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. -SELECT p.id, p.name, p.template_id, p.created_at, a.agent_id, a.lifecycle_state, a.ready_at, cp.template_version_preset_id AS current_preset_id +SELECT p.id, p.name, p.template_id, p.created_at, COALESCE(a.ready, false) AS ready, cp.template_version_preset_id AS current_preset_id FROM all_prebuilds p - LEFT JOIN workspace_agents a ON a.workspace_id = p.id + LEFT JOIN workspaces_with_agents_status a ON a.workspace_id = p.id INNER JOIN current_presets cp ON cp.prebuild_id = p.id; CREATE VIEW workspace_prebuild_builds AS diff --git a/coderd/database/migrations/000311_preset_prebuilds.down.sql b/coderd/database/migrations/000313_preset_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000311_preset_prebuilds.down.sql rename to coderd/database/migrations/000313_preset_prebuilds.down.sql diff --git a/coderd/database/migrations/000311_preset_prebuilds.up.sql b/coderd/database/migrations/000313_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000311_preset_prebuilds.up.sql rename to coderd/database/migrations/000313_preset_prebuilds.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000313_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000311_preset_prebuilds.up.sql rename to coderd/database/migrations/testdata/fixtures/000313_preset_prebuilds.up.sql diff --git a/coderd/database/models.go b/coderd/database/models.go index 5f4a355d9db7e..c30257195e620 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3584,14 +3584,12 @@ type WorkspaceModule struct { } type WorkspacePrebuild struct { - ID uuid.UUID `db:"id" json:"id"` - Name string `db:"name" json:"name"` - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - AgentID uuid.NullUUID `db:"agent_id" json:"agent_id"` - LifecycleState NullWorkspaceAgentLifecycleState `db:"lifecycle_state" json:"lifecycle_state"` - ReadyAt sql.NullTime `db:"ready_at" json:"ready_at"` - CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` + ID uuid.UUID `db:"id" json:"id"` + Name string `db:"name" json:"name"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + Ready bool `db:"ready" json:"ready"` + CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` } type WorkspacePrebuildBuild struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index acd0873b18394..423b7c1ef375a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5843,7 +5843,7 @@ WHERE w.id IN (SELECT p.id AND pj.job_status IN ('succeeded'::provisioner_job_status)) AND b.template_version_id = t.active_version_id AND b.template_version_preset_id = $3::uuid - AND p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state + AND p.ready ORDER BY random() LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. RETURNING w.id, w.name @@ -5871,6 +5871,7 @@ const getPrebuildMetrics = `-- name: GetPrebuildMetrics :many SELECT t.name as template_name, tvp.name as preset_name, + o.name as organization_name, COUNT(*) as created_count, COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, COUNT(*) FILTER ( @@ -5881,17 +5882,19 @@ INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id INNER JOIN templates t ON t.id = w.template_id INNER JOIN template_version_presets tvp ON tvp.id = wpb.template_version_preset_id INNER JOIN provisioner_jobs pj ON pj.id = wpb.job_id -WHERE wpb.build_number = 1 -GROUP BY t.name, tvp.name -ORDER BY t.name, tvp.name +INNER JOIN organizations o ON o.id = w.organization_id +WHERE NOT t.deleted AND wpb.build_number = 1 +GROUP BY t.name, tvp.name, o.name +ORDER BY t.name, tvp.name, o.name ` type GetPrebuildMetricsRow struct { - TemplateName string `db:"template_name" json:"template_name"` - PresetName string `db:"preset_name" json:"preset_name"` - CreatedCount int64 `db:"created_count" json:"created_count"` - FailedCount int64 `db:"failed_count" json:"failed_count"` - ClaimedCount int64 `db:"claimed_count" json:"claimed_count"` + TemplateName string `db:"template_name" json:"template_name"` + PresetName string `db:"preset_name" json:"preset_name"` + OrganizationName string `db:"organization_name" json:"organization_name"` + CreatedCount int64 `db:"created_count" json:"created_count"` + FailedCount int64 `db:"failed_count" json:"failed_count"` + ClaimedCount int64 `db:"claimed_count" json:"claimed_count"` } func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) { @@ -5906,6 +5909,7 @@ func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetri if err := rows.Scan( &i.TemplateName, &i.PresetName, + &i.OrganizationName, &i.CreatedCount, &i.FailedCount, &i.ClaimedCount, @@ -5997,7 +6001,7 @@ failed_count AS ( SELECT tsb.template_version_id, tsb.preset_id, COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at::timestamptz) AS last_build_at + MAX(tsb.created_at)::timestamptz AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff @@ -6007,10 +6011,10 @@ GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed ` type GetPresetsBackoffRow struct { - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - PresetID uuid.UUID `db:"preset_id" json:"preset_id"` - NumFailed int32 `db:"num_failed" json:"num_failed"` - LastBuildAt interface{} `db:"last_build_at" json:"last_build_at"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + PresetID uuid.UUID `db:"preset_id" json:"preset_id"` + NumFailed int32 `db:"num_failed" json:"num_failed"` + LastBuildAt time.Time `db:"last_build_at" json:"last_build_at"` } // GetPresetsBackoff groups workspace builds by template version ID. @@ -6060,9 +6064,7 @@ SELECT p.id AS workspace_id, p.template_id, b.template_version_id, p.current_preset_id AS current_preset_id, - CASE - WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE - ELSE FALSE END AS ready, + p.ready, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id @@ -6113,19 +6115,22 @@ func (q *sqlQuerier) GetRunningPrebuilds(ctx context.Context) ([]GetRunningPrebu } const getTemplatePresetsWithPrebuilds = `-- name: GetTemplatePresetsWithPrebuilds :many -SELECT t.id AS template_id, - t.name AS template_name, - tv.id AS template_version_id, - tv.name AS template_version_name, - tv.id = t.active_version_id AS using_active_version, - tvp.id, - tvp.name, - tvp.desired_instances AS desired_instances, - t.deleted, - t.deprecated != '' AS deprecated +SELECT + t.id AS template_id, + t.name AS template_name, + o.name AS organization_name, + tv.id AS template_version_id, + tv.name AS template_version_name, + tv.id = t.active_version_id AS using_active_version, + tvp.id, + tvp.name, + tvp.desired_instances AS desired_instances, + t.deleted, + t.deprecated != '' AS deprecated FROM templates t INNER JOIN template_versions tv ON tv.template_id = t.id INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN organizations o ON o.id = t.organization_id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND (t.id = $1::uuid OR $1 IS NULL) ` @@ -6133,6 +6138,7 @@ WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a pre type GetTemplatePresetsWithPrebuildsRow struct { TemplateID uuid.UUID `db:"template_id" json:"template_id"` TemplateName string `db:"template_name" json:"template_name"` + OrganizationName string `db:"organization_name" json:"organization_name"` TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` TemplateVersionName string `db:"template_version_name" json:"template_version_name"` UsingActiveVersion bool `db:"using_active_version" json:"using_active_version"` @@ -6158,6 +6164,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa if err := rows.Scan( &i.TemplateID, &i.TemplateName, + &i.OrganizationName, &i.TemplateVersionID, &i.TemplateVersionName, &i.UsingActiveVersion, diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index b9e29689e82ad..ca7c7e7db07a4 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -2,19 +2,22 @@ -- GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. -- It also returns the number of desired instances for each preset. -- If template_id is specified, only template versions associated with that template will be returned. -SELECT t.id AS template_id, - t.name AS template_name, - tv.id AS template_version_id, - tv.name AS template_version_name, - tv.id = t.active_version_id AS using_active_version, - tvp.id, - tvp.name, - tvp.desired_instances AS desired_instances, - t.deleted, - t.deprecated != '' AS deprecated +SELECT + t.id AS template_id, + t.name AS template_name, + o.name AS organization_name, + tv.id AS template_version_id, + tv.name AS template_version_name, + tv.id = t.active_version_id AS using_active_version, + tvp.id, + tvp.name, + tvp.desired_instances AS desired_instances, + t.deleted, + t.deprecated != '' AS deprecated FROM templates t INNER JOIN template_versions tv ON tv.template_id = t.id INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN organizations o ON o.id = t.organization_id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); @@ -24,9 +27,7 @@ SELECT p.id AS workspace_id, p.template_id, b.template_version_id, p.current_preset_id AS current_preset_id, - CASE - WHEN p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state THEN TRUE - ELSE FALSE END AS ready, + p.ready, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id @@ -85,7 +86,7 @@ failed_count AS ( SELECT tsb.template_version_id, tsb.preset_id, COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at::timestamptz) AS last_build_at + MAX(tsb.created_at)::timestamptz AS last_build_at FROM time_sorted_builds tsb LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff @@ -107,7 +108,7 @@ WHERE w.id IN (SELECT p.id AND pj.job_status IN ('succeeded'::provisioner_job_status)) AND b.template_version_id = t.active_version_id AND b.template_version_preset_id = @preset_id::uuid - AND p.lifecycle_state = 'ready'::workspace_agent_lifecycle_state + AND p.ready ORDER BY random() LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. RETURNING w.id, w.name; @@ -116,6 +117,7 @@ RETURNING w.id, w.name; SELECT t.name as template_name, tvp.name as preset_name, + o.name as organization_name, COUNT(*) as created_count, COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, COUNT(*) FILTER ( @@ -126,6 +128,7 @@ INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id INNER JOIN templates t ON t.id = w.template_id INNER JOIN template_version_presets tvp ON tvp.id = wpb.template_version_preset_id INNER JOIN provisioner_jobs pj ON pj.id = wpb.job_id -WHERE wpb.build_number = 1 -GROUP BY t.name, tvp.name -ORDER BY t.name, tvp.name; +INNER JOIN organizations o ON o.id = w.organization_id +WHERE NOT t.deleted AND wpb.build_number = 1 +GROUP BY t.name, tvp.name, o.name +ORDER BY t.name, tvp.name, o.name; From a64d661cfd2a2974af68e4f02446fa664d93c2bb Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 26 Mar 2025 20:04:21 -0400 Subject: [PATCH 39/72] test: fix db tests --- coderd/database/querier_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 107878bd9119b..10553b9a34e19 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -4147,7 +4147,7 @@ func TestGetPresetsBackoff(t *testing.T) { require.Equal(t, backoff.PresetID, tmpl1V1.preset.ID) require.Equal(t, int32(5), backoff.NumFailed) // make sure LastBuildAt is equal to latest failed build timestamp - require.Equal(t, 0, now.Compare(backoff.LastBuildAt.(time.Time))) + require.Equal(t, 0, now.Compare(backoff.LastBuildAt)) } }) From c787cd29bb49d6812558e614998a3a441e7045e7 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 26 Mar 2025 21:40:39 -0400 Subject: [PATCH 40/72] test: added tests for workspaces with multiple agents --- coderd/database/querier_test.go | 242 ++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 10553b9a34e19..1cdd5bbbdeb9d 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3588,6 +3588,248 @@ func TestOrganizationDeleteTrigger(t *testing.T) { }) } +func TestWorkspacePrebuildsView(t *testing.T) { + t.Parallel() + if !dbtestutil.WillUsePostgres() { + t.SkipNow() + } + + type extTmplVersion struct { + database.TemplateVersion + preset database.TemplateVersionPreset + } + + now := dbtime.Now() + orgID := uuid.New() + userID := uuid.New() + + createTemplate := func(db database.Store) database.Template { + // create template + tmpl := dbgen.Template(t, db, database.Template{ + OrganizationID: orgID, + CreatedBy: userID, + ActiveVersionID: uuid.New(), + }) + + return tmpl + } + type tmplVersionOpts struct { + DesiredInstances int + } + createTmplVersion := func(db database.Store, tmpl database.Template, versionId uuid.UUID, opts *tmplVersionOpts) extTmplVersion { + // Create template version with corresponding preset and preset prebuild + tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + ID: versionId, + TemplateID: uuid.NullUUID{ + UUID: tmpl.ID, + Valid: true, + }, + OrganizationID: tmpl.OrganizationID, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: tmpl.CreatedBy, + }) + desiredInstances := 1 + if opts != nil { + desiredInstances = opts.DesiredInstances + } + preset := dbgen.Preset(t, db, database.InsertPresetParams{ + TemplateVersionID: tmplVersion.ID, + Name: "preset", + DesiredInstances: sql.NullInt32{ + Int32: int32(desiredInstances), + Valid: true, + }, + }) + + return extTmplVersion{ + TemplateVersion: tmplVersion, + preset: preset, + } + } + type workspaceBuildOpts struct { + successfulJob bool + createdAt time.Time + readyAgents int + notReadyAgents int + } + createWorkspaceBuild := func( + ctx context.Context, + db database.Store, + tmpl database.Template, + extTmplVersion extTmplVersion, + opts *workspaceBuildOpts, + ) { + // Create job with corresponding resource and agent + jobError := sql.NullString{String: "failed", Valid: true} + if opts != nil && opts.successfulJob { + jobError = sql.NullString{} + } + job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + Type: database.ProvisionerJobTypeWorkspaceBuild, + OrganizationID: orgID, + + CreatedAt: now.Add(-1 * time.Minute), + Error: jobError, + }) + + // create ready agents + readyAgents := 0 + if opts != nil { + readyAgents = opts.readyAgents + } + for i := 0; i < readyAgents; i++ { + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ + JobID: job.ID, + }) + agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ + ResourceID: resource.ID, + }) + err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ + ID: agent.ID, + LifecycleState: database.WorkspaceAgentLifecycleStateReady, + }) + require.NoError(t, err) + } + + // create not ready agents + notReadyAgents := 1 + if opts != nil { + notReadyAgents = opts.notReadyAgents + } + for i := 0; i < notReadyAgents; i++ { + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ + JobID: job.ID, + }) + agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ + ResourceID: resource.ID, + }) + err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ + ID: agent.ID, + LifecycleState: database.WorkspaceAgentLifecycleStateCreated, + }) + require.NoError(t, err) + } + + // Create corresponding workspace and workspace build + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0"), + OrganizationID: tmpl.OrganizationID, + TemplateID: tmpl.ID, + }) + createdAt := now + if opts != nil { + createdAt = opts.createdAt + } + dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ + CreatedAt: createdAt, + WorkspaceID: workspace.ID, + TemplateVersionID: extTmplVersion.ID, + BuildNumber: 1, + Transition: database.WorkspaceTransitionStart, + InitiatorID: tmpl.CreatedBy, + JobID: job.ID, + TemplateVersionPresetID: uuid.NullUUID{ + UUID: extTmplVersion.preset.ID, + Valid: true, + }, + }) + } + + type workspacePrebuild struct { + ID uuid.UUID + Name string + CreatedAt time.Time + Ready bool + CurrentPresetID uuid.UUID + } + getWorkspacePrebuilds := func(sqlDB *sql.DB) []*workspacePrebuild { + rows, err := sqlDB.Query("SELECT id, name, created_at, ready, current_preset_id FROM workspace_prebuilds") + require.NoError(t, err) + defer rows.Close() + + workspacePrebuilds := make([]*workspacePrebuild, 0) + for rows.Next() { + var wp workspacePrebuild + err := rows.Scan(&wp.ID, &wp.Name, &wp.CreatedAt, &wp.Ready, &wp.CurrentPresetID) + require.NoError(t, err) + + workspacePrebuilds = append(workspacePrebuilds, &wp) + } + + return workspacePrebuilds + } + + testCases := []struct { + name string + readyAgents int + notReadyAgents int + expectReady bool + }{ + { + name: "one ready agent", + readyAgents: 1, + notReadyAgents: 0, + expectReady: true, + }, + { + name: "one not ready agent", + readyAgents: 0, + notReadyAgents: 1, + expectReady: false, + }, + { + name: "one ready, one not ready", + readyAgents: 1, + notReadyAgents: 1, + expectReady: false, + }, + { + name: "both ready", + readyAgents: 2, + notReadyAgents: 0, + expectReady: true, + }, + { + name: "five ready, one not ready", + readyAgents: 5, + notReadyAgents: 1, + expectReady: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + sqlDB := testSQLDB(t) + err := migrations.Up(sqlDB) + require.NoError(t, err) + db := database.New(sqlDB) + + ctx := testutil.Context(t, testutil.WaitShort) + + dbgen.Organization(t, db, database.Organization{ + ID: orgID, + }) + dbgen.User(t, db, database.User{ + ID: userID, + }) + + tmpl := createTemplate(db) + tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) + createWorkspaceBuild(ctx, db, tmpl, tmplV1, &workspaceBuildOpts{ + readyAgents: tc.readyAgents, + notReadyAgents: tc.notReadyAgents, + }) + + workspacePrebuilds := getWorkspacePrebuilds(sqlDB) + require.Len(t, workspacePrebuilds, 1) + require.Equal(t, tc.expectReady, workspacePrebuilds[0].Ready) + }) + } +} + func TestGetPresetsBackoff(t *testing.T) { t.Parallel() if !dbtestutil.WillUsePostgres() { From bd38603aeb13df9565556de8529ce04a60be04e1 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 26 Mar 2025 22:12:50 -0400 Subject: [PATCH 41/72] refactor: avoid code duplication --- coderd/database/querier_test.go | 537 +++++++++++++------------------- 1 file changed, 225 insertions(+), 312 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 1cdd5bbbdeb9d..e0c8c523f30a7 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3588,154 +3588,168 @@ func TestOrganizationDeleteTrigger(t *testing.T) { }) } -func TestWorkspacePrebuildsView(t *testing.T) { - t.Parallel() - if !dbtestutil.WillUsePostgres() { - t.SkipNow() - } +type extTmplVersion struct { + database.TemplateVersion + preset database.TemplateVersionPreset +} - type extTmplVersion struct { - database.TemplateVersion - preset database.TemplateVersionPreset - } +func createTemplate(t *testing.T, db database.Store, orgID uuid.UUID, userID uuid.UUID) database.Template { + // create template + tmpl := dbgen.Template(t, db, database.Template{ + OrganizationID: orgID, + CreatedBy: userID, + ActiveVersionID: uuid.New(), + }) - now := dbtime.Now() - orgID := uuid.New() - userID := uuid.New() + return tmpl +} - createTemplate := func(db database.Store) database.Template { - // create template - tmpl := dbgen.Template(t, db, database.Template{ - OrganizationID: orgID, - CreatedBy: userID, - ActiveVersionID: uuid.New(), - }) +type tmplVersionOpts struct { + DesiredInstances int +} - return tmpl - } - type tmplVersionOpts struct { - DesiredInstances int +func createTmplVersion( + t *testing.T, + db database.Store, + tmpl database.Template, + versionId uuid.UUID, + now time.Time, + opts *tmplVersionOpts, +) extTmplVersion { + // Create template version with corresponding preset and preset prebuild + tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + ID: versionId, + TemplateID: uuid.NullUUID{ + UUID: tmpl.ID, + Valid: true, + }, + OrganizationID: tmpl.OrganizationID, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: tmpl.CreatedBy, + }) + desiredInstances := 1 + if opts != nil { + desiredInstances = opts.DesiredInstances } - createTmplVersion := func(db database.Store, tmpl database.Template, versionId uuid.UUID, opts *tmplVersionOpts) extTmplVersion { - // Create template version with corresponding preset and preset prebuild - tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ - ID: versionId, - TemplateID: uuid.NullUUID{ - UUID: tmpl.ID, - Valid: true, - }, - OrganizationID: tmpl.OrganizationID, - CreatedAt: now, - UpdatedAt: now, - CreatedBy: tmpl.CreatedBy, - }) - desiredInstances := 1 - if opts != nil { - desiredInstances = opts.DesiredInstances - } - preset := dbgen.Preset(t, db, database.InsertPresetParams{ - TemplateVersionID: tmplVersion.ID, - Name: "preset", - DesiredInstances: sql.NullInt32{ - Int32: int32(desiredInstances), - Valid: true, - }, - }) + preset := dbgen.Preset(t, db, database.InsertPresetParams{ + TemplateVersionID: tmplVersion.ID, + Name: "preset", + DesiredInstances: sql.NullInt32{ + Int32: int32(desiredInstances), + Valid: true, + }, + }) - return extTmplVersion{ - TemplateVersion: tmplVersion, - preset: preset, - } - } - type workspaceBuildOpts struct { - successfulJob bool - createdAt time.Time - readyAgents int - notReadyAgents int + return extTmplVersion{ + TemplateVersion: tmplVersion, + preset: preset, } - createWorkspaceBuild := func( - ctx context.Context, - db database.Store, - tmpl database.Template, - extTmplVersion extTmplVersion, - opts *workspaceBuildOpts, - ) { - // Create job with corresponding resource and agent - jobError := sql.NullString{String: "failed", Valid: true} - if opts != nil && opts.successfulJob { - jobError = sql.NullString{} - } - job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ - Type: database.ProvisionerJobTypeWorkspaceBuild, - OrganizationID: orgID, +} - CreatedAt: now.Add(-1 * time.Minute), - Error: jobError, - }) +type workspaceBuildOpts struct { + successfulJob bool + createdAt time.Time + readyAgents int + notReadyAgents int +} - // create ready agents - readyAgents := 0 - if opts != nil { - readyAgents = opts.readyAgents - } - for i := 0; i < readyAgents; i++ { - resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ - JobID: job.ID, - }) - agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ - ResourceID: resource.ID, - }) - err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ - ID: agent.ID, - LifecycleState: database.WorkspaceAgentLifecycleStateReady, - }) - require.NoError(t, err) - } +func createWorkspaceBuild( + t *testing.T, + ctx context.Context, + db database.Store, + tmpl database.Template, + extTmplVersion extTmplVersion, + orgID uuid.UUID, + now time.Time, + opts *workspaceBuildOpts, +) { + // Create job with corresponding resource and agent + jobError := sql.NullString{String: "failed", Valid: true} + if opts != nil && opts.successfulJob { + jobError = sql.NullString{} + } + job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + Type: database.ProvisionerJobTypeWorkspaceBuild, + OrganizationID: orgID, - // create not ready agents - notReadyAgents := 1 - if opts != nil { - notReadyAgents = opts.notReadyAgents - } - for i := 0; i < notReadyAgents; i++ { - resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ - JobID: job.ID, - }) - agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ - ResourceID: resource.ID, - }) - err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ - ID: agent.ID, - LifecycleState: database.WorkspaceAgentLifecycleStateCreated, - }) - require.NoError(t, err) - } + CreatedAt: now.Add(-1 * time.Minute), + Error: jobError, + }) - // Create corresponding workspace and workspace build - workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0"), - OrganizationID: tmpl.OrganizationID, - TemplateID: tmpl.ID, + // create ready agents + readyAgents := 0 + if opts != nil { + readyAgents = opts.readyAgents + } + for i := 0; i < readyAgents; i++ { + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ + JobID: job.ID, }) - createdAt := now - if opts != nil { - createdAt = opts.createdAt - } - dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - CreatedAt: createdAt, - WorkspaceID: workspace.ID, - TemplateVersionID: extTmplVersion.ID, - BuildNumber: 1, - Transition: database.WorkspaceTransitionStart, - InitiatorID: tmpl.CreatedBy, - JobID: job.ID, - TemplateVersionPresetID: uuid.NullUUID{ - UUID: extTmplVersion.preset.ID, - Valid: true, - }, + agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ + ResourceID: resource.ID, }) + err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ + ID: agent.ID, + LifecycleState: database.WorkspaceAgentLifecycleStateReady, + }) + require.NoError(t, err) } + // create not ready agents + notReadyAgents := 1 + if opts != nil { + notReadyAgents = opts.notReadyAgents + } + for i := 0; i < notReadyAgents; i++ { + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ + JobID: job.ID, + }) + agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ + ResourceID: resource.ID, + }) + err := db.UpdateWorkspaceAgentLifecycleStateByID(ctx, database.UpdateWorkspaceAgentLifecycleStateByIDParams{ + ID: agent.ID, + LifecycleState: database.WorkspaceAgentLifecycleStateCreated, + }) + require.NoError(t, err) + } + + // Create corresponding workspace and workspace build + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: uuid.MustParse("c42fdf75-3097-471c-8c33-fb52454d81c0"), + OrganizationID: tmpl.OrganizationID, + TemplateID: tmpl.ID, + }) + createdAt := now + if opts != nil { + createdAt = opts.createdAt + } + dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ + CreatedAt: createdAt, + WorkspaceID: workspace.ID, + TemplateVersionID: extTmplVersion.ID, + BuildNumber: 1, + Transition: database.WorkspaceTransitionStart, + InitiatorID: tmpl.CreatedBy, + JobID: job.ID, + TemplateVersionPresetID: uuid.NullUUID{ + UUID: extTmplVersion.preset.ID, + Valid: true, + }, + }) +} + +func TestWorkspacePrebuildsView(t *testing.T) { + t.Parallel() + if !dbtestutil.WillUsePostgres() { + t.SkipNow() + } + + now := dbtime.Now() + orgID := uuid.New() + userID := uuid.New() + type workspacePrebuild struct { ID uuid.UUID Name string @@ -3816,9 +3830,9 @@ func TestWorkspacePrebuildsView(t *testing.T) { ID: userID, }) - tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) - createWorkspaceBuild(ctx, db, tmpl, tmplV1, &workspaceBuildOpts{ + tmpl := createTemplate(t, db, orgID, userID) + tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ readyAgents: tc.readyAgents, notReadyAgents: tc.notReadyAgents, }) @@ -3836,112 +3850,10 @@ func TestGetPresetsBackoff(t *testing.T) { t.SkipNow() } - type extTmplVersion struct { - database.TemplateVersion - preset database.TemplateVersionPreset - } - now := dbtime.Now() orgID := uuid.New() userID := uuid.New() - createTemplate := func(db database.Store) database.Template { - // create template - tmpl := dbgen.Template(t, db, database.Template{ - OrganizationID: orgID, - CreatedBy: userID, - ActiveVersionID: uuid.New(), - }) - - return tmpl - } - type tmplVersionOpts struct { - DesiredInstances int - } - createTmplVersion := func(db database.Store, tmpl database.Template, versionId uuid.UUID, opts *tmplVersionOpts) extTmplVersion { - // Create template version with corresponding preset and preset prebuild - tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ - ID: versionId, - TemplateID: uuid.NullUUID{ - UUID: tmpl.ID, - Valid: true, - }, - OrganizationID: tmpl.OrganizationID, - CreatedAt: now, - UpdatedAt: now, - CreatedBy: tmpl.CreatedBy, - }) - desiredInstances := 1 - if opts != nil { - desiredInstances = opts.DesiredInstances - } - preset := dbgen.Preset(t, db, database.InsertPresetParams{ - TemplateVersionID: tmplVersion.ID, - Name: "preset", - DesiredInstances: sql.NullInt32{ - Int32: int32(desiredInstances), - Valid: true, - }, - }) - - return extTmplVersion{ - TemplateVersion: tmplVersion, - preset: preset, - } - } - type workspaceBuildOpts struct { - successfulJob bool - createdAt time.Time - } - createWorkspaceBuild := func( - db database.Store, - tmpl database.Template, - extTmplVersion extTmplVersion, - opts *workspaceBuildOpts, - ) { - // Create job with corresponding resource and agent - jobError := sql.NullString{String: "failed", Valid: true} - if opts != nil && opts.successfulJob { - jobError = sql.NullString{} - } - job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ - Type: database.ProvisionerJobTypeWorkspaceBuild, - OrganizationID: orgID, - - CreatedAt: now.Add(-1 * time.Minute), - Error: jobError, - }) - resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{ - JobID: job.ID, - }) - dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ - ResourceID: resource.ID, - }) - - // Create corresponding workspace and workspace build - workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: tmpl.CreatedBy, - OrganizationID: tmpl.OrganizationID, - TemplateID: tmpl.ID, - }) - createdAt := now - if opts != nil { - createdAt = opts.createdAt - } - dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - CreatedAt: createdAt, - WorkspaceID: workspace.ID, - TemplateVersionID: extTmplVersion.ID, - BuildNumber: 1, - Transition: database.WorkspaceTransitionStart, - InitiatorID: tmpl.CreatedBy, - JobID: job.ID, - TemplateVersionPresetID: uuid.NullUUID{ - UUID: extTmplVersion.preset.ID, - Valid: true, - }, - }) - } findBackoffByTmplVersionID := func(backoffs []database.GetPresetsBackoffRow, tmplVersionID uuid.UUID) *database.GetPresetsBackoffRow { for _, backoff := range backoffs { if backoff.TemplateVersionID == tmplVersionID { @@ -3964,9 +3876,9 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl, tmplV1, nil) + tmpl := createTemplate(t, db, orgID, userID) + tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3990,11 +3902,11 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl, tmplV1, nil) - createWorkspaceBuild(db, tmpl, tmplV1, nil) - createWorkspaceBuild(db, tmpl, tmplV1, nil) + tmpl := createTemplate(t, db, orgID, userID) + tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4018,14 +3930,14 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl := createTemplate(db) - tmplV1 := createTmplVersion(db, tmpl, uuid.New(), nil) - createWorkspaceBuild(db, tmpl, tmplV1, nil) + tmpl := createTemplate(t, db, orgID, userID) + tmplV1 := createTmplVersion(t, db, tmpl, uuid.New(), now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) // Active Version - tmplV2 := createTmplVersion(db, tmpl, tmpl.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl, tmplV2, nil) - createWorkspaceBuild(db, tmpl, tmplV2, nil) + tmplV2 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV2, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl, tmplV2, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4049,13 +3961,13 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) - tmpl2 := createTemplate(db) - tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) + tmpl2 := createTemplate(t, db, orgID, userID) + tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4087,23 +3999,23 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl1, tmpl1V1, nil) + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) - tmpl2 := createTemplate(db) - tmpl2V1 := createTmplVersion(db, tmpl2, tmpl2.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) - createWorkspaceBuild(db, tmpl2, tmpl2V1, nil) + tmpl2 := createTemplate(t, db, orgID, userID) + tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) - tmpl3 := createTemplate(db) - tmpl3V1 := createTmplVersion(db, tmpl3, uuid.New(), nil) - createWorkspaceBuild(db, tmpl3, tmpl3V1, nil) + tmpl3 := createTemplate(t, db, orgID, userID) + tmpl3V1 := createTmplVersion(t, db, tmpl3, uuid.New(), now, nil) + createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V1, orgID, now, nil) - tmpl3V2 := createTmplVersion(db, tmpl3, tmpl3.ActiveVersionID, nil) - createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) - createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) - createWorkspaceBuild(db, tmpl3, tmpl3V2, nil) + tmpl3V2 := createTmplVersion(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4141,8 +4053,8 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) _ = tmpl1V1 backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) @@ -4162,14 +4074,14 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, nil) + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) successfulJobOpts := workspaceBuildOpts{ successfulJob: true, } - createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4188,8 +4100,8 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 1, }) failedJobOpts := workspaceBuildOpts{ @@ -4200,8 +4112,8 @@ func TestGetPresetsBackoff(t *testing.T) { successfulJob: true, createdAt: now.Add(-1 * time.Minute), } - createWorkspaceBuild(db, tmpl1, tmpl1V1, &failedJobOpts) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &successfulJobOpts) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4220,23 +4132,23 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4258,19 +4170,19 @@ func TestGetPresetsBackoff(t *testing.T) { ID: userID, }) - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4300,27 +4212,27 @@ func TestGetPresetsBackoff(t *testing.T) { }) lookbackPeriod := time.Hour - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4350,31 +4262,31 @@ func TestGetPresetsBackoff(t *testing.T) { }) lookbackPeriod := time.Hour - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 6, }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-0 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-1 * time.Minute), }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-2 * time.Minute), }) @@ -4406,11 +4318,12 @@ func TestGetPresetsBackoff(t *testing.T) { }) lookbackPeriod := time.Hour - tmpl1 := createTemplate(db) - tmpl1V1 := createTmplVersion(db, tmpl1, tmpl1.ActiveVersionID, &tmplVersionOpts{ + tmpl1 := createTemplate(t, db, orgID, userID) + tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 1, }) - createWorkspaceBuild(db, tmpl1, tmpl1V1, &workspaceBuildOpts{ + + createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) From 097f9c38d7ef0ea23f43d94277f9247df8491357 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Thu, 27 Mar 2025 12:05:34 +0000 Subject: [PATCH 42/72] clarify query clause --- coderd/database/queries.sql.go | 27 +++++++++++++++------------ coderd/database/queries/prebuilds.sql | 27 +++++++++++++++------------ 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 423b7c1ef375a..c1f9c85ce54c6 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5834,18 +5834,21 @@ UPDATE workspaces w SET owner_id = $1::uuid, name = $2::text, updated_at = NOW() -WHERE w.id IN (SELECT p.id - FROM workspace_prebuilds p - INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - INNER JOIN templates t ON p.template_id = t.id - WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status IN ('succeeded'::provisioner_job_status)) - AND b.template_version_id = t.active_version_id - AND b.template_version_preset_id = $3::uuid - AND p.ready - ORDER BY random() - LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. +WHERE w.id IN ( + SELECT p.id + FROM workspace_prebuilds p + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status IN ('succeeded'::provisioner_job_status)) + -- The prebuilds system should never try to claim a prebuild for an inactive template version. + -- Nevertheless, this filter is here as a defensive measure: + AND b.template_version_id = t.active_version_id + AND b.template_version_preset_id = $3::uuid + AND p.ready + LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. +) RETURNING w.id, w.name ` diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index ca7c7e7db07a4..88c947c8a45e2 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -99,18 +99,21 @@ UPDATE workspaces w SET owner_id = @new_user_id::uuid, name = @new_name::text, updated_at = NOW() -WHERE w.id IN (SELECT p.id - FROM workspace_prebuilds p - INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id - INNER JOIN templates t ON p.template_id = t.id - WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status IN ('succeeded'::provisioner_job_status)) - AND b.template_version_id = t.active_version_id - AND b.template_version_preset_id = @preset_id::uuid - AND p.ready - ORDER BY random() - LIMIT 1 FOR UPDATE OF p SKIP LOCKED) -- Ensure that a concurrent request will not select the same prebuild. +WHERE w.id IN ( + SELECT p.id + FROM workspace_prebuilds p + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id + INNER JOIN provisioner_jobs pj ON b.job_id = pj.id + INNER JOIN templates t ON p.template_id = t.id + WHERE (b.transition = 'start'::workspace_transition + AND pj.job_status IN ('succeeded'::provisioner_job_status)) + -- The prebuilds system should never try to claim a prebuild for an inactive template version. + -- Nevertheless, this filter is here as a defensive measure: + AND b.template_version_id = t.active_version_id + AND b.template_version_preset_id = @preset_id::uuid + AND p.ready + LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. +) RETURNING w.id, w.name; -- name: GetPrebuildMetrics :many From 4cfdd6f1dbdb98ad4bc83fecf2f7dfc7d8c9716e Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Thu, 27 Mar 2025 12:26:58 +0000 Subject: [PATCH 43/72] tidy up dbauthz_test.go --- coderd/database/dbauthz/dbauthz_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index b6aac8a167868..bb0535aafb886 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4783,31 +4783,6 @@ func (s *MethodTestSuite) TestPrebuilds() { Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) - // TODO: remove? - // s.Run("InsertPresetPrebuild", s.Subtest(func(db database.Store, check *expects) { - // org := dbgen.Organization(s.T(), db, database.Organization{}) - // user := dbgen.User(s.T(), db, database.User{}) - // template := dbgen.Template(s.T(), db, database.Template{ - // CreatedBy: user.ID, - // OrganizationID: org.ID, - // }) - // templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ - // TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true}, - // OrganizationID: org.ID, - // CreatedBy: user.ID, - // }) - // preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ - // Name: coderdtest.RandomName(s.T()), - // TemplateVersionID: templateVersion.ID, - // }) - // check.Args(database.InsertPresetPrebuildParams{ - // ID: uuid.New(), - // PresetID: preset.ID, - // DesiredInstances: 1, - // }). - // Asserts(rbac.ResourceSystem, policy.ActionCreate). - // ErrorsWithInMemDB(dbmem.ErrUnimplemented) - // })) } func (s *MethodTestSuite) TestOAuth2ProviderApps() { From 4a34d5272ad685f9d56c36f589a944748aef4572 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 09:50:45 -0400 Subject: [PATCH 44/72] refactor: remove * usage from prebuilds.sql queries --- coderd/database/queries.sql.go | 6 +++--- coderd/database/queries/prebuilds.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index c1f9c85ce54c6..d6027d28b4714 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5978,7 +5978,7 @@ func (q *sqlQuerier) GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuilds const getPresetsBackoff = `-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.id, wlb.created_at, wlb.updated_at, wlb.workspace_id, wlb.template_version_id, wlb.build_number, wlb.transition, wlb.initiator_id, wlb.provisioner_state, wlb.job_id, wlb.deadline, wlb.reason, wlb.daily_cost, wlb.max_deadline, wlb.template_version_preset_id, tvp.id AS preset_id, pj.job_status, tvp.desired_instances + SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id @@ -5989,8 +5989,8 @@ WITH filtered_builds AS ( ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. - SELECT fb.id, fb.created_at, fb.updated_at, fb.workspace_id, fb.template_version_id, fb.build_number, fb.transition, fb.initiator_id, fb.provisioner_state, fb.job_id, fb.deadline, fb.reason, fb.daily_cost, fb.max_deadline, fb.template_version_preset_id, fb.preset_id, fb.job_status, fb.desired_instances, - ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, + ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 88c947c8a45e2..143a36477efcf 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -60,7 +60,7 @@ GROUP BY t.id, wpb.template_version_id, wpb.transition; -- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.*, tvp.id AS preset_id, pj.job_status, tvp.desired_instances + SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, pj.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id @@ -71,8 +71,8 @@ WITH filtered_builds AS ( ), time_sorted_builds AS ( -- Group builds by template version, then sort each group by created_at. - SELECT fb.*, - ROW_NUMBER() OVER (PARTITION BY fb.template_version_preset_id ORDER BY fb.created_at DESC) as rn + SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, + ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( From 8d9cd45acfcec6e1725e5f50675020de7676b514 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 10:07:22 -0400 Subject: [PATCH 45/72] refactor: remove * usage from prebuilds views --- coderd/database/dump.sql | 26 ++++--------------- .../migrations/000312_prebuilds.up.sql | 4 +-- coderd/database/models.go | 22 +++------------- 3 files changed, 10 insertions(+), 42 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 7f5fa103f8468..8260f22b22083 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1967,20 +1967,12 @@ COMMENT ON VIEW workspace_build_with_user IS 'Joins in the username + avatar url CREATE VIEW workspace_latest_builds AS SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.id, - workspace_builds.created_at, - workspace_builds.updated_at, workspace_builds.workspace_id, workspace_builds.template_version_id, - workspace_builds.build_number, - workspace_builds.transition, - workspace_builds.initiator_id, - workspace_builds.provisioner_state, workspace_builds.job_id, - workspace_builds.deadline, - workspace_builds.reason, - workspace_builds.daily_cost, - workspace_builds.max_deadline, - workspace_builds.template_version_preset_id + workspace_builds.template_version_preset_id, + workspace_builds.transition, + workspace_builds.created_at FROM workspace_builds ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC; @@ -1996,20 +1988,12 @@ CREATE TABLE workspace_modules ( CREATE VIEW workspace_prebuild_builds AS SELECT workspace_builds.id, - workspace_builds.created_at, - workspace_builds.updated_at, workspace_builds.workspace_id, workspace_builds.template_version_id, - workspace_builds.build_number, workspace_builds.transition, - workspace_builds.initiator_id, - workspace_builds.provisioner_state, workspace_builds.job_id, - workspace_builds.deadline, - workspace_builds.reason, - workspace_builds.daily_cost, - workspace_builds.max_deadline, - workspace_builds.template_version_preset_id + workspace_builds.template_version_preset_id, + workspace_builds.build_number FROM workspace_builds WHERE (workspace_builds.initiator_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid); diff --git a/coderd/database/migrations/000312_prebuilds.up.sql b/coderd/database/migrations/000312_prebuilds.up.sql index 57be23d4af34d..194285d83dc88 100644 --- a/coderd/database/migrations/000312_prebuilds.up.sql +++ b/coderd/database/migrations/000312_prebuilds.up.sql @@ -1,6 +1,6 @@ -- workspace_latest_builds contains latest build for every workspace CREATE VIEW workspace_latest_builds AS -SELECT DISTINCT ON (workspace_id) * +SELECT DISTINCT ON (workspace_id) id, workspace_id, template_version_id, job_id, template_version_preset_id, transition, created_at FROM workspace_builds ORDER BY workspace_id, build_number DESC; @@ -48,6 +48,6 @@ FROM all_prebuilds p INNER JOIN current_presets cp ON cp.prebuild_id = p.id; CREATE VIEW workspace_prebuild_builds AS -SELECT * +SELECT id, workspace_id, template_version_id, transition, job_id, template_version_preset_id, build_number FROM workspace_builds WHERE initiator_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0'; -- The system user responsible for prebuilds. diff --git a/coderd/database/models.go b/coderd/database/models.go index c30257195e620..61051e47d544f 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3557,20 +3557,12 @@ type WorkspaceBuildTable struct { type WorkspaceLatestBuild struct { ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - BuildNumber int32 `db:"build_number" json:"build_number"` - Transition WorkspaceTransition `db:"transition" json:"transition"` - InitiatorID uuid.UUID `db:"initiator_id" json:"initiator_id"` - ProvisionerState []byte `db:"provisioner_state" json:"provisioner_state"` JobID uuid.UUID `db:"job_id" json:"job_id"` - Deadline time.Time `db:"deadline" json:"deadline"` - Reason BuildReason `db:"reason" json:"reason"` - DailyCost int32 `db:"daily_cost" json:"daily_cost"` - MaxDeadline time.Time `db:"max_deadline" json:"max_deadline"` TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + CreatedAt time.Time `db:"created_at" json:"created_at"` } type WorkspaceModule struct { @@ -3594,20 +3586,12 @@ type WorkspacePrebuild struct { type WorkspacePrebuildBuild struct { ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - BuildNumber int32 `db:"build_number" json:"build_number"` Transition WorkspaceTransition `db:"transition" json:"transition"` - InitiatorID uuid.UUID `db:"initiator_id" json:"initiator_id"` - ProvisionerState []byte `db:"provisioner_state" json:"provisioner_state"` JobID uuid.UUID `db:"job_id" json:"job_id"` - Deadline time.Time `db:"deadline" json:"deadline"` - Reason BuildReason `db:"reason" json:"reason"` - DailyCost int32 `db:"daily_cost" json:"daily_cost"` - MaxDeadline time.Time `db:"max_deadline" json:"max_deadline"` TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` + BuildNumber int32 `db:"build_number" json:"build_number"` } type WorkspaceProxy struct { From f870d7ef2fff2ff23f3974738ac47d5970057933 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 12:16:13 -0400 Subject: [PATCH 46/72] refactor: join wlb with pj --- coderd/database/dump.sql | 20 ++++++++++--------- .../migrations/000312_prebuilds.up.sql | 15 +++++++++++--- coderd/database/models.go | 15 +++++++------- coderd/database/queries.sql.go | 12 ++++------- coderd/database/queries/prebuilds.sql | 12 ++++------- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 8260f22b22083..48108fc750b5f 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1966,15 +1966,17 @@ CREATE VIEW workspace_build_with_user AS COMMENT ON VIEW workspace_build_with_user IS 'Joins in the username + avatar url of the initiated by user.'; CREATE VIEW workspace_latest_builds AS - SELECT DISTINCT ON (workspace_builds.workspace_id) workspace_builds.id, - workspace_builds.workspace_id, - workspace_builds.template_version_id, - workspace_builds.job_id, - workspace_builds.template_version_preset_id, - workspace_builds.transition, - workspace_builds.created_at - FROM workspace_builds - ORDER BY workspace_builds.workspace_id, workspace_builds.build_number DESC; + SELECT DISTINCT ON (wb.workspace_id) wb.id, + wb.workspace_id, + wb.template_version_id, + wb.job_id, + wb.template_version_preset_id, + wb.transition, + wb.created_at, + pj.job_status + FROM (workspace_builds wb + JOIN provisioner_jobs pj ON ((wb.job_id = pj.id))) + ORDER BY wb.workspace_id, wb.build_number DESC; CREATE TABLE workspace_modules ( id uuid NOT NULL, diff --git a/coderd/database/migrations/000312_prebuilds.up.sql b/coderd/database/migrations/000312_prebuilds.up.sql index 194285d83dc88..94d9f30a152f2 100644 --- a/coderd/database/migrations/000312_prebuilds.up.sql +++ b/coderd/database/migrations/000312_prebuilds.up.sql @@ -1,8 +1,17 @@ -- workspace_latest_builds contains latest build for every workspace CREATE VIEW workspace_latest_builds AS -SELECT DISTINCT ON (workspace_id) id, workspace_id, template_version_id, job_id, template_version_preset_id, transition, created_at -FROM workspace_builds -ORDER BY workspace_id, build_number DESC; +SELECT DISTINCT ON (workspace_id) + wb.id, + wb.workspace_id, + wb.template_version_id, + wb.job_id, + wb.template_version_preset_id, + wb.transition, + wb.created_at, + pj.job_status +FROM workspace_builds wb + INNER JOIN provisioner_jobs pj ON wb.job_id = pj.id +ORDER BY wb.workspace_id, wb.build_number DESC; -- workspace_prebuilds contains all prebuilt workspaces with corresponding agent information -- (including lifecycle_state which indicates is agent ready or not) and corresponding preset_id for prebuild diff --git a/coderd/database/models.go b/coderd/database/models.go index 61051e47d544f..bed3c15f31ee8 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3556,13 +3556,14 @@ type WorkspaceBuildTable struct { } type WorkspaceLatestBuild struct { - ID uuid.UUID `db:"id" json:"id"` - WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - JobID uuid.UUID `db:"job_id" json:"job_id"` - TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` - Transition WorkspaceTransition `db:"transition" json:"transition"` - CreatedAt time.Time `db:"created_at" json:"created_at"` + ID uuid.UUID `db:"id" json:"id"` + WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + JobID uuid.UUID `db:"job_id" json:"job_id"` + TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` } type WorkspaceModule struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index d6027d28b4714..cb8d0872c9f02 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5838,10 +5838,9 @@ WHERE w.id IN ( SELECT p.id FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status IN ('succeeded'::provisioner_job_status)) + AND b.job_status IN ('succeeded'::provisioner_job_status)) -- The prebuilds system should never try to claim a prebuild for an inactive template version. -- Nevertheless, this filter is here as a defensive measure: AND b.template_version_id = t.active_version_id @@ -5933,10 +5932,9 @@ func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetri const getPrebuildsInProgress = `-- name: GetPrebuildsInProgress :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb - INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id INNER JOIN templates t ON t.active_version_id = wlb.template_version_id -WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) +WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition ` @@ -5978,10 +5976,9 @@ func (q *sqlQuerier) GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuilds const getPresetsBackoff = `-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, pj.job_status, tvp.desired_instances + SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. @@ -6071,9 +6068,8 @@ SELECT p.id AS workspace_id, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status = 'succeeded'::provisioner_job_status) + AND b.job_status = 'succeeded'::provisioner_job_status) ` type GetRunningPrebuildsRow struct { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 143a36477efcf..063e3114c6385 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -31,17 +31,15 @@ SELECT p.id AS workspace_id, p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id -- See https://github.com/coder/internal/issues/398. WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status = 'succeeded'::provisioner_job_status); + AND b.job_status = 'succeeded'::provisioner_job_status); -- name: GetPrebuildsInProgress :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb - INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id INNER JOIN templates t ON t.active_version_id = wlb.template_version_id -WHERE pj.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) +WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; -- GetPresetsBackoff groups workspace builds by template version ID. @@ -60,10 +58,9 @@ GROUP BY t.id, wpb.template_version_id, wpb.transition; -- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations - SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, pj.job_status, tvp.desired_instances + SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - INNER JOIN provisioner_jobs pj ON wlb.job_id = pj.id INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. @@ -103,10 +100,9 @@ WHERE w.id IN ( SELECT p.id FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN provisioner_jobs pj ON b.job_id = pj.id INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition - AND pj.job_status IN ('succeeded'::provisioner_job_status)) + AND b.job_status IN ('succeeded'::provisioner_job_status)) -- The prebuilds system should never try to claim a prebuild for an inactive template version. -- Nevertheless, this filter is here as a defensive measure: AND b.template_version_id = t.active_version_id From 18ad9318e4e4d6ff14d06abdde02f01e2415b465 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 14:12:14 -0400 Subject: [PATCH 47/72] refactor: Rename SQL query --- coderd/database/dbauthz/dbauthz.go | 14 ++-- coderd/database/dbauthz/dbauthz_test.go | 24 +++--- coderd/database/dbmem/dbmem.go | 8 +- coderd/database/dbmetrics/querymetrics.go | 14 ++-- coderd/database/dbmock/dbmock.go | 30 ++++---- coderd/database/querier.go | 4 +- coderd/database/queries.sql.go | 90 ++++++++++++----------- coderd/database/queries/prebuilds.sql | 8 +- 8 files changed, 99 insertions(+), 93 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 36b1c51f44da4..f67f90e10880a 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1165,6 +1165,13 @@ func (q *querier) CleanTailnetTunnels(ctx context.Context) error { return q.db.CleanTailnetTunnels(ctx) } +func (q *querier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + return nil, err + } + return q.db.CountInProgressPrebuilds(ctx) +} + func (q *querier) CountUnreadInboxNotificationsByUserID(ctx context.Context, userID uuid.UUID) (int64, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceInboxNotification.WithOwner(userID.String())); err != nil { return 0, err @@ -2094,13 +2101,6 @@ func (q *querier) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuil return q.db.GetPrebuildMetrics(ctx) } -func (q *querier) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { - return nil, err - } - return q.db.GetPrebuildsInProgress(ctx) -} - func (q *querier) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceID uuid.UUID) (database.TemplateVersionPreset, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { return database.TemplateVersionPreset{}, err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index bb0535aafb886..3fe9723459325 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1748,7 +1748,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1779,7 +1779,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1832,7 +1832,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3843,7 +3843,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4020,7 +4020,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -4065,14 +4065,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -4080,7 +4080,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4091,21 +4091,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4762,7 +4762,7 @@ func (s *MethodTestSuite) TestPrebuilds() { Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) - s.Run("GetPrebuildsInProgress", s.Subtest(func(_ database.Store, check *expects) { + s.Run("CountInProgressPrebuilds", s.Subtest(func(_ database.Store, check *expects) { check.Args(). Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 6005265354bed..2e1762ac9c5d3 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1753,6 +1753,10 @@ func (*FakeQuerier) CleanTailnetTunnels(context.Context) error { return ErrUnimplemented } +func (q *FakeQuerier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { + return nil, ErrUnimplemented +} + func (q *FakeQuerier) CountUnreadInboxNotificationsByUserID(_ context.Context, userID uuid.UUID) (int64, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -4124,10 +4128,6 @@ func (*FakeQuerier) GetPrebuildMetrics(_ context.Context) ([]database.GetPrebuil return nil, ErrUnimplemented } -func (*FakeQuerier) GetPrebuildsInProgress(_ context.Context) ([]database.GetPrebuildsInProgressRow, error) { - return nil, ErrUnimplemented -} - func (q *FakeQuerier) GetPresetByWorkspaceBuildID(_ context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 9894d9b7aacde..c811774e1ebef 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -186,6 +186,13 @@ func (m queryMetricsStore) CleanTailnetTunnels(ctx context.Context) error { return r0 } +func (m queryMetricsStore) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { + start := time.Now() + r0, r1 := m.s.CountInProgressPrebuilds(ctx) + m.queryLatencies.WithLabelValues("CountInProgressPrebuilds").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) CountUnreadInboxNotificationsByUserID(ctx context.Context, userID uuid.UUID) (int64, error) { start := time.Now() r0, r1 := m.s.CountUnreadInboxNotificationsByUserID(ctx, userID) @@ -1054,13 +1061,6 @@ func (m queryMetricsStore) GetPrebuildMetrics(ctx context.Context) ([]database.G return r0, r1 } -func (m queryMetricsStore) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { - start := time.Now() - r0, r1 := m.s.GetPrebuildsInProgress(ctx) - m.queryLatencies.WithLabelValues("GetPrebuildsInProgress").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m queryMetricsStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { start := time.Now() r0, r1 := m.s.GetPresetByWorkspaceBuildID(ctx, workspaceBuildID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 487725237358f..19cfa7402b5e7 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -247,6 +247,21 @@ func (mr *MockStoreMockRecorder) CleanTailnetTunnels(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanTailnetTunnels", reflect.TypeOf((*MockStore)(nil).CleanTailnetTunnels), ctx) } +// CountInProgressPrebuilds mocks base method. +func (m *MockStore) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountInProgressPrebuilds", ctx) + ret0, _ := ret[0].([]database.CountInProgressPrebuildsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountInProgressPrebuilds indicates an expected call of CountInProgressPrebuilds. +func (mr *MockStoreMockRecorder) CountInProgressPrebuilds(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountInProgressPrebuilds", reflect.TypeOf((*MockStore)(nil).CountInProgressPrebuilds), ctx) +} + // CountUnreadInboxNotificationsByUserID mocks base method. func (m *MockStore) CountUnreadInboxNotificationsByUserID(ctx context.Context, userID uuid.UUID) (int64, error) { m.ctrl.T.Helper() @@ -2152,21 +2167,6 @@ func (mr *MockStoreMockRecorder) GetPrebuildMetrics(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrebuildMetrics", reflect.TypeOf((*MockStore)(nil).GetPrebuildMetrics), ctx) } -// GetPrebuildsInProgress mocks base method. -func (m *MockStore) GetPrebuildsInProgress(ctx context.Context) ([]database.GetPrebuildsInProgressRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPrebuildsInProgress", ctx) - ret0, _ := ret[0].([]database.GetPrebuildsInProgressRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPrebuildsInProgress indicates an expected call of GetPrebuildsInProgress. -func (mr *MockStoreMockRecorder) GetPrebuildsInProgress(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrebuildsInProgress", reflect.TypeOf((*MockStore)(nil).GetPrebuildsInProgress), ctx) -} - // GetPresetByWorkspaceBuildID mocks base method. func (m *MockStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 459158cdb70fa..d1ce38dffd4aa 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -64,6 +64,9 @@ type sqlcQuerier interface { CleanTailnetCoordinators(ctx context.Context) error CleanTailnetLostPeers(ctx context.Context) error CleanTailnetTunnels(ctx context.Context) error + // CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by template version ID and transition. + // Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state. + CountInProgressPrebuilds(ctx context.Context) ([]CountInProgressPrebuildsRow, error) CountUnreadInboxNotificationsByUserID(ctx context.Context, userID uuid.UUID) (int64, error) CustomRoles(ctx context.Context, arg CustomRolesParams) ([]CustomRole, error) DeleteAPIKeyByID(ctx context.Context, id string) error @@ -223,7 +226,6 @@ type sqlcQuerier interface { GetOrganizationsByUserID(ctx context.Context, arg GetOrganizationsByUserIDParams) ([]Organization, error) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) - GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuildsInProgressRow, error) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by template version ID. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index cb8d0872c9f02..efc8f7b42f562 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5869,6 +5869,52 @@ func (q *sqlQuerier) ClaimPrebuild(ctx context.Context, arg ClaimPrebuildParams) return i, err } +const countInProgressPrebuilds = `-- name: CountInProgressPrebuilds :many +SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count +FROM workspace_latest_builds wlb + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id +WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) +GROUP BY t.id, wpb.template_version_id, wpb.transition +` + +type CountInProgressPrebuildsRow struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + Transition WorkspaceTransition `db:"transition" json:"transition"` + Count int32 `db:"count" json:"count"` +} + +// CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by template version ID and transition. +// Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state. +func (q *sqlQuerier) CountInProgressPrebuilds(ctx context.Context) ([]CountInProgressPrebuildsRow, error) { + rows, err := q.db.QueryContext(ctx, countInProgressPrebuilds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []CountInProgressPrebuildsRow + for rows.Next() { + var i CountInProgressPrebuildsRow + if err := rows.Scan( + &i.TemplateID, + &i.TemplateVersionID, + &i.Transition, + &i.Count, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getPrebuildMetrics = `-- name: GetPrebuildMetrics :many SELECT t.name as template_name, @@ -5929,50 +5975,6 @@ func (q *sqlQuerier) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetri return items, nil } -const getPrebuildsInProgress = `-- name: GetPrebuildsInProgress :many -SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count -FROM workspace_latest_builds wlb - INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id - INNER JOIN templates t ON t.active_version_id = wlb.template_version_id -WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) -GROUP BY t.id, wpb.template_version_id, wpb.transition -` - -type GetPrebuildsInProgressRow struct { - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` - Transition WorkspaceTransition `db:"transition" json:"transition"` - Count int32 `db:"count" json:"count"` -} - -func (q *sqlQuerier) GetPrebuildsInProgress(ctx context.Context) ([]GetPrebuildsInProgressRow, error) { - rows, err := q.db.QueryContext(ctx, getPrebuildsInProgress) - if err != nil { - return nil, err - } - defer rows.Close() - var items []GetPrebuildsInProgressRow - for rows.Next() { - var i GetPrebuildsInProgressRow - if err := rows.Scan( - &i.TemplateID, - &i.TemplateVersionID, - &i.Transition, - &i.Count, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getPresetsBackoff = `-- name: GetPresetsBackoff :many WITH filtered_builds AS ( -- Only select builds which are for prebuild creations diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 063e3114c6385..d0960caab9c77 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -34,11 +34,13 @@ FROM workspace_prebuilds p WHERE (b.transition = 'start'::workspace_transition AND b.job_status = 'succeeded'::provisioner_job_status); --- name: GetPrebuildsInProgress :many +-- name: CountInProgressPrebuilds :many +-- CountInProgressPrebuilds returns the number of in-progress prebuilds, grouped by template version ID and transition. +-- Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state. SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb - INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id - INNER JOIN templates t ON t.active_version_id = wlb.template_version_id + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; From 4667171b7183a27655e627a3b9f2ed54f1faafe2 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 14:44:15 -0400 Subject: [PATCH 48/72] Added comments for SQL query --- coderd/database/queries.sql.go | 6 ++++++ coderd/database/queries/prebuilds.sql | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index efc8f7b42f562..7d494347ade20 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5873,6 +5873,12 @@ const countInProgressPrebuilds = `-- name: CountInProgressPrebuilds :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + -- We only need these counts for active template versions. + -- It doesn't influence whether we create or delete prebuilds + -- for inactive template versions. This is because we never create + -- prebuilds for inactive template versions, we always delete + -- running prebuilds for inactive template versions, and we ignore + -- prebuilds that are still building. INNER JOIN templates t ON t.active_version_id = wlb.template_version_id WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index d0960caab9c77..793dc136c2a4c 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -40,6 +40,12 @@ WHERE (b.transition = 'start'::workspace_transition SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + -- We only need these counts for active template versions. + -- It doesn't influence whether we create or delete prebuilds + -- for inactive template versions. This is because we never create + -- prebuilds for inactive template versions, we always delete + -- running prebuilds for inactive template versions, and we ignore + -- prebuilds that are still building. INNER JOIN templates t ON t.active_version_id = wlb.template_version_id WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; From a26c0940199fd48c4eeaeec1c4b9561caf72d06f Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 27 Mar 2025 16:26:29 -0400 Subject: [PATCH 49/72] refactor: fix down migration --- coderd/database/migrations/000313_preset_prebuilds.down.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coderd/database/migrations/000313_preset_prebuilds.down.sql b/coderd/database/migrations/000313_preset_prebuilds.down.sql index 5e949be505020..b5bd083e56037 100644 --- a/coderd/database/migrations/000313_preset_prebuilds.down.sql +++ b/coderd/database/migrations/000313_preset_prebuilds.down.sql @@ -1,2 +1,5 @@ -DROP TABLE IF EXISTS template_version_preset_prebuilds; +ALTER TABLE template_version_presets + DROP COLUMN desired_instances, + DROP COLUMN invalidate_after_secs; + DROP INDEX IF EXISTS idx_unique_preset_name; From 2312f41dee667ab9451f79e215afd1cbc54c9020 Mon Sep 17 00:00:00 2001 From: Sas Swart Date: Fri, 28 Mar 2025 07:58:29 +0000 Subject: [PATCH 50/72] renumber migrations --- .../{000312_prebuilds.down.sql => 000313_prebuilds.down.sql} | 0 .../{000312_prebuilds.up.sql => 000313_prebuilds.up.sql} | 0 ...preset_prebuilds.down.sql => 000314_preset_prebuilds.down.sql} | 0 ...313_preset_prebuilds.up.sql => 000314_preset_prebuilds.up.sql} | 0 ...313_preset_prebuilds.up.sql => 000314_preset_prebuilds.up.sql} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000312_prebuilds.down.sql => 000313_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000312_prebuilds.up.sql => 000313_prebuilds.up.sql} (100%) rename coderd/database/migrations/{000313_preset_prebuilds.down.sql => 000314_preset_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000313_preset_prebuilds.up.sql => 000314_preset_prebuilds.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000313_preset_prebuilds.up.sql => 000314_preset_prebuilds.up.sql} (100%) diff --git a/coderd/database/migrations/000312_prebuilds.down.sql b/coderd/database/migrations/000313_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000312_prebuilds.down.sql rename to coderd/database/migrations/000313_prebuilds.down.sql diff --git a/coderd/database/migrations/000312_prebuilds.up.sql b/coderd/database/migrations/000313_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000312_prebuilds.up.sql rename to coderd/database/migrations/000313_prebuilds.up.sql diff --git a/coderd/database/migrations/000313_preset_prebuilds.down.sql b/coderd/database/migrations/000314_preset_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000313_preset_prebuilds.down.sql rename to coderd/database/migrations/000314_preset_prebuilds.down.sql diff --git a/coderd/database/migrations/000313_preset_prebuilds.up.sql b/coderd/database/migrations/000314_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000313_preset_prebuilds.up.sql rename to coderd/database/migrations/000314_preset_prebuilds.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000313_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000314_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000313_preset_prebuilds.up.sql rename to coderd/database/migrations/testdata/fixtures/000314_preset_prebuilds.up.sql From 5150a5ceb563751d2cfd136465ad0fd6937f0a69 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 11:47:33 -0400 Subject: [PATCH 51/72] refactor: clarify comment for SQL query --- coderd/database/querier.go | 7 ++++--- coderd/database/queries.sql.go | 11 ++++++----- coderd/database/queries/prebuilds.sql | 11 ++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 68e9ef861f3b9..4ecf3295dfad2 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -235,11 +235,12 @@ type sqlcQuerier interface { GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) - // GetPresetsBackoff groups workspace builds by template version ID. + // GetPresetsBackoff groups workspace builds by preset ID. + // Each preset is associated with exactly once template version ID. // For each group, the query checks up to N of the most recent jobs that occurred within the // lookback period, where N equals the number of desired instances for the corresponding preset. - // If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. - // Query returns a list of template version IDs for which we should backoff. + // If at least one of the job within a group has failed, we should backoff on the corresponding preset ID. + // Query returns a list of preset IDs for which we should backoff. // Only active template versions with configured presets are considered. // We also return the number of failed workspace builds that occurred during the lookback period. // diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index d8e4017da47fd..5db91cbe6334a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6101,13 +6101,13 @@ WITH filtered_builds AS ( AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( - -- Group builds by template version, then sort each group by created_at. + -- Group builds by preset, then sort each group by created_at. SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( - -- Count failed builds per template version/preset in the given period + -- Count failed builds per preset in the given period SELECT preset_id, COUNT(*) AS num_failed FROM filtered_builds WHERE job_status = 'failed'::provisioner_job_status @@ -6133,11 +6133,12 @@ type GetPresetsBackoffRow struct { LastBuildAt time.Time `db:"last_build_at" json:"last_build_at"` } -// GetPresetsBackoff groups workspace builds by template version ID. +// GetPresetsBackoff groups workspace builds by preset ID. +// Each preset is associated with exactly once template version ID. // For each group, the query checks up to N of the most recent jobs that occurred within the // lookback period, where N equals the number of desired instances for the corresponding preset. -// If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. -// Query returns a list of template version IDs for which we should backoff. +// If at least one of the job within a group has failed, we should backoff on the corresponding preset ID. +// Query returns a list of preset IDs for which we should backoff. // Only active template versions with configured presets are considered. // We also return the number of failed workspace builds that occurred during the lookback period. // diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 793dc136c2a4c..40b1e534f41af 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -50,11 +50,12 @@ FROM workspace_latest_builds wlb WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; --- GetPresetsBackoff groups workspace builds by template version ID. +-- GetPresetsBackoff groups workspace builds by preset ID. +-- Each preset is associated with exactly one template version ID. -- For each group, the query checks up to N of the most recent jobs that occurred within the -- lookback period, where N equals the number of desired instances for the corresponding preset. --- If at least one of the job within a group has failed, we should backoff on the corresponding template version ID. --- Query returns a list of template version IDs for which we should backoff. +-- If at least one of the job within a group has failed, we should backoff on the corresponding preset ID. +-- Query returns a list of preset IDs for which we should backoff. -- Only active template versions with configured presets are considered. -- We also return the number of failed workspace builds that occurred during the lookback period. -- @@ -75,13 +76,13 @@ WITH filtered_builds AS ( AND wlb.transition = 'start'::workspace_transition ), time_sorted_builds AS ( - -- Group builds by template version, then sort each group by created_at. + -- Group builds by preset, then sort each group by created_at. SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( - -- Count failed builds per template version/preset in the given period + -- Count failed builds per preset in the given period SELECT preset_id, COUNT(*) AS num_failed FROM filtered_builds WHERE job_status = 'failed'::provisioner_job_status From bff34eae7d861f1fdcf4f5948ca0b4cb5c264112 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 11:58:29 -0400 Subject: [PATCH 52/72] refactor: fix indentations --- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 12 ++++++------ coderd/database/queries/prebuilds.sql | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 4ecf3295dfad2..08300285cf278 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -236,7 +236,7 @@ type sqlcQuerier interface { GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by preset ID. - // Each preset is associated with exactly once template version ID. + // Each preset is associated with exactly one template version ID. // For each group, the query checks up to N of the most recent jobs that occurred within the // lookback period, where N equals the number of desired instances for the corresponding preset. // If at least one of the job within a group has failed, we should backoff on the corresponding preset ID. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5db91cbe6334a..94e196e7dc11e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6134,7 +6134,7 @@ type GetPresetsBackoffRow struct { } // GetPresetsBackoff groups workspace builds by preset ID. -// Each preset is associated with exactly once template version ID. +// Each preset is associated with exactly one template version ID. // For each group, the query checks up to N of the most recent jobs that occurred within the // lookback period, where N equals the number of desired instances for the corresponding preset. // If at least one of the job within a group has failed, we should backoff on the corresponding preset ID. @@ -6177,12 +6177,12 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) const getRunningPrebuilds = `-- name: GetRunningPrebuilds :many SELECT p.id AS workspace_id, - p.name AS workspace_name, - p.template_id, - b.template_version_id, + p.name AS workspace_name, + p.template_id, + b.template_version_id, p.current_preset_id AS current_preset_id, - p.ready, - p.created_at + p.ready, + p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id WHERE (b.transition = 'start'::workspace_transition diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 40b1e534f41af..644582510faa4 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -23,12 +23,12 @@ WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a pre -- name: GetRunningPrebuilds :many SELECT p.id AS workspace_id, - p.name AS workspace_name, - p.template_id, - b.template_version_id, + p.name AS workspace_name, + p.template_id, + b.template_version_id, p.current_preset_id AS current_preset_id, - p.ready, - p.created_at + p.ready, + p.created_at FROM workspace_prebuilds p INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id WHERE (b.transition = 'start'::workspace_transition From ef462b696fbb8a71fe4c668ccfd3a9c7f97762b7 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 12:10:07 -0400 Subject: [PATCH 53/72] refactor: rename helper func in test package --- coderd/database/querier_test.go | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index e0c8c523f30a7..3ace5f42325a5 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3654,7 +3654,7 @@ type workspaceBuildOpts struct { notReadyAgents int } -func createWorkspaceBuild( +func createPrebuiltWorkspace( t *testing.T, ctx context.Context, db database.Store, @@ -3832,7 +3832,7 @@ func TestWorkspacePrebuildsView(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ readyAgents: tc.readyAgents, notReadyAgents: tc.notReadyAgents, }) @@ -3878,7 +3878,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3904,9 +3904,9 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3932,12 +3932,12 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, uuid.New(), now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) // Active Version tmplV2 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV2, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl, tmplV2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3963,11 +3963,11 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4001,21 +4001,21 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) tmpl3 := createTemplate(t, db, orgID, userID) tmpl3V1 := createTmplVersion(t, db, tmpl3, uuid.New(), now, nil) - createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, nil) tmpl3V2 := createTmplVersion(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) - createWorkspaceBuild(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4079,9 +4079,9 @@ func TestGetPresetsBackoff(t *testing.T) { successfulJobOpts := workspaceBuildOpts{ successfulJob: true, } - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4112,8 +4112,8 @@ func TestGetPresetsBackoff(t *testing.T) { successfulJob: true, createdAt: now.Add(-1 * time.Minute), } - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4136,19 +4136,19 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4174,15 +4174,15 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4216,23 +4216,23 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-2 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: true, createdAt: now.Add(-1 * time.Minute), }) @@ -4266,27 +4266,27 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 6, }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-4 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-0 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-3 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-1 * time.Minute), }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-2 * time.Minute), }) @@ -4323,7 +4323,7 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 1, }) - createWorkspaceBuild(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ successfulJob: false, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) From dc45165d7f66317674a9cb696e506f6dc598e48b Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 12:59:49 -0400 Subject: [PATCH 54/72] refactor: database level tests --- coderd/database/querier_test.go | 160 +++++++++++++++++++------------- 1 file changed, 95 insertions(+), 65 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 3ace5f42325a5..43ef3ed26944a 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3648,7 +3648,7 @@ func createTmplVersion( } type workspaceBuildOpts struct { - successfulJob bool + failedJob bool createdAt time.Time readyAgents int notReadyAgents int @@ -3665,9 +3665,9 @@ func createPrebuiltWorkspace( opts *workspaceBuildOpts, ) { // Create job with corresponding resource and agent - jobError := sql.NullString{String: "failed", Valid: true} - if opts != nil && opts.successfulJob { - jobError = sql.NullString{} + jobError := sql.NullString{} + if opts != nil && opts.failedJob { + jobError = sql.NullString{String: "failed", Valid: true} } job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ Type: database.ProvisionerJobTypeWorkspaceBuild, @@ -3878,7 +3878,9 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3904,9 +3906,15 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3932,12 +3940,18 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) // Active Version tmplV2 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -3963,11 +3977,15 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4001,21 +4019,35 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) tmpl3 := createTemplate(t, db, orgID, userID) tmpl3V1 := createTmplVersion(t, db, tmpl3, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) tmpl3V2 := createTmplVersion(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, nil) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + failedJob: true, + }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4076,9 +4108,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - successfulJobOpts := workspaceBuildOpts{ - successfulJob: true, - } + successfulJobOpts := workspaceBuildOpts{} createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) @@ -4105,12 +4135,12 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 1, }) failedJobOpts := workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-2 * time.Minute), + failedJob: true, + createdAt: now.Add(-2 * time.Minute), } successfulJobOpts := workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-1 * time.Minute), + failedJob: false, + createdAt: now.Add(-1 * time.Minute), } createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) @@ -4137,20 +4167,20 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-4 * time.Minute), + failedJob: true, + createdAt: now.Add(-4 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-3 * time.Minute), + failedJob: false, + createdAt: now.Add(-3 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-2 * time.Minute), + failedJob: false, + createdAt: now.Add(-2 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-1 * time.Minute), + failedJob: false, + createdAt: now.Add(-1 * time.Minute), }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) @@ -4175,16 +4205,16 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-3 * time.Minute), + failedJob: true, + createdAt: now.Add(-3 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-2 * time.Minute), + failedJob: false, + createdAt: now.Add(-2 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-1 * time.Minute), + failedJob: false, + createdAt: now.Add(-1 * time.Minute), }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) @@ -4217,24 +4247,24 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped + failedJob: true, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job + failedJob: true, + createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job + failedJob: true, + createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-2 * time.Minute), + failedJob: false, + createdAt: now.Add(-2 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: true, - createdAt: now.Add(-1 * time.Minute), + failedJob: false, + createdAt: now.Add(-1 * time.Minute), }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) @@ -4267,28 +4297,28 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 6, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped + failedJob: true, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-4 * time.Minute), + failedJob: true, + createdAt: now.Add(-4 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-0 * time.Minute), + failedJob: true, + createdAt: now.Add(-0 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-3 * time.Minute), + failedJob: true, + createdAt: now.Add(-3 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-1 * time.Minute), + failedJob: true, + createdAt: now.Add(-1 * time.Minute), }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-2 * time.Minute), + failedJob: true, + createdAt: now.Add(-2 * time.Minute), }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) @@ -4324,8 +4354,8 @@ func TestGetPresetsBackoff(t *testing.T) { }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ - successfulJob: false, - createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped + failedJob: true, + createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-lookbackPeriod)) From 9c8a352a4593aa95490deea00a0ff1b73f43e9c8 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 13:07:11 -0400 Subject: [PATCH 55/72] refactor: database level tests --- coderd/database/querier_test.go | 82 ++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 43ef3ed26944a..ef9a1660a206e 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3647,7 +3647,7 @@ func createTmplVersion( } } -type workspaceBuildOpts struct { +type createPrebuiltWorkspaceOpts struct { failedJob bool createdAt time.Time readyAgents int @@ -3662,7 +3662,7 @@ func createPrebuiltWorkspace( extTmplVersion extTmplVersion, orgID uuid.UUID, now time.Time, - opts *workspaceBuildOpts, + opts *createPrebuiltWorkspaceOpts, ) { // Create job with corresponding resource and agent jobError := sql.NullString{} @@ -3832,7 +3832,7 @@ func TestWorkspacePrebuildsView(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ readyAgents: tc.readyAgents, notReadyAgents: tc.notReadyAgents, }) @@ -3878,7 +3878,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3906,13 +3906,13 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3940,16 +3940,16 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersion(t, db, tmpl, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) // Active Version tmplV2 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3977,13 +3977,13 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4019,33 +4019,33 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl3 := createTemplate(t, db, orgID, userID) tmpl3V1 := createTmplVersion(t, db, tmpl3, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl3V2 := createTmplVersion(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4108,7 +4108,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - successfulJobOpts := workspaceBuildOpts{} + successfulJobOpts := createPrebuiltWorkspaceOpts{} createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) @@ -4134,11 +4134,11 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 1, }) - failedJobOpts := workspaceBuildOpts{ + failedJobOpts := createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-2 * time.Minute), } - successfulJobOpts := workspaceBuildOpts{ + successfulJobOpts := createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), } @@ -4166,19 +4166,19 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4204,15 +4204,15 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4246,23 +4246,23 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4296,27 +4296,27 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 6, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-0 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-1 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-2 * time.Minute), }) @@ -4353,7 +4353,7 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 1, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &workspaceBuildOpts{ + createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) From eb80919cb498cff848856872f0ee88bc244f6bbb Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 13:11:47 -0400 Subject: [PATCH 56/72] refactor: helper funcs in db-level tests --- coderd/database/querier_test.go | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index ef9a1660a206e..7a7ead26311c2 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3588,7 +3588,7 @@ func TestOrganizationDeleteTrigger(t *testing.T) { }) } -type extTmplVersion struct { +type templateVersionWithPreset struct { database.TemplateVersion preset database.TemplateVersionPreset } @@ -3608,14 +3608,14 @@ type tmplVersionOpts struct { DesiredInstances int } -func createTmplVersion( +func createTmplVersionAndPreset( t *testing.T, db database.Store, tmpl database.Template, versionId uuid.UUID, now time.Time, opts *tmplVersionOpts, -) extTmplVersion { +) templateVersionWithPreset { // Create template version with corresponding preset and preset prebuild tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ ID: versionId, @@ -3641,7 +3641,7 @@ func createTmplVersion( }, }) - return extTmplVersion{ + return templateVersionWithPreset{ TemplateVersion: tmplVersion, preset: preset, } @@ -3659,7 +3659,7 @@ func createPrebuiltWorkspace( ctx context.Context, db database.Store, tmpl database.Template, - extTmplVersion extTmplVersion, + extTmplVersion templateVersionWithPreset, orgID uuid.UUID, now time.Time, opts *createPrebuiltWorkspaceOpts, @@ -3831,7 +3831,7 @@ func TestWorkspacePrebuildsView(t *testing.T) { }) tmpl := createTemplate(t, db, orgID, userID) - tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ readyAgents: tc.readyAgents, notReadyAgents: tc.notReadyAgents, @@ -3877,7 +3877,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl := createTemplate(t, db, orgID, userID) - tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3905,7 +3905,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl := createTemplate(t, db, orgID, userID) - tmplV1 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3939,13 +3939,13 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl := createTemplate(t, db, orgID, userID) - tmplV1 := createTmplVersion(t, db, tmpl, uuid.New(), now, nil) + tmplV1 := createTmplVersionAndPreset(t, db, tmpl, uuid.New(), now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) // Active Version - tmplV2 := createTmplVersion(t, db, tmpl, tmpl.ActiveVersionID, now, nil) + tmplV2 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3976,13 +3976,13 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) - tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) + tmpl2V1 := createTmplVersionAndPreset(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4018,13 +4018,13 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) - tmpl2V1 := createTmplVersion(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) + tmpl2V1 := createTmplVersionAndPreset(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4033,12 +4033,12 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl3 := createTemplate(t, db, orgID, userID) - tmpl3V1 := createTmplVersion(t, db, tmpl3, uuid.New(), now, nil) + tmpl3V1 := createTmplVersionAndPreset(t, db, tmpl3, uuid.New(), now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - tmpl3V2 := createTmplVersion(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) + tmpl3V2 := createTmplVersionAndPreset(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4086,7 +4086,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) _ = tmpl1V1 backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) @@ -4107,7 +4107,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) successfulJobOpts := createPrebuiltWorkspaceOpts{} createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) @@ -4131,7 +4131,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 1, }) failedJobOpts := createPrebuiltWorkspaceOpts{ @@ -4163,7 +4163,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ @@ -4201,7 +4201,7 @@ func TestGetPresetsBackoff(t *testing.T) { }) tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ @@ -4243,7 +4243,7 @@ func TestGetPresetsBackoff(t *testing.T) { lookbackPeriod := time.Hour tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ @@ -4293,7 +4293,7 @@ func TestGetPresetsBackoff(t *testing.T) { lookbackPeriod := time.Hour tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 6, }) createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ @@ -4349,7 +4349,7 @@ func TestGetPresetsBackoff(t *testing.T) { lookbackPeriod := time.Hour tmpl1 := createTemplate(t, db, orgID, userID) - tmpl1V1 := createTmplVersion(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ + tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 1, }) From 0b2bbee18d09749ac91161bef426ee470a584b18 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 14:04:56 -0400 Subject: [PATCH 57/72] refactor: minor improvement in SQL query --- coderd/database/queries.sql.go | 2 +- coderd/database/queries/prebuilds.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 94e196e7dc11e..d4fd25e18fa26 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5952,7 +5952,7 @@ WHERE w.id IN ( -- The prebuilds system should never try to claim a prebuild for an inactive template version. -- Nevertheless, this filter is here as a defensive measure: AND b.template_version_id = t.active_version_id - AND b.template_version_preset_id = $3::uuid + AND p.current_preset_id = $3::uuid AND p.ready LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. ) diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 644582510faa4..d6b9952972de4 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -115,7 +115,7 @@ WHERE w.id IN ( -- The prebuilds system should never try to claim a prebuild for an inactive template version. -- Nevertheless, this filter is here as a defensive measure: AND b.template_version_id = t.active_version_id - AND b.template_version_preset_id = @preset_id::uuid + AND p.current_preset_id = @preset_id::uuid AND p.ready LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. ) From 3a97bf64258e51e6f33fc32fd3248ee75169fdda Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 16:05:04 -0400 Subject: [PATCH 58/72] refactor: rename SQL queries --- coderd/database/dbauthz/dbauthz.go | 4 ++-- coderd/database/dbauthz/dbauthz_test.go | 2 +- coderd/database/dbmem/dbmem.go | 2 +- coderd/database/dbmetrics/querymetrics.go | 6 +++--- coderd/database/dbmock/dbmock.go | 14 +++++++------- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 12 ++++++------ coderd/database/queries/prebuilds.sql | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 809a3a3120e1a..3c53b76d6c973 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2298,11 +2298,11 @@ func (q *querier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Ti return q.db.GetReplicasUpdatedAfter(ctx, updatedAt) } -func (q *querier) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { +func (q *querier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { return nil, err } - return q.db.GetRunningPrebuilds(ctx) + return q.db.GetRunningPrebuiltWorkspaces(ctx) } func (q *querier) GetRuntimeConfig(ctx context.Context, key string) (string, error) { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index cad34aa022bb5..2a33b9b54bfe0 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4821,7 +4821,7 @@ func (s *MethodTestSuite) TestPrebuilds() { Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) - s.Run("GetRunningPrebuilds", s.Subtest(func(_ database.Store, check *expects) { + s.Run("GetRunningPrebuiltWorkspaces", s.Subtest(func(_ database.Store, check *expects) { check.Args(). Asserts(rbac.ResourceTemplate, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 9d797a72b5024..94b3d8e050427 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -4880,7 +4880,7 @@ func (q *FakeQuerier) GetReplicasUpdatedAfter(_ context.Context, updatedAt time. return replicas, nil } -func (*FakeQuerier) GetRunningPrebuilds(_ context.Context) ([]database.GetRunningPrebuildsRow, error) { +func (q *FakeQuerier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { return nil, ErrUnimplemented } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index e3fca0bf713ad..f4ef3dbc64a27 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1236,10 +1236,10 @@ func (m queryMetricsStore) GetReplicasUpdatedAfter(ctx context.Context, updatedA return replicas, err } -func (m queryMetricsStore) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { +func (m queryMetricsStore) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { start := time.Now() - r0, r1 := m.s.GetRunningPrebuilds(ctx) - m.queryLatencies.WithLabelValues("GetRunningPrebuilds").Observe(time.Since(start).Seconds()) + r0, r1 := m.s.GetRunningPrebuiltWorkspaces(ctx) + m.queryLatencies.WithLabelValues("GetRunningPrebuiltWorkspaces").Observe(time.Since(start).Seconds()) return r0, r1 } diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 063f321f89a58..55fd81b3db632 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2539,19 +2539,19 @@ func (mr *MockStoreMockRecorder) GetReplicasUpdatedAfter(ctx, updatedAt any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReplicasUpdatedAfter", reflect.TypeOf((*MockStore)(nil).GetReplicasUpdatedAfter), ctx, updatedAt) } -// GetRunningPrebuilds mocks base method. -func (m *MockStore) GetRunningPrebuilds(ctx context.Context) ([]database.GetRunningPrebuildsRow, error) { +// GetRunningPrebuiltWorkspaces mocks base method. +func (m *MockStore) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRunningPrebuilds", ctx) - ret0, _ := ret[0].([]database.GetRunningPrebuildsRow) + ret := m.ctrl.Call(m, "GetRunningPrebuiltWorkspaces", ctx) + ret0, _ := ret[0].([]database.GetRunningPrebuiltWorkspacesRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetRunningPrebuilds indicates an expected call of GetRunningPrebuilds. -func (mr *MockStoreMockRecorder) GetRunningPrebuilds(ctx any) *gomock.Call { +// GetRunningPrebuiltWorkspaces indicates an expected call of GetRunningPrebuiltWorkspaces. +func (mr *MockStoreMockRecorder) GetRunningPrebuiltWorkspaces(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPrebuilds", reflect.TypeOf((*MockStore)(nil).GetRunningPrebuilds), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPrebuiltWorkspaces", reflect.TypeOf((*MockStore)(nil).GetRunningPrebuiltWorkspaces), ctx) } // GetRuntimeConfig mocks base method. diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 08300285cf278..b571c99661978 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -271,7 +271,7 @@ type sqlcQuerier interface { GetQuotaConsumedForUser(ctx context.Context, arg GetQuotaConsumedForUserParams) (int64, error) GetReplicaByID(ctx context.Context, id uuid.UUID) (Replica, error) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error) - GetRunningPrebuilds(ctx context.Context) ([]GetRunningPrebuildsRow, error) + GetRunningPrebuiltWorkspaces(ctx context.Context) ([]GetRunningPrebuiltWorkspacesRow, error) GetRuntimeConfig(ctx context.Context, key string) (string, error) GetTailnetAgents(ctx context.Context, id uuid.UUID) ([]TailnetAgent, error) GetTailnetClientsForAgent(ctx context.Context, agentID uuid.UUID) ([]TailnetClient, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index d4fd25e18fa26..a69d9c2e3ac69 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6175,7 +6175,7 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) return items, nil } -const getRunningPrebuilds = `-- name: GetRunningPrebuilds :many +const getRunningPrebuiltWorkspaces = `-- name: GetRunningPrebuiltWorkspaces :many SELECT p.id AS workspace_id, p.name AS workspace_name, p.template_id, @@ -6189,7 +6189,7 @@ WHERE (b.transition = 'start'::workspace_transition AND b.job_status = 'succeeded'::provisioner_job_status) ` -type GetRunningPrebuildsRow struct { +type GetRunningPrebuiltWorkspacesRow struct { WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` WorkspaceName string `db:"workspace_name" json:"workspace_name"` TemplateID uuid.UUID `db:"template_id" json:"template_id"` @@ -6199,15 +6199,15 @@ type GetRunningPrebuildsRow struct { CreatedAt time.Time `db:"created_at" json:"created_at"` } -func (q *sqlQuerier) GetRunningPrebuilds(ctx context.Context) ([]GetRunningPrebuildsRow, error) { - rows, err := q.db.QueryContext(ctx, getRunningPrebuilds) +func (q *sqlQuerier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]GetRunningPrebuiltWorkspacesRow, error) { + rows, err := q.db.QueryContext(ctx, getRunningPrebuiltWorkspaces) if err != nil { return nil, err } defer rows.Close() - var items []GetRunningPrebuildsRow + var items []GetRunningPrebuiltWorkspacesRow for rows.Next() { - var i GetRunningPrebuildsRow + var i GetRunningPrebuiltWorkspacesRow if err := rows.Scan( &i.WorkspaceID, &i.WorkspaceName, diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index d6b9952972de4..d322cf1903466 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -21,7 +21,7 @@ FROM templates t WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); --- name: GetRunningPrebuilds :many +-- name: GetRunningPrebuiltWorkspaces :many SELECT p.id AS workspace_id, p.name AS workspace_name, p.template_id, From 2eeb884123d82cc23d0697ce26fe0e4ee8d6940e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 28 Mar 2025 17:07:36 -0400 Subject: [PATCH 59/72] refactor: rename SQL queries --- coderd/database/dbauthz/dbauthz.go | 6 +++--- coderd/database/dbauthz/dbauthz_test.go | 26 +++++++++++------------ coderd/database/dbmem/dbmem.go | 4 ++-- coderd/database/dbmetrics/querymetrics.go | 6 +++--- coderd/database/dbmock/dbmock.go | 14 ++++++------ coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 12 +++++------ coderd/database/queries/prebuilds.sql | 2 +- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 3c53b76d6c973..b6a8c78eae460 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1137,13 +1137,13 @@ func (q *querier) BulkMarkNotificationMessagesSent(ctx context.Context, arg data return q.db.BulkMarkNotificationMessagesSent(ctx, arg) } -func (q *querier) ClaimPrebuild(ctx context.Context, newOwnerID database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { +func (q *querier) ClaimPrebuiltWorkspace(ctx context.Context, newOwnerID database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspace); err != nil { - return database.ClaimPrebuildRow{ + return database.ClaimPrebuiltWorkspaceRow{ ID: uuid.Nil, }, err } - return q.db.ClaimPrebuild(ctx, newOwnerID) + return q.db.ClaimPrebuiltWorkspace(ctx, newOwnerID) } func (q *querier) CleanTailnetCoordinators(ctx context.Context) error { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 2a33b9b54bfe0..299c5957d6a60 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1748,7 +1748,7 @@ func (s *MethodTestSuite) TestUser() { check.Args(database.DeleteCustomRoleParams{ Name: customRole.Name, }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("Blank/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1779,7 +1779,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/UpdateCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -1832,7 +1832,7 @@ func (s *MethodTestSuite) TestUser() { codersdk.ResourceWorkspace: {codersdk.ActionRead}, }), convertSDKPerm), }).Asserts( - // fails immediately, missing organization id + // fails immediately, missing organization id ).Errors(dbauthz.NotAuthorizedError{Err: xerrors.New("custom roles must belong to an organization")}) })) s.Run("OrgPermissions/InsertCustomRole", s.Subtest(func(db database.Store, check *expects) { @@ -3843,7 +3843,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetProvisionerJobsCreatedAfter", s.Subtest(func(db database.Store, check *expects) { // TODO: add provisioner job resource type _ = dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{CreatedAt: time.Now().Add(-time.Hour)}) - check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/) + check.Args(time.Now()).Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ) })) s.Run("GetTemplateVersionsByIDs", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4020,7 +4020,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { a := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) b := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args([]uuid.UUID{a.ID, b.ID}). - Asserts( /*rbac.ResourceSystem, policy.ActionRead*/). + Asserts( /*rbac.ResourceSystem, policy.ActionRead*/ ). Returns(slice.New(a, b)) })) s.Run("InsertWorkspaceAgent", s.Subtest(func(db database.Store, check *expects) { @@ -4065,14 +4065,14 @@ func (s *MethodTestSuite) TestSystemFunctions() { OrganizationID: j.OrganizationID, Types: []database.ProvisionerType{j.Provisioner}, ProvisionerTags: must(json.Marshal(j.Tags)), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobWithCompleteByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.UpdateProvisionerJobWithCompleteByIDParams{ ID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("UpdateProvisionerJobByID", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource @@ -4080,7 +4080,7 @@ func (s *MethodTestSuite) TestSystemFunctions() { check.Args(database.UpdateProvisionerJobByIDParams{ ID: j.ID, UpdatedAt: time.Now(), - }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionUpdate*/ ) })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4091,21 +4091,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { StorageMethod: database.ProvisionerStorageMethodFile, Type: database.ProvisionerJobTypeWorkspaceBuild, Input: json.RawMessage("{}"), - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobLogs", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobLogsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("InsertProvisionerJobTimings", s.Subtest(func(db database.Store, check *expects) { // TODO: we need to create a ProvisionerJob resource j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) check.Args(database.InsertProvisionerJobTimingsParams{ JobID: j.ID, - }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/) + }).Asserts( /*rbac.ResourceSystem, policy.ActionCreate*/ ) })) s.Run("UpsertProvisionerDaemon", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) @@ -4800,8 +4800,8 @@ func (s *MethodTestSuite) TestNotifications() { } func (s *MethodTestSuite) TestPrebuilds() { - s.Run("ClaimPrebuild", s.Subtest(func(db database.Store, check *expects) { - check.Args(database.ClaimPrebuildParams{}). + s.Run("ClaimPrebuiltWorkspace", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.ClaimPrebuiltWorkspaceParams{}). Asserts(rbac.ResourceWorkspace, policy.ActionUpdate). ErrorsWithInMemDB(dbmem.ErrUnimplemented). ErrorsWithPG(sql.ErrNoRows) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 94b3d8e050427..59995cfcc51d6 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1740,8 +1740,8 @@ func (*FakeQuerier) BulkMarkNotificationMessagesSent(_ context.Context, arg data return int64(len(arg.IDs)), nil } -func (*FakeQuerier) ClaimPrebuild(_ context.Context, _ database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { - return database.ClaimPrebuildRow{}, ErrUnimplemented +func (q *FakeQuerier) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { + return database.ClaimPrebuiltWorkspaceRow{}, ErrUnimplemented } func (*FakeQuerier) CleanTailnetCoordinators(_ context.Context) error { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index f4ef3dbc64a27..f2b36f1197977 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -158,10 +158,10 @@ func (m queryMetricsStore) BulkMarkNotificationMessagesSent(ctx context.Context, return r0, r1 } -func (m queryMetricsStore) ClaimPrebuild(ctx context.Context, newOwnerID database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { +func (m queryMetricsStore) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { start := time.Now() - r0, r1 := m.s.ClaimPrebuild(ctx, newOwnerID) - m.queryLatencies.WithLabelValues("ClaimPrebuild").Observe(time.Since(start).Seconds()) + r0, r1 := m.s.ClaimPrebuiltWorkspace(ctx, arg) + m.queryLatencies.WithLabelValues("ClaimPrebuiltWorkspace").Observe(time.Since(start).Seconds()) return r0, r1 } diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 55fd81b3db632..bc96993ef0868 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -190,19 +190,19 @@ func (mr *MockStoreMockRecorder) BulkMarkNotificationMessagesSent(ctx, arg any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BulkMarkNotificationMessagesSent", reflect.TypeOf((*MockStore)(nil).BulkMarkNotificationMessagesSent), ctx, arg) } -// ClaimPrebuild mocks base method. -func (m *MockStore) ClaimPrebuild(ctx context.Context, arg database.ClaimPrebuildParams) (database.ClaimPrebuildRow, error) { +// ClaimPrebuiltWorkspace mocks base method. +func (m *MockStore) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClaimPrebuild", ctx, arg) - ret0, _ := ret[0].(database.ClaimPrebuildRow) + ret := m.ctrl.Call(m, "ClaimPrebuiltWorkspace", ctx, arg) + ret0, _ := ret[0].(database.ClaimPrebuiltWorkspaceRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// ClaimPrebuild indicates an expected call of ClaimPrebuild. -func (mr *MockStoreMockRecorder) ClaimPrebuild(ctx, arg any) *gomock.Call { +// ClaimPrebuiltWorkspace indicates an expected call of ClaimPrebuiltWorkspace. +func (mr *MockStoreMockRecorder) ClaimPrebuiltWorkspace(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimPrebuild", reflect.TypeOf((*MockStore)(nil).ClaimPrebuild), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClaimPrebuiltWorkspace", reflect.TypeOf((*MockStore)(nil).ClaimPrebuiltWorkspace), ctx, arg) } // CleanTailnetCoordinators mocks base method. diff --git a/coderd/database/querier.go b/coderd/database/querier.go index b571c99661978..45e41924da718 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -60,7 +60,7 @@ type sqlcQuerier interface { BatchUpdateWorkspaceNextStartAt(ctx context.Context, arg BatchUpdateWorkspaceNextStartAtParams) error BulkMarkNotificationMessagesFailed(ctx context.Context, arg BulkMarkNotificationMessagesFailedParams) (int64, error) BulkMarkNotificationMessagesSent(ctx context.Context, arg BulkMarkNotificationMessagesSentParams) (int64, error) - ClaimPrebuild(ctx context.Context, arg ClaimPrebuildParams) (ClaimPrebuildRow, error) + ClaimPrebuiltWorkspace(ctx context.Context, arg ClaimPrebuiltWorkspaceParams) (ClaimPrebuiltWorkspaceRow, error) CleanTailnetCoordinators(ctx context.Context) error CleanTailnetLostPeers(ctx context.Context) error CleanTailnetTunnels(ctx context.Context) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index a69d9c2e3ac69..7885cb1029a93 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5937,7 +5937,7 @@ func (q *sqlQuerier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid. return items, nil } -const claimPrebuild = `-- name: ClaimPrebuild :one +const claimPrebuiltWorkspace = `-- name: ClaimPrebuiltWorkspace :one UPDATE workspaces w SET owner_id = $1::uuid, name = $2::text, @@ -5959,20 +5959,20 @@ WHERE w.id IN ( RETURNING w.id, w.name ` -type ClaimPrebuildParams struct { +type ClaimPrebuiltWorkspaceParams struct { NewUserID uuid.UUID `db:"new_user_id" json:"new_user_id"` NewName string `db:"new_name" json:"new_name"` PresetID uuid.UUID `db:"preset_id" json:"preset_id"` } -type ClaimPrebuildRow struct { +type ClaimPrebuiltWorkspaceRow struct { ID uuid.UUID `db:"id" json:"id"` Name string `db:"name" json:"name"` } -func (q *sqlQuerier) ClaimPrebuild(ctx context.Context, arg ClaimPrebuildParams) (ClaimPrebuildRow, error) { - row := q.db.QueryRowContext(ctx, claimPrebuild, arg.NewUserID, arg.NewName, arg.PresetID) - var i ClaimPrebuildRow +func (q *sqlQuerier) ClaimPrebuiltWorkspace(ctx context.Context, arg ClaimPrebuiltWorkspaceParams) (ClaimPrebuiltWorkspaceRow, error) { + row := q.db.QueryRowContext(ctx, claimPrebuiltWorkspace, arg.NewUserID, arg.NewName, arg.PresetID) + var i ClaimPrebuiltWorkspaceRow err := row.Scan(&i.ID, &i.Name) return i, err } diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index d322cf1903466..6f5078d0e3bba 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -100,7 +100,7 @@ WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the AND created_at >= @lookback::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; --- name: ClaimPrebuild :one +-- name: ClaimPrebuiltWorkspace :one UPDATE workspaces w SET owner_id = @new_user_id::uuid, name = @new_name::text, From 73f99e828c08bd8302aade8ce6a2383b0a28e08e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Mon, 31 Mar 2025 09:03:17 -0400 Subject: [PATCH 60/72] refactor: rename fields in SQL query --- coderd/database/queries.sql.go | 12 ++++++------ coderd/database/queries/prebuilds.sql | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 7885cb1029a93..924be57c01fbf 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6176,8 +6176,8 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) } const getRunningPrebuiltWorkspaces = `-- name: GetRunningPrebuiltWorkspaces :many -SELECT p.id AS workspace_id, - p.name AS workspace_name, +SELECT p.id, + p.name, p.template_id, b.template_version_id, p.current_preset_id AS current_preset_id, @@ -6190,8 +6190,8 @@ WHERE (b.transition = 'start'::workspace_transition ` type GetRunningPrebuiltWorkspacesRow struct { - WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` - WorkspaceName string `db:"workspace_name" json:"workspace_name"` + ID uuid.UUID `db:"id" json:"id"` + Name string `db:"name" json:"name"` TemplateID uuid.UUID `db:"template_id" json:"template_id"` TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` CurrentPresetID uuid.NullUUID `db:"current_preset_id" json:"current_preset_id"` @@ -6209,8 +6209,8 @@ func (q *sqlQuerier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]GetRun for rows.Next() { var i GetRunningPrebuiltWorkspacesRow if err := rows.Scan( - &i.WorkspaceID, - &i.WorkspaceName, + &i.ID, + &i.Name, &i.TemplateID, &i.TemplateVersionID, &i.CurrentPresetID, diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 6f5078d0e3bba..289e496dce3d7 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -22,8 +22,8 @@ WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a pre AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); -- name: GetRunningPrebuiltWorkspaces :many -SELECT p.id AS workspace_id, - p.name AS workspace_name, +SELECT p.id, + p.name, p.template_id, b.template_version_id, p.current_preset_id AS current_preset_id, From 113e12bfdc0ab57f36e7c235a2bb9574254f7bb1 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 1 Apr 2025 14:36:29 -0400 Subject: [PATCH 61/72] fix: improve rbac policies --- coderd/database/dbauthz/dbauthz.go | 54 +++++++++++++---- coderd/database/dbauthz/dbauthz_test.go | 72 ++++++++++++++++++++--- coderd/database/dbmem/dbmem.go | 4 ++ coderd/database/dbmetrics/querymetrics.go | 7 +++ coderd/database/dbmock/dbmock.go | 15 +++++ coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 34 +++++++++++ coderd/database/queries/presets.sql | 6 ++ 8 files changed, 174 insertions(+), 19 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index b6a8c78eae460..b46f07c1630e6 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -378,6 +378,7 @@ var ( policy.ActionCreate, policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, }, + rbac.ResourceSystem.Type: {policy.ActionRead}, }), }, }), @@ -1137,13 +1138,29 @@ func (q *querier) BulkMarkNotificationMessagesSent(ctx context.Context, arg data return q.db.BulkMarkNotificationMessagesSent(ctx, arg) } -func (q *querier) ClaimPrebuiltWorkspace(ctx context.Context, newOwnerID database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { - if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspace); err != nil { - return database.ClaimPrebuiltWorkspaceRow{ - ID: uuid.Nil, - }, err +func (q *querier) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { + empty := database.ClaimPrebuiltWorkspaceRow{} + + preset, err := q.db.GetPresetByID(ctx, arg.PresetID) + if err != nil { + return empty, err + } + + workspaceObject := rbac.ResourceWorkspace.WithOwner(arg.NewUserID.String()).InOrg(preset.OrganizationID) + err = q.authorizeContext(ctx, policy.ActionCreate, workspaceObject.RBACObject()) + if err != nil { + return empty, err + } + + tpl, err := q.GetTemplateByID(ctx, preset.TemplateID.UUID) + if err != nil { + return empty, xerrors.Errorf("verify template by id: %w", err) } - return q.db.ClaimPrebuiltWorkspace(ctx, newOwnerID) + if err := q.authorizeContext(ctx, policy.ActionUse, tpl); err != nil { + return empty, xerrors.Errorf("use template for workspace: %w", err) + } + + return q.db.ClaimPrebuiltWorkspace(ctx, arg) } func (q *querier) CleanTailnetCoordinators(ctx context.Context) error { @@ -1168,7 +1185,7 @@ func (q *querier) CleanTailnetTunnels(ctx context.Context) error { } func (q *querier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } return q.db.CountInProgressPrebuilds(ctx) @@ -2118,12 +2135,27 @@ func (q *querier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUI } func (q *querier) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuildMetricsRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } return q.db.GetPrebuildMetrics(ctx) } +func (q *querier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (database.GetPresetByIDRow, error) { + empty := database.GetPresetByIDRow{} + + preset, err := q.db.GetPresetByID(ctx, presetID) + if err != nil { + return empty, err + } + _, err = q.GetTemplateByID(ctx, preset.TemplateID.UUID) + if err != nil { + return empty, err + } + + return preset, nil +} + func (q *querier) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceID uuid.UUID) (database.TemplateVersionPreset, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { return database.TemplateVersionPreset{}, err @@ -2142,7 +2174,7 @@ func (q *querier) GetPresetParametersByTemplateVersionID(ctx context.Context, te } func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } return q.db.GetPresetsBackoff(ctx, lookback) @@ -2299,7 +2331,7 @@ func (q *querier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Ti } func (q *querier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } return q.db.GetRunningPrebuiltWorkspaces(ctx) @@ -2433,7 +2465,7 @@ func (q *querier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateI // Although this fetches presets. It filters them by prebuilds and is only of use to the prebuild system. // As such, we authorize this in line with other prebuild queries, not with other preset queries. - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return nil, err } return q.db.GetTemplatePresetsWithPrebuilds(ctx, templateID) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 299c5957d6a60..e8a62138255c9 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4801,37 +4801,93 @@ func (s *MethodTestSuite) TestNotifications() { func (s *MethodTestSuite) TestPrebuilds() { s.Run("ClaimPrebuiltWorkspace", s.Subtest(func(db database.Store, check *expects) { - check.Args(database.ClaimPrebuiltWorkspaceParams{}). - Asserts(rbac.ResourceWorkspace, policy.ActionUpdate). - ErrorsWithInMemDB(dbmem.ErrUnimplemented). + org := dbgen.Organization(s.T(), db, database.Organization{}) + user := dbgen.User(s.T(), db, database.User{}) + template := dbgen.Template(s.T(), db, database.Template{ + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ + TemplateID: uuid.NullUUID{ + UUID: template.ID, + Valid: true, + }, + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ + TemplateVersionID: templateVersion.ID, + }) + check.Args(database.ClaimPrebuiltWorkspaceParams{ + NewUserID: user.ID, + NewName: "", + PresetID: preset.ID, + }).Asserts( + rbac.ResourceWorkspace.WithOwner(user.ID.String()).InOrg(org.ID), policy.ActionCreate, + template, policy.ActionRead, + template, policy.ActionUse, + ).ErrorsWithInMemDB(dbmem.ErrUnimplemented). ErrorsWithPG(sql.ErrNoRows) })) s.Run("GetPrebuildMetrics", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceTemplate, policy.ActionRead). + Asserts(rbac.ResourceSystem, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("CountInProgressPrebuilds", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceTemplate, policy.ActionRead). + Asserts(rbac.ResourceSystem, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) { check.Args(time.Time{}). - Asserts(rbac.ResourceTemplate, policy.ActionRead). + Asserts(rbac.ResourceSystem, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetRunningPrebuiltWorkspaces", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceTemplate, policy.ActionRead). + Asserts(rbac.ResourceSystem, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetTemplatePresetsWithPrebuilds", s.Subtest(func(db database.Store, check *expects) { user := dbgen.User(s.T(), db, database.User{}) check.Args(uuid.NullUUID{UUID: user.ID, Valid: true}). - Asserts(rbac.ResourceTemplate, policy.ActionRead). + Asserts(rbac.ResourceSystem, policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) + s.Run("GetPresetByID", s.Subtest(func(db database.Store, check *expects) { + org := dbgen.Organization(s.T(), db, database.Organization{}) + user := dbgen.User(s.T(), db, database.User{}) + template := dbgen.Template(s.T(), db, database.Template{ + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + templateVersion := dbgen.TemplateVersion(s.T(), db, database.TemplateVersion{ + TemplateID: uuid.NullUUID{ + UUID: template.ID, + Valid: true, + }, + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + preset := dbgen.Preset(s.T(), db, database.InsertPresetParams{ + TemplateVersionID: templateVersion.ID, + }) + check.Args(preset.ID). + Asserts(template, policy.ActionRead). + ErrorsWithInMemDB(dbmem.ErrUnimplemented). + Returns(database.GetPresetByIDRow{ + ID: preset.ID, + TemplateVersionID: preset.TemplateVersionID, + Name: preset.Name, + CreatedAt: preset.CreatedAt, + TemplateID: uuid.NullUUID{ + UUID: template.ID, + Valid: true, + }, + OrganizationID: org.ID, + }) + })) } func (s *MethodTestSuite) TestOAuth2ProviderApps() { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 59995cfcc51d6..bb71cedc0a939 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -4171,6 +4171,10 @@ func (*FakeQuerier) GetPrebuildMetrics(_ context.Context) ([]database.GetPrebuil return nil, ErrUnimplemented } +func (q *FakeQuerier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (database.GetPresetByIDRow, error) { + return database.GetPresetByIDRow{}, ErrUnimplemented +} + func (q *FakeQuerier) GetPresetByWorkspaceBuildID(_ context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { q.mutex.RLock() defer q.mutex.RUnlock() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index f2b36f1197977..f29da9714d5f3 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1082,6 +1082,13 @@ func (m queryMetricsStore) GetPrebuildMetrics(ctx context.Context) ([]database.G return r0, r1 } +func (m queryMetricsStore) GetPresetByID(ctx context.Context, presetID uuid.UUID) (database.GetPresetByIDRow, error) { + start := time.Now() + r0, r1 := m.s.GetPresetByID(ctx, presetID) + m.queryLatencies.WithLabelValues("GetPresetByID").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { start := time.Now() r0, r1 := m.s.GetPresetByWorkspaceBuildID(ctx, workspaceBuildID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index bc96993ef0868..49e9ff244ab94 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2209,6 +2209,21 @@ func (mr *MockStoreMockRecorder) GetPrebuildMetrics(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrebuildMetrics", reflect.TypeOf((*MockStore)(nil).GetPrebuildMetrics), ctx) } +// GetPresetByID mocks base method. +func (m *MockStore) GetPresetByID(ctx context.Context, presetID uuid.UUID) (database.GetPresetByIDRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPresetByID", ctx, presetID) + ret0, _ := ret[0].(database.GetPresetByIDRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPresetByID indicates an expected call of GetPresetByID. +func (mr *MockStoreMockRecorder) GetPresetByID(ctx, presetID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPresetByID", reflect.TypeOf((*MockStore)(nil).GetPresetByID), ctx, presetID) +} + // GetPresetByWorkspaceBuildID mocks base method. func (m *MockStore) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 45e41924da718..e5130ee18eadd 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -233,6 +233,7 @@ type sqlcQuerier interface { GetOrganizationsByUserID(ctx context.Context, arg GetOrganizationsByUserIDParams) ([]Organization, error) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) GetPrebuildMetrics(ctx context.Context) ([]GetPrebuildMetricsRow, error) + GetPresetByID(ctx context.Context, presetID uuid.UUID) (GetPresetByIDRow, error) GetPresetByWorkspaceBuildID(ctx context.Context, workspaceBuildID uuid.UUID) (TemplateVersionPreset, error) GetPresetParametersByTemplateVersionID(ctx context.Context, templateVersionID uuid.UUID) ([]TemplateVersionPresetParameter, error) // GetPresetsBackoff groups workspace builds by preset ID. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 924be57c01fbf..4876a77afb209 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6303,6 +6303,40 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa return items, nil } +const getPresetByID = `-- name: GetPresetByID :one +SELECT tvp.id, tvp.template_version_id, tvp.name, tvp.created_at, tvp.desired_instances, tvp.invalidate_after_secs, tv.template_id, tv.organization_id FROM + template_version_presets tvp + INNER JOIN template_versions tv ON tvp.template_version_id = tv.id +WHERE tvp.id = $1 +` + +type GetPresetByIDRow struct { + ID uuid.UUID `db:"id" json:"id"` + TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"` + Name string `db:"name" json:"name"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"` + InvalidateAfterSecs sql.NullInt32 `db:"invalidate_after_secs" json:"invalidate_after_secs"` + TemplateID uuid.NullUUID `db:"template_id" json:"template_id"` + OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"` +} + +func (q *sqlQuerier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (GetPresetByIDRow, error) { + row := q.db.QueryRowContext(ctx, getPresetByID, presetID) + var i GetPresetByIDRow + err := row.Scan( + &i.ID, + &i.TemplateVersionID, + &i.Name, + &i.CreatedAt, + &i.DesiredInstances, + &i.InvalidateAfterSecs, + &i.TemplateID, + &i.OrganizationID, + ) + return i, err +} + const getPresetByWorkspaceBuildID = `-- name: GetPresetByWorkspaceBuildID :one SELECT template_version_presets.id, template_version_presets.template_version_id, template_version_presets.name, template_version_presets.created_at, template_version_presets.desired_instances, template_version_presets.invalidate_after_secs diff --git a/coderd/database/queries/presets.sql b/coderd/database/queries/presets.sql index 93c44fbc5898a..5329e6d999954 100644 --- a/coderd/database/queries/presets.sql +++ b/coderd/database/queries/presets.sql @@ -48,3 +48,9 @@ FROM INNER JOIN template_version_presets ON template_version_preset_parameters.template_version_preset_id = template_version_presets.id WHERE template_version_presets.template_version_id = @template_version_id; + +-- name: GetPresetByID :one +SELECT tvp.*, tv.template_id, tv.organization_id FROM + template_version_presets tvp + INNER JOIN template_versions tv ON tvp.template_version_id = tv.id +WHERE tvp.id = @preset_id; From 217cae090704eb5dabd48941ae14c1969f365249 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 1 Apr 2025 19:52:52 -0400 Subject: [PATCH 62/72] fix: minor fix in filtered_builds CTE --- coderd/database/queries.sql.go | 2 ++ coderd/database/queries/prebuilds.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 4876a77afb209..0674be6f1c824 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6095,10 +6095,12 @@ WITH filtered_builds AS ( SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN workspaces w ON wlb.workspace_id = w.id INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND wlb.transition = 'start'::workspace_transition + AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' ), time_sorted_builds AS ( -- Group builds by preset, then sort each group by created_at. diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 289e496dce3d7..f9158dbda7e74 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -70,10 +70,12 @@ WITH filtered_builds AS ( SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN workspaces w ON wlb.workspace_id = w.id INNER JOIN template_versions tv ON wlb.template_version_id = tv.id INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. AND wlb.transition = 'start'::workspace_transition + AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' ), time_sorted_builds AS ( -- Group builds by preset, then sort each group by created_at. From b0bf2202fc4f093721c79fae6732cbbc21f95f4a Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 1 Apr 2025 20:22:23 -0400 Subject: [PATCH 63/72] fix: formatting --- .../migrations/000313_prebuilds.up.sql | 40 ++++---- .../migrations/000314_preset_prebuilds.up.sql | 4 +- coderd/database/queries.sql.go | 98 ++++++++++--------- coderd/database/queries/prebuilds.sql | 96 +++++++++--------- coderd/database/queries/presets.sql | 2 +- 5 files changed, 122 insertions(+), 118 deletions(-) diff --git a/coderd/database/migrations/000313_prebuilds.up.sql b/coderd/database/migrations/000313_prebuilds.up.sql index 94d9f30a152f2..0e8ff4ef6e408 100644 --- a/coderd/database/migrations/000313_prebuilds.up.sql +++ b/coderd/database/migrations/000313_prebuilds.up.sql @@ -17,29 +17,29 @@ ORDER BY wb.workspace_id, wb.build_number DESC; -- (including lifecycle_state which indicates is agent ready or not) and corresponding preset_id for prebuild CREATE VIEW workspace_prebuilds AS WITH - -- All workspaces owned by the "prebuilds" user. - all_prebuilds AS ( + -- All workspaces owned by the "prebuilds" user. + all_prebuilds AS ( SELECT w.id, w.name, w.template_id, w.created_at FROM workspaces w WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. ), - -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the - -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, - -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. - -- - -- See https://github.com/coder/internal/issues/398 - workspaces_with_latest_presets AS ( - SELECT DISTINCT ON (workspace_id) workspace_id, template_version_preset_id - FROM workspace_builds - WHERE template_version_preset_id IS NOT NULL - ORDER BY workspace_id, build_number DESC - ), + -- We can't rely on the template_version_preset_id in the workspace_builds table because this value is only set on the + -- initial workspace creation. Subsequent stop/start transitions will not have a value for template_version_preset_id, + -- and therefore we can't rely on (say) the latest build's chosen template_version_preset_id. + -- + -- See https://github.com/coder/internal/issues/398 + workspaces_with_latest_presets AS ( + SELECT DISTINCT ON (workspace_id) workspace_id, template_version_preset_id + FROM workspace_builds + WHERE template_version_preset_id IS NOT NULL + ORDER BY workspace_id, build_number DESC + ), -- workspaces_with_agents_status contains workspaces owned by the "prebuilds" user, -- along with the readiness status of their agents. -- A workspace is marked as 'ready' only if ALL of its agents are ready. workspaces_with_agents_status AS ( SELECT w.id AS workspace_id, - BOOL_AND(wa.lifecycle_state = 'ready'::workspace_agent_lifecycle_state) AS ready + BOOL_AND(wa.lifecycle_state = 'ready'::workspace_agent_lifecycle_state) AS ready FROM workspaces w INNER JOIN workspace_latest_builds wlb ON wlb.workspace_id = w.id INNER JOIN workspace_resources wr ON wr.job_id = wlb.job_id @@ -47,14 +47,14 @@ WITH WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' -- The system user responsible for prebuilds. GROUP BY w.id ), - current_presets AS (SELECT w.id AS prebuild_id, wlp.template_version_preset_id - FROM workspaces w - INNER JOIN workspaces_with_latest_presets wlp ON wlp.workspace_id = w.id - WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. + current_presets AS (SELECT w.id AS prebuild_id, wlp.template_version_preset_id + FROM workspaces w + INNER JOIN workspaces_with_latest_presets wlp ON wlp.workspace_id = w.id + WHERE w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0') -- The system user responsible for prebuilds. SELECT p.id, p.name, p.template_id, p.created_at, COALESCE(a.ready, false) AS ready, cp.template_version_preset_id AS current_preset_id FROM all_prebuilds p - LEFT JOIN workspaces_with_agents_status a ON a.workspace_id = p.id - INNER JOIN current_presets cp ON cp.prebuild_id = p.id; + LEFT JOIN workspaces_with_agents_status a ON a.workspace_id = p.id + INNER JOIN current_presets cp ON cp.prebuild_id = p.id; CREATE VIEW workspace_prebuild_builds AS SELECT id, workspace_id, template_version_id, transition, job_id, template_version_preset_id, build_number diff --git a/coderd/database/migrations/000314_preset_prebuilds.up.sql b/coderd/database/migrations/000314_preset_prebuilds.up.sql index 03ee6515e741c..392bcf741b233 100644 --- a/coderd/database/migrations/000314_preset_prebuilds.up.sql +++ b/coderd/database/migrations/000314_preset_prebuilds.up.sql @@ -1,6 +1,6 @@ ALTER TABLE template_version_presets - ADD COLUMN desired_instances INT NULL, - ADD COLUMN invalidate_after_secs INT NULL DEFAULT 0; + ADD COLUMN desired_instances INT NULL, + ADD COLUMN invalidate_after_secs INT NULL DEFAULT 0; -- We should not be able to have presets with the same name for a particular template version. CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets (name, template_version_id); diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0674be6f1c824..0fcd9453c432e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5949,11 +5949,11 @@ WHERE w.id IN ( INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition AND b.job_status IN ('succeeded'::provisioner_job_status)) - -- The prebuilds system should never try to claim a prebuild for an inactive template version. - -- Nevertheless, this filter is here as a defensive measure: - AND b.template_version_id = t.active_version_id - AND p.current_preset_id = $3::uuid - AND p.ready + -- The prebuilds system should never try to claim a prebuild for an inactive template version. + -- Nevertheless, this filter is here as a defensive measure: + AND b.template_version_id = t.active_version_id + AND p.current_preset_id = $3::uuid + AND p.ready LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. ) RETURNING w.id, w.name @@ -5980,14 +5980,14 @@ func (q *sqlQuerier) ClaimPrebuiltWorkspace(ctx context.Context, arg ClaimPrebui const countInProgressPrebuilds = `-- name: CountInProgressPrebuilds :many SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb - INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id - -- We only need these counts for active template versions. - -- It doesn't influence whether we create or delete prebuilds - -- for inactive template versions. This is because we never create - -- prebuilds for inactive template versions, we always delete - -- running prebuilds for inactive template versions, and we ignore - -- prebuilds that are still building. - INNER JOIN templates t ON t.active_version_id = wlb.template_version_id + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + -- We only need these counts for active template versions. + -- It doesn't influence whether we create or delete prebuilds + -- for inactive template versions. This is because we never create + -- prebuilds for inactive template versions, we always delete + -- running prebuilds for inactive template versions, and we ignore + -- prebuilds that are still building. + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition ` @@ -6031,13 +6031,13 @@ func (q *sqlQuerier) CountInProgressPrebuilds(ctx context.Context) ([]CountInPro const getPrebuildMetrics = `-- name: GetPrebuildMetrics :many SELECT - t.name as template_name, - tvp.name as preset_name, + t.name as template_name, + tvp.name as preset_name, o.name as organization_name, - COUNT(*) as created_count, - COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, - COUNT(*) FILTER ( - WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. + COUNT(*) as created_count, + COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, + COUNT(*) FILTER ( + WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. ) as claimed_count FROM workspaces w INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id @@ -6094,37 +6094,38 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp - INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - INNER JOIN workspaces w ON wlb.workspace_id = w.id - INNER JOIN template_versions tv ON wlb.template_version_id = tv.id - INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN workspaces w ON wlb.workspace_id = w.id + INNER JOIN template_versions tv ON wlb.template_version_id = tv.id + INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition - AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' + AND wlb.transition = 'start'::workspace_transition + AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' ), time_sorted_builds AS ( - -- Group builds by preset, then sort each group by created_at. + -- Group builds by preset, then sort each group by created_at. SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, - ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn + ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( - -- Count failed builds per preset in the given period + -- Count failed builds per preset in the given period SELECT preset_id, COUNT(*) AS num_failed FROM filtered_builds WHERE job_status = 'failed'::provisioner_job_status AND created_at >= $1::timestamptz GROUP BY preset_id ) -SELECT tsb.template_version_id, - tsb.preset_id, - COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at)::timestamptz AS last_build_at +SELECT + tsb.template_version_id, + tsb.preset_id, + COALESCE(fc.num_failed, 0)::int AS num_failed, + MAX(tsb.created_at)::timestamptz AS last_build_at FROM time_sorted_builds tsb - LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff - AND tsb.job_status = 'failed'::provisioner_job_status - AND created_at >= $1::timestamptz + AND tsb.job_status = 'failed'::provisioner_job_status + AND created_at >= $1::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed ` @@ -6178,15 +6179,16 @@ func (q *sqlQuerier) GetPresetsBackoff(ctx context.Context, lookback time.Time) } const getRunningPrebuiltWorkspaces = `-- name: GetRunningPrebuiltWorkspaces :many -SELECT p.id, - p.name, - p.template_id, - b.template_version_id, - p.current_preset_id AS current_preset_id, - p.ready, - p.created_at +SELECT + p.id, + p.name, + p.template_id, + b.template_version_id, + p.current_preset_id AS current_preset_id, + p.ready, + p.created_at FROM workspace_prebuilds p - INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id WHERE (b.transition = 'start'::workspace_transition AND b.job_status = 'succeeded'::provisioner_job_status) ` @@ -6246,11 +6248,11 @@ SELECT t.deleted, t.deprecated != '' AS deprecated FROM templates t - INNER JOIN template_versions tv ON tv.template_id = t.id - INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id - INNER JOIN organizations o ON o.id = t.organization_id + INNER JOIN template_versions tv ON tv.template_id = t.id + INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN organizations o ON o.id = t.organization_id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND (t.id = $1::uuid OR $1 IS NULL) + AND (t.id = $1::uuid OR $1 IS NULL) ` type GetTemplatePresetsWithPrebuildsRow struct { @@ -6308,7 +6310,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa const getPresetByID = `-- name: GetPresetByID :one SELECT tvp.id, tvp.template_version_id, tvp.name, tvp.created_at, tvp.desired_instances, tvp.invalidate_after_secs, tv.template_id, tv.organization_id FROM template_version_presets tvp - INNER JOIN template_versions tv ON tvp.template_version_id = tv.id + INNER JOIN template_versions tv ON tvp.template_version_id = tv.id WHERE tvp.id = $1 ` diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index f9158dbda7e74..206f30cad5a1f 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -15,22 +15,23 @@ SELECT t.deleted, t.deprecated != '' AS deprecated FROM templates t - INNER JOIN template_versions tv ON tv.template_id = t.id - INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id - INNER JOIN organizations o ON o.id = t.organization_id + INNER JOIN template_versions tv ON tv.template_id = t.id + INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id + INNER JOIN organizations o ON o.id = t.organization_id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); + AND (t.id = sqlc.narg('template_id')::uuid OR sqlc.narg('template_id') IS NULL); -- name: GetRunningPrebuiltWorkspaces :many -SELECT p.id, - p.name, - p.template_id, - b.template_version_id, - p.current_preset_id AS current_preset_id, - p.ready, - p.created_at +SELECT + p.id, + p.name, + p.template_id, + b.template_version_id, + p.current_preset_id AS current_preset_id, + p.ready, + p.created_at FROM workspace_prebuilds p - INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id WHERE (b.transition = 'start'::workspace_transition AND b.job_status = 'succeeded'::provisioner_job_status); @@ -39,14 +40,14 @@ WHERE (b.transition = 'start'::workspace_transition -- Prebuild considered in-progress if it's in the "starting", "stopping", or "deleting" state. SELECT t.id AS template_id, wpb.template_version_id, wpb.transition, COUNT(wpb.transition)::int AS count FROM workspace_latest_builds wlb - INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id - -- We only need these counts for active template versions. - -- It doesn't influence whether we create or delete prebuilds - -- for inactive template versions. This is because we never create - -- prebuilds for inactive template versions, we always delete - -- running prebuilds for inactive template versions, and we ignore - -- prebuilds that are still building. - INNER JOIN templates t ON t.active_version_id = wlb.template_version_id + INNER JOIN workspace_prebuild_builds wpb ON wpb.id = wlb.id + -- We only need these counts for active template versions. + -- It doesn't influence whether we create or delete prebuilds + -- for inactive template versions. This is because we never create + -- prebuilds for inactive template versions, we always delete + -- running prebuilds for inactive template versions, and we ignore + -- prebuilds that are still building. + INNER JOIN templates t ON t.active_version_id = wlb.template_version_id WHERE wlb.job_status IN ('pending'::provisioner_job_status, 'running'::provisioner_job_status) GROUP BY t.id, wpb.template_version_id, wpb.transition; @@ -69,37 +70,38 @@ WITH filtered_builds AS ( -- Only select builds which are for prebuild creations SELECT wlb.template_version_id, wlb.created_at, tvp.id AS preset_id, wlb.job_status, tvp.desired_instances FROM template_version_presets tvp - INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id - INNER JOIN workspaces w ON wlb.workspace_id = w.id - INNER JOIN template_versions tv ON wlb.template_version_id = tv.id - INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id + INNER JOIN workspace_latest_builds wlb ON wlb.template_version_preset_id = tvp.id + INNER JOIN workspaces w ON wlb.workspace_id = w.id + INNER JOIN template_versions tv ON wlb.template_version_id = tv.id + INNER JOIN templates t ON tv.template_id = t.id AND t.active_version_id = tv.id WHERE tvp.desired_instances IS NOT NULL -- Consider only presets that have a prebuild configuration. - AND wlb.transition = 'start'::workspace_transition - AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' + AND wlb.transition = 'start'::workspace_transition + AND w.owner_id = 'c42fdf75-3097-471c-8c33-fb52454d81c0' ), time_sorted_builds AS ( - -- Group builds by preset, then sort each group by created_at. + -- Group builds by preset, then sort each group by created_at. SELECT fb.template_version_id, fb.created_at, fb.preset_id, fb.job_status, fb.desired_instances, - ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn + ROW_NUMBER() OVER (PARTITION BY fb.preset_id ORDER BY fb.created_at DESC) as rn FROM filtered_builds fb ), failed_count AS ( - -- Count failed builds per preset in the given period + -- Count failed builds per preset in the given period SELECT preset_id, COUNT(*) AS num_failed FROM filtered_builds WHERE job_status = 'failed'::provisioner_job_status AND created_at >= @lookback::timestamptz GROUP BY preset_id ) -SELECT tsb.template_version_id, - tsb.preset_id, - COALESCE(fc.num_failed, 0)::int AS num_failed, - MAX(tsb.created_at)::timestamptz AS last_build_at +SELECT + tsb.template_version_id, + tsb.preset_id, + COALESCE(fc.num_failed, 0)::int AS num_failed, + MAX(tsb.created_at)::timestamptz AS last_build_at FROM time_sorted_builds tsb - LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id + LEFT JOIN failed_count fc ON fc.preset_id = tsb.preset_id WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the number of desired instances; if any fail, we backoff - AND tsb.job_status = 'failed'::provisioner_job_status - AND created_at >= @lookback::timestamptz + AND tsb.job_status = 'failed'::provisioner_job_status + AND created_at >= @lookback::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; -- name: ClaimPrebuiltWorkspace :one @@ -114,24 +116,24 @@ WHERE w.id IN ( INNER JOIN templates t ON p.template_id = t.id WHERE (b.transition = 'start'::workspace_transition AND b.job_status IN ('succeeded'::provisioner_job_status)) - -- The prebuilds system should never try to claim a prebuild for an inactive template version. - -- Nevertheless, this filter is here as a defensive measure: - AND b.template_version_id = t.active_version_id - AND p.current_preset_id = @preset_id::uuid - AND p.ready + -- The prebuilds system should never try to claim a prebuild for an inactive template version. + -- Nevertheless, this filter is here as a defensive measure: + AND b.template_version_id = t.active_version_id + AND p.current_preset_id = @preset_id::uuid + AND p.ready LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. ) RETURNING w.id, w.name; -- name: GetPrebuildMetrics :many SELECT - t.name as template_name, - tvp.name as preset_name, + t.name as template_name, + tvp.name as preset_name, o.name as organization_name, - COUNT(*) as created_count, - COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, - COUNT(*) FILTER ( - WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. + COUNT(*) as created_count, + COUNT(*) FILTER (WHERE pj.job_status = 'failed'::provisioner_job_status) as failed_count, + COUNT(*) FILTER ( + WHERE w.owner_id != 'c42fdf75-3097-471c-8c33-fb52454d81c0'::uuid -- The system user responsible for prebuilds. ) as claimed_count FROM workspaces w INNER JOIN workspace_prebuild_builds wpb ON wpb.workspace_id = w.id diff --git a/coderd/database/queries/presets.sql b/coderd/database/queries/presets.sql index 5329e6d999954..526d7d0a95c3c 100644 --- a/coderd/database/queries/presets.sql +++ b/coderd/database/queries/presets.sql @@ -52,5 +52,5 @@ WHERE -- name: GetPresetByID :one SELECT tvp.*, tv.template_id, tv.organization_id FROM template_version_presets tvp - INNER JOIN template_versions tv ON tvp.template_version_id = tv.id + INNER JOIN template_versions tv ON tvp.template_version_id = tv.id WHERE tvp.id = @preset_id; From ff72a00992f2b392259af6f4409080ee72e2e2de Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 1 Apr 2025 20:44:01 -0400 Subject: [PATCH 64/72] refactor: minor refactoring --- coderd/database/queries/prebuilds.sql | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 206f30cad5a1f..53a24db5aa47a 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -1,3 +1,24 @@ +-- name: ClaimPrebuiltWorkspace :one +UPDATE workspaces w +SET owner_id = @new_user_id::uuid, + name = @new_name::text, + updated_at = NOW() +WHERE w.id IN ( + SELECT p.id + FROM workspace_prebuilds p + INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id + INNER JOIN templates t ON p.template_id = t.id + WHERE (b.transition = 'start'::workspace_transition + AND b.job_status IN ('succeeded'::provisioner_job_status)) + -- The prebuilds system should never try to claim a prebuild for an inactive template version. + -- Nevertheless, this filter is here as a defensive measure: + AND b.template_version_id = t.active_version_id + AND p.current_preset_id = @preset_id::uuid + AND p.ready + LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. +) +RETURNING w.id, w.name; + -- name: GetTemplatePresetsWithPrebuilds :many -- GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. -- It also returns the number of desired instances for each preset. @@ -104,27 +125,6 @@ WHERE tsb.rn <= tsb.desired_instances -- Fetch the last N builds, where N is the AND created_at >= @lookback::timestamptz GROUP BY tsb.template_version_id, tsb.preset_id, fc.num_failed; --- name: ClaimPrebuiltWorkspace :one -UPDATE workspaces w -SET owner_id = @new_user_id::uuid, - name = @new_name::text, - updated_at = NOW() -WHERE w.id IN ( - SELECT p.id - FROM workspace_prebuilds p - INNER JOIN workspace_latest_builds b ON b.workspace_id = p.id - INNER JOIN templates t ON p.template_id = t.id - WHERE (b.transition = 'start'::workspace_transition - AND b.job_status IN ('succeeded'::provisioner_job_status)) - -- The prebuilds system should never try to claim a prebuild for an inactive template version. - -- Nevertheless, this filter is here as a defensive measure: - AND b.template_version_id = t.active_version_id - AND p.current_preset_id = @preset_id::uuid - AND p.ready - LIMIT 1 FOR UPDATE OF p SKIP LOCKED -- Ensure that a concurrent request will not select the same prebuild. -) -RETURNING w.id, w.name; - -- name: GetPrebuildMetrics :many SELECT t.name as template_name, From a007d4adfe1fa5e4ad031da9a4e31cf404415358 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 10:46:46 -0400 Subject: [PATCH 65/72] fix: handle presets with the same tv.id and name --- .../migrations/000314_preset_prebuilds.up.sql | 13 +++++++++++ .../000291_workspace_parameter_presets.up.sql | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/coderd/database/migrations/000314_preset_prebuilds.up.sql b/coderd/database/migrations/000314_preset_prebuilds.up.sql index 392bcf741b233..a4b31a5960539 100644 --- a/coderd/database/migrations/000314_preset_prebuilds.up.sql +++ b/coderd/database/migrations/000314_preset_prebuilds.up.sql @@ -2,5 +2,18 @@ ALTER TABLE template_version_presets ADD COLUMN desired_instances INT NULL, ADD COLUMN invalidate_after_secs INT NULL DEFAULT 0; +-- Ensure that the idx_unique_preset_name index creation won't fail. +-- This is necessary because presets were released before the index was introduced, +-- so existing data might violate the uniqueness constraint. +WITH ranked AS ( + SELECT id, name, template_version_id, + ROW_NUMBER() OVER (PARTITION BY name, template_version_id ORDER BY id) AS row_num + FROM template_version_presets +) +UPDATE template_version_presets +SET name = ranked.name || '_auto_' || row_num +FROM ranked +WHERE template_version_presets.id = ranked.id AND row_num > 1; + -- We should not be able to have presets with the same name for a particular template version. CREATE UNIQUE INDEX idx_unique_preset_name ON template_version_presets (name, template_version_id); diff --git a/coderd/database/migrations/testdata/fixtures/000291_workspace_parameter_presets.up.sql b/coderd/database/migrations/testdata/fixtures/000291_workspace_parameter_presets.up.sql index 8eebf58e3f39c..296df73a587c3 100644 --- a/coderd/database/migrations/testdata/fixtures/000291_workspace_parameter_presets.up.sql +++ b/coderd/database/migrations/testdata/fixtures/000291_workspace_parameter_presets.up.sql @@ -7,4 +7,26 @@ INSERT INTO public.template_versions (id, template_id, organization_id, created_ INSERT INTO public.template_version_presets (id, template_version_id, name, created_at) VALUES ('28b42cc0-c4fe-4907-a0fe-e4d20f1e9bfe', 'af58bd62-428c-4c33-849b-d43a3be07d93', 'test', '0001-01-01 00:00:00.000000 +00:00'); +-- Add presets with the same template version ID and name +-- to ensure they're correctly handled by the 00031*_preset_prebuilds migration. +INSERT INTO public.template_version_presets ( + id, template_version_id, name, created_at +) +VALUES ( + 'c9dd1a63-f0cf-446e-8d6f-2d29d7c8e38b', + 'af58bd62-428c-4c33-849b-d43a3be07d93', + 'duplicate_name', + '0001-01-01 00:00:00.000000 +00:00' +); + +INSERT INTO public.template_version_presets ( + id, template_version_id, name, created_at +) +VALUES ( + '80f93d57-3948-487a-8990-bb011fb80a18', + 'af58bd62-428c-4c33-849b-d43a3be07d93', + 'duplicate_name', + '0001-01-01 00:00:00.000000 +00:00' +); + INSERT INTO public.template_version_preset_parameters (id, template_version_preset_id, name, value) VALUES ('ea90ccd2-5024-459e-87e4-879afd24de0f', '28b42cc0-c4fe-4907-a0fe-e4d20f1e9bfe', 'test', 'test'); From ea9c53b1a0b64f6724aaff87416065a31bc076a8 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 13:10:23 -0400 Subject: [PATCH 66/72] fix: redefine RBAC permissions for prebuilds --- coderd/database/dbauthz/dbauthz.go | 22 ++++++++++++---------- coderd/database/dbauthz/dbauthz_test.go | 10 +++++----- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 2 +- coderd/database/queries/prebuilds.sql | 2 +- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index b46f07c1630e6..40fc0c04bd571 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -372,13 +372,12 @@ var ( DisplayName: "Coder", Site: rbac.Permissions(map[string][]policy.Action{ // May use template, read template-related info, & insert template-related resources (preset prebuilds). - rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionUse}, + rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionUse, policy.ActionViewInsights}, // May CRUD workspaces, and start/stop them. rbac.ResourceWorkspace.Type: { policy.ActionCreate, policy.ActionDelete, policy.ActionRead, policy.ActionUpdate, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, }, - rbac.ResourceSystem.Type: {policy.ActionRead}, }), }, }), @@ -1185,7 +1184,7 @@ func (q *querier) CleanTailnetTunnels(ctx context.Context) error { } func (q *querier) CountInProgressPrebuilds(ctx context.Context) ([]database.CountInProgressPrebuildsRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspace.All()); err != nil { return nil, err } return q.db.CountInProgressPrebuilds(ctx) @@ -2135,7 +2134,9 @@ func (q *querier) GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUI } func (q *querier) GetPrebuildMetrics(ctx context.Context) ([]database.GetPrebuildMetricsRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + // GetPrebuildMetrics returns metrics related to prebuilt workspaces, + // such as the number of created and failed prebuilt workspaces. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspace.All()); err != nil { return nil, err } return q.db.GetPrebuildMetrics(ctx) @@ -2174,7 +2175,8 @@ func (q *querier) GetPresetParametersByTemplateVersionID(ctx context.Context, te } func (q *querier) GetPresetsBackoff(ctx context.Context, lookback time.Time) ([]database.GetPresetsBackoffRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + // GetPresetsBackoff returns a list of template version presets along with metadata such as the number of failed prebuilds. + if err := q.authorizeContext(ctx, policy.ActionViewInsights, rbac.ResourceTemplate.All()); err != nil { return nil, err } return q.db.GetPresetsBackoff(ctx, lookback) @@ -2331,7 +2333,8 @@ func (q *querier) GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Ti } func (q *querier) GetRunningPrebuiltWorkspaces(ctx context.Context) ([]database.GetRunningPrebuiltWorkspacesRow, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + // This query returns only prebuilt workspaces, but we decided to require permissions for all workspaces. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspace.All()); err != nil { return nil, err } return q.db.GetRunningPrebuiltWorkspaces(ctx) @@ -2462,10 +2465,9 @@ func (q *querier) GetTemplateParameterInsights(ctx context.Context, arg database } func (q *querier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]database.GetTemplatePresetsWithPrebuildsRow, error) { - // Although this fetches presets. It filters them by prebuilds and is only of use to the prebuild system. - // As such, we authorize this in line with other prebuild queries, not with other preset queries. - - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + // GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets and prebuilds. + // Presets and prebuilds are part of the template, so if you can access templates - you can access them as well. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate.All()); err != nil { return nil, err } return q.db.GetTemplatePresetsWithPrebuilds(ctx, templateID) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index e8a62138255c9..75870d2ec473b 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4831,28 +4831,28 @@ func (s *MethodTestSuite) TestPrebuilds() { })) s.Run("GetPrebuildMetrics", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceSystem, policy.ActionRead). + Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("CountInProgressPrebuilds", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceSystem, policy.ActionRead). + Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) { check.Args(time.Time{}). - Asserts(rbac.ResourceSystem, policy.ActionRead). + Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetRunningPrebuiltWorkspaces", s.Subtest(func(_ database.Store, check *expects) { check.Args(). - Asserts(rbac.ResourceSystem, policy.ActionRead). + Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetTemplatePresetsWithPrebuilds", s.Subtest(func(db database.Store, check *expects) { user := dbgen.User(s.T(), db, database.User{}) check.Args(uuid.NullUUID{UUID: user.ID, Valid: true}). - Asserts(rbac.ResourceSystem, policy.ActionRead). + Asserts(rbac.ResourceTemplate.All(), policy.ActionRead). ErrorsWithInMemDB(dbmem.ErrUnimplemented) })) s.Run("GetPresetByID", s.Subtest(func(db database.Store, check *expects) { diff --git a/coderd/database/querier.go b/coderd/database/querier.go index e5130ee18eadd..86235070e809d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -315,7 +315,7 @@ type sqlcQuerier interface { // created in the timeframe and return the aggregate usage counts of parameter // values. GetTemplateParameterInsights(ctx context.Context, arg GetTemplateParameterInsightsParams) ([]GetTemplateParameterInsightsRow, error) - // GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. + // GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets and prebuilds. // It also returns the number of desired instances for each preset. // If template_id is specified, only template versions associated with that template will be returned. GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0fcd9453c432e..daac12ac7f2c3 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6269,7 +6269,7 @@ type GetTemplatePresetsWithPrebuildsRow struct { Deprecated bool `db:"deprecated" json:"deprecated"` } -// GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. +// GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets and prebuilds. // It also returns the number of desired instances for each preset. // If template_id is specified, only template versions associated with that template will be returned. func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templateID uuid.NullUUID) ([]GetTemplatePresetsWithPrebuildsRow, error) { diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 53a24db5aa47a..53f5020f3607e 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -20,7 +20,7 @@ WHERE w.id IN ( RETURNING w.id, w.name; -- name: GetTemplatePresetsWithPrebuilds :many --- GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets. +-- GetTemplatePresetsWithPrebuilds retrieves template versions with configured presets and prebuilds. -- It also returns the number of desired instances for each preset. -- If template_id is specified, only template versions associated with that template will be returned. SELECT From 29e121f0c2195aec6a971d6f91d990978d4022b7 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 13:57:18 -0400 Subject: [PATCH 67/72] fix: fix migration numbers --- .../{000313_prebuilds.down.sql => 000314_prebuilds.down.sql} | 0 .../{000313_prebuilds.up.sql => 000314_prebuilds.up.sql} | 0 ...preset_prebuilds.down.sql => 000315_preset_prebuilds.down.sql} | 0 ...314_preset_prebuilds.up.sql => 000315_preset_prebuilds.up.sql} | 0 ...314_preset_prebuilds.up.sql => 000315_preset_prebuilds.up.sql} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000313_prebuilds.down.sql => 000314_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000313_prebuilds.up.sql => 000314_prebuilds.up.sql} (100%) rename coderd/database/migrations/{000314_preset_prebuilds.down.sql => 000315_preset_prebuilds.down.sql} (100%) rename coderd/database/migrations/{000314_preset_prebuilds.up.sql => 000315_preset_prebuilds.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000314_preset_prebuilds.up.sql => 000315_preset_prebuilds.up.sql} (100%) diff --git a/coderd/database/migrations/000313_prebuilds.down.sql b/coderd/database/migrations/000314_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000313_prebuilds.down.sql rename to coderd/database/migrations/000314_prebuilds.down.sql diff --git a/coderd/database/migrations/000313_prebuilds.up.sql b/coderd/database/migrations/000314_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000313_prebuilds.up.sql rename to coderd/database/migrations/000314_prebuilds.up.sql diff --git a/coderd/database/migrations/000314_preset_prebuilds.down.sql b/coderd/database/migrations/000315_preset_prebuilds.down.sql similarity index 100% rename from coderd/database/migrations/000314_preset_prebuilds.down.sql rename to coderd/database/migrations/000315_preset_prebuilds.down.sql diff --git a/coderd/database/migrations/000314_preset_prebuilds.up.sql b/coderd/database/migrations/000315_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/000314_preset_prebuilds.up.sql rename to coderd/database/migrations/000315_preset_prebuilds.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000314_preset_prebuilds.up.sql b/coderd/database/migrations/testdata/fixtures/000315_preset_prebuilds.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000314_preset_prebuilds.up.sql rename to coderd/database/migrations/testdata/fixtures/000315_preset_prebuilds.up.sql From f64754f274ab99ac619fe5d9d87af0d03a798335 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 16:12:26 -0400 Subject: [PATCH 68/72] test: fix dbmem tests --- coderd/database/dbmem/dbmem.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 8d32139bb526b..bfae69fa68b98 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -4225,7 +4225,37 @@ func (*FakeQuerier) GetPrebuildMetrics(_ context.Context) ([]database.GetPrebuil } func (q *FakeQuerier) GetPresetByID(ctx context.Context, presetID uuid.UUID) (database.GetPresetByIDRow, error) { - return database.GetPresetByIDRow{}, ErrUnimplemented + q.mutex.RLock() + defer q.mutex.RUnlock() + + empty := database.GetPresetByIDRow{} + + // Create an index for faster lookup + versionMap := make(map[uuid.UUID]database.TemplateVersionTable) + for _, tv := range q.templateVersions { + versionMap[tv.ID] = tv + } + + for _, preset := range q.presets { + if preset.ID == presetID { + tv, ok := versionMap[preset.TemplateVersionID] + if !ok { + return empty, fmt.Errorf("template version %v does not exist", preset.TemplateVersionID) + } + return database.GetPresetByIDRow{ + ID: preset.ID, + TemplateVersionID: preset.TemplateVersionID, + Name: preset.Name, + CreatedAt: preset.CreatedAt, + DesiredInstances: preset.DesiredInstances, + InvalidateAfterSecs: preset.InvalidateAfterSecs, + TemplateID: tv.TemplateID, + OrganizationID: tv.OrganizationID, + }, nil + } + } + + return empty, fmt.Errorf("preset %v does not exist", presetID) } func (q *FakeQuerier) GetPresetByWorkspaceBuildID(_ context.Context, workspaceBuildID uuid.UUID) (database.TemplateVersionPreset, error) { From 7e02397216e15b194c3ec16791447e95184bf2ea Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 16:34:41 -0400 Subject: [PATCH 69/72] test: fix dbmem tests --- coderd/database/dbauthz/dbauthz_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index fff165c391f3b..0fe17f886b1b2 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4914,7 +4914,6 @@ func (s *MethodTestSuite) TestPrebuilds() { }) check.Args(preset.ID). Asserts(template, policy.ActionRead). - ErrorsWithInMemDB(dbmem.ErrUnimplemented). Returns(database.GetPresetByIDRow{ ID: preset.ID, TemplateVersionID: preset.TemplateVersionID, From 55d9827f96c51b6c88f7d165d669e24782af03fc Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 16:45:24 -0400 Subject: [PATCH 70/72] fix: linter --- coderd/database/querier_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 7a7ead26311c2..05cd2af08dc0f 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3605,20 +3605,20 @@ func createTemplate(t *testing.T, db database.Store, orgID uuid.UUID, userID uui } type tmplVersionOpts struct { - DesiredInstances int + DesiredInstances int32 } func createTmplVersionAndPreset( t *testing.T, db database.Store, tmpl database.Template, - versionId uuid.UUID, + versionID uuid.UUID, now time.Time, opts *tmplVersionOpts, ) templateVersionWithPreset { // Create template version with corresponding preset and preset prebuild tmplVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{ - ID: versionId, + ID: versionID, TemplateID: uuid.NullUUID{ UUID: tmpl.ID, Valid: true, @@ -3628,7 +3628,7 @@ func createTmplVersionAndPreset( UpdatedAt: now, CreatedBy: tmpl.CreatedBy, }) - desiredInstances := 1 + desiredInstances := int32(1) if opts != nil { desiredInstances = opts.DesiredInstances } From 866454b92f9ee7af1135fbe710b386678cde3bbd Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 16:46:54 -0400 Subject: [PATCH 71/72] fix: linter --- coderd/database/querier_test.go | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 05cd2af08dc0f..26d7e4a7434bf 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3655,8 +3655,8 @@ type createPrebuiltWorkspaceOpts struct { } func createPrebuiltWorkspace( - t *testing.T, ctx context.Context, + t *testing.T, db database.Store, tmpl database.Template, extTmplVersion templateVersionWithPreset, @@ -3832,7 +3832,7 @@ func TestWorkspacePrebuildsView(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ readyAgents: tc.readyAgents, notReadyAgents: tc.notReadyAgents, }) @@ -3878,7 +3878,7 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3906,13 +3906,13 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3940,16 +3940,16 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl := createTemplate(t, db, orgID, userID) tmplV1 := createTmplVersionAndPreset(t, db, tmpl, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) // Active Version tmplV2 := createTmplVersionAndPreset(t, db, tmpl, tmpl.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl, tmplV2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -3977,13 +3977,13 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersionAndPreset(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4019,33 +4019,33 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl2 := createTemplate(t, db, orgID, userID) tmpl2V1 := createTmplVersionAndPreset(t, db, tmpl2, tmpl2.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl2, tmpl2V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl3 := createTemplate(t, db, orgID, userID) tmpl3V1 := createTmplVersionAndPreset(t, db, tmpl3, uuid.New(), now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl3, tmpl3V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) tmpl3V2 := createTmplVersionAndPreset(t, db, tmpl3, tmpl3.ActiveVersionID, now, nil) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) - createPrebuiltWorkspace(t, ctx, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl3, tmpl3V2, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, }) @@ -4109,9 +4109,9 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1 := createTemplate(t, db, orgID, userID) tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, nil) successfulJobOpts := createPrebuiltWorkspaceOpts{} - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4142,8 +4142,8 @@ func TestGetPresetsBackoff(t *testing.T) { failedJob: false, createdAt: now.Add(-1 * time.Minute), } - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &failedJobOpts) + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &successfulJobOpts) backoffs, err := db.GetPresetsBackoff(ctx, now.Add(-time.Hour)) require.NoError(t, err) @@ -4166,19 +4166,19 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4204,15 +4204,15 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4246,23 +4246,23 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 3, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), // within lookback period - counted as failed job }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), // within lookback period - counted as failed job }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-2 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: false, createdAt: now.Add(-1 * time.Minute), }) @@ -4296,27 +4296,27 @@ func TestGetPresetsBackoff(t *testing.T) { tmpl1V1 := createTmplVersionAndPreset(t, db, tmpl1, tmpl1.ActiveVersionID, now, &tmplVersionOpts{ DesiredInstances: 6, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-4 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-0 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-3 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-1 * time.Minute), }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-2 * time.Minute), }) @@ -4353,7 +4353,7 @@ func TestGetPresetsBackoff(t *testing.T) { DesiredInstances: 1, }) - createPrebuiltWorkspace(t, ctx, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ + createPrebuiltWorkspace(ctx, t, db, tmpl1, tmpl1V1, orgID, now, &createPrebuiltWorkspaceOpts{ failedJob: true, createdAt: now.Add(-lookbackPeriod - time.Minute), // earlier than lookback period - skipped }) From 4fa959a9f81315f609364ec078fb7cb2c304424e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 2 Apr 2025 16:53:45 -0400 Subject: [PATCH 72/72] fix: linter --- coderd/database/querier_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 26d7e4a7434bf..4a2edb4451c34 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -3636,7 +3636,7 @@ func createTmplVersionAndPreset( TemplateVersionID: tmplVersion.ID, Name: "preset", DesiredInstances: sql.NullInt32{ - Int32: int32(desiredInstances), + Int32: desiredInstances, Valid: true, }, })