coder · dannykopping · May 14, 2025 · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -1793,6 +1793,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 			Clock:               api.Clock,
 		},
 		api.NotificationsEnqueuer,
+		&api.PrebuildsReconciler,
 	)
 	if err != nil {
 		return nil, err

diff --git a/coderd/database/migrations/000322_resource_replacements_notification.down.sql b/coderd/database/migrations/000322_resource_replacements_notification.down.sql
@@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';
diff --git a/coderd/database/migrations/000322_resource_replacements_notification.up.sql b/coderd/database/migrations/000322_resource_replacements_notification.up.sql
@@ -0,0 +1,34 @@
+INSERT INTO notification_templates
+	(id, name, title_template, body_template, "group", actions)
+VALUES ('89d9745a-816e-4695-a17f-3d0a229e2b8d',
+		'Prebuilt Workspace Resource Replaced',
+		E'There might be a problem with a recently claimed prebuilt workspace',
+		$$
+Workspace **{{.Labels.workspace}}** was claimed from a prebuilt workspace by **{{.Labels.claimant}}**.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+{{range $resource, $paths := .Data.replacements -}}
+- _{{ $resource }}_  was replaced due to changes to _{{ $paths }}_
+{{end}}
+
+When Terraform must change an immutable attribute, it replaces the entire resource.
+If you’re using prebuilds to speed up provisioning, unexpected replacements will slow down
+workspace startup—even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the **{{.Labels.preset}}** preset.
+$$,
+		'Template Events',
+		'[
+		{
+			"label": "View workspace build",
+			"url": "{{base_url}}/@{{.Labels.claimant}}/{{.Labels.workspace}}/builds/{{.Labels.workspace_build_num}}"
+		},
+		{
+			"label": "View template version",
+			"url": "{{base_url}}/templates/{{.Labels.org}}/{{.Labels.template}}/versions/{{.Labels.template_version}}"
+		}
+	]'::jsonb);
diff --git a/coderd/notifications/enqueuer.go b/coderd/notifications/enqueuer.go
@@ -50,7 +50,7 @@ type StoreEnqueuer struct {
 }
 
 // NewStoreEnqueuer creates an Enqueuer implementation which can persist notification messages in the store.
-func NewStoreEnqueuer(cfg codersdk.NotificationsConfig, store Store, helpers template.FuncMap, log slog.Logger, clock quartz.Clock) (*StoreEnqueuer, error) {
+func NewStoreEnqueuer(cfg codersdk.NotificationsConfig, store Store, helpers template.FuncMap, log slog.Logger, clock quartz.Clock) (Enqueuer, error) {
 	var method database.NotificationMethod
 	// TODO(DanielleMaywood):
 	// Currently we do not want to allow setting `inbox` as the default notification method.
@@ -203,7 +203,7 @@ func (s *StoreEnqueuer) buildPayload(metadata database.FetchNewMessageMetadataRo
 type NoopEnqueuer struct{}
 
 // NewNoopEnqueuer builds a NoopEnqueuer which is used to fulfill the contract for enqueuing notifications, if ExperimentNotifications is not set.
-func NewNoopEnqueuer() *NoopEnqueuer {
+func NewNoopEnqueuer() Enqueuer {
 	return &NoopEnqueuer{}
 }
 

diff --git a/coderd/notifications/events.go b/coderd/notifications/events.go
@@ -39,6 +39,7 @@ var (
 	TemplateTemplateDeprecated = uuid.MustParse("f40fae84-55a2-42cd-99fa-b41c1ca64894")
 
 	TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")
+	TemplateWorkspaceResourceReplaced   = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
 )
 
 // Notification-related events.

diff --git a/coderd/notifications/notifications_test.go b/coderd/notifications/notifications_test.go
@@ -35,6 +35,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/quartz"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
@@ -48,8 +51,6 @@ import (
 	"github.com/coder/coder/v2/coderd/util/syncmap"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
-	"github.com/coder/serpent"
 )
 
 // updateGoldenFiles is a flag that can be set to update golden files.
@@ -1226,6 +1227,29 @@ func TestNotificationTemplates_Golden(t *testing.T) {
 				Labels:       map[string]string{},
 			},
 		},
+		{
+			name: "TemplateWorkspaceResourceReplaced",
+			id:   notifications.TemplateWorkspaceResourceReplaced,
+			payload: types.MessagePayload{
+				UserName:     "Bobby",
+				UserEmail:    "bobby@coder.com",
+				UserUsername: "bobby",
+				Labels: map[string]string{
+					"org":                 "cern",
+					"workspace":           "my-workspace",
+					"workspace_build_num": "2",
+					"template":            "docker",
+					"template_version":    "angry_torvalds",
+					"preset":              "particle-accelerator",
+					"claimant":            "prebuilds-claimer",
+				},
+				Data: map[string]any{
+					"replacements": map[string]string{
+						"docker_container[0]": "env, hostname",
+					},
+				},
+			},
+		},
 	}
 
 	// We must have a test case for every notification_template. This is enforced below:

diff --git a/coderd/notifications/notificationstest/fake_enqueuer.go b/coderd/notifications/notificationstest/fake_enqueuer.go
@@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 )
@@ -19,6 +20,12 @@ type FakeEnqueuer struct {
 	sent       []*FakeNotification
 }
 
+var _ notifications.Enqueuer = &FakeEnqueuer{}
+
+func NewFakeEnqueuer() notifications.Enqueuer {
+	return &FakeEnqueuer{}
+}
+
 type FakeNotification struct {
 	UserID, TemplateID uuid.UUID
 	Labels             map[string]string

diff --git a/...ifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden b/...ifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden
@@ -0,0 +1,131 @@
+From: system@coder.com
+To: bobby@coder.com
+Subject: There might be a problem with a recently claimed prebuilt workspace
+Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
+Date: Fri, 11 Oct 2024 09:03:06 +0000
+Content-Type: multipart/alternative;  boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+MIME-Version: 1.0
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=UTF-8
+
+Hi Bobby,
+
+Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-c=
+laimer.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+docker_container[0]  was replaced due to changes to env, hostname
+
+When Terraform must change an immutable attribute, it replaces the entire r=
+esource.
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down
+workspace startup=E2=80=94even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see th=
+is guide (https://coder.com/docs/admin/templates/extending-templates/prebui=
+lt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the particle-accelerator preset.
+
+
+View workspace build: http://test.com/@prebuilds-claimer/my-workspace/build=
+s/2
+
+View template version: http://test.com/templates/cern/docker/versions/angry=
+_torvalds
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=UTF-8
+
+<!doctype html>
+<html lang=3D"en">
+  <head>
+    <meta charset=3D"UTF-8" />
+    <meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
+=3D1.0" />
+    <title>There might be a problem with a recently claimed prebuilt worksp=
+ace</title>
+  </head>
+  <body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
+ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
+l', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; color: #020617=
+; background: #f8fafc;">
+    <div style=3D"max-width: 600px; margin: 20px auto; padding: 60px; borde=
+r: 1px solid #e2e8f0; border-radius: 8px; background-color: #fff; text-alig=
+n: left; font-size: 14px; line-height: 1.5;">
+      <div style=3D"text-align: center;">
+        <img src=3D"https://coder.com/coder-logo-horizontal.png" alt=3D"Cod=
+er Logo" style=3D"height: 40px;" />
+      </div>
+      <h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
+argin: 8px 0 32px; line-height: 1.5;">
+        There might be a problem with a recently claimed prebuilt workspace
+      </h1>
+      <div style=3D"line-height: 1.5;">
+        <p>Hi Bobby,</p>
+        <p>Workspace <strong>my-workspace</strong> was claimed from a prebu=
+ilt workspace by <strong>prebuilds-claimer</strong>.</p>
+
+<p>During the claim, Terraform destroyed and recreated the following resour=
+ces<br>
+because one or more immutable attributes changed:</p>
+
+<ul>
+<li>_docker<em>container[0]</em>  was replaced due to changes to <em>env, h=
+ostname</em><br>
+</li>
+</ul>
+
+<p>When Terraform must change an immutable attribute, it replaces the entir=
+e resource.<br>
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down<br>
+workspace startup=E2=80=94even when claiming a prebuilt environment.</p>
+
+<p>For tips on preventing replacements and improving claim performance, see=
+ <a href=3D"https://coder.com/docs/admin/templates/extending-templates/preb=
+uilt-workspaces#preventing-resource-replacement">this guide</a>.</p>
+
+<p>NOTE: this prebuilt workspace used the <strong>particle-accelerator</str=
+ong> preset.</p>
+      </div>
+      <div style=3D"text-align: center; margin-top: 32px;">
+       =20
+        <a href=3D"http://test.com/@prebuilds-claimer/my-workspace/builds/2=
+" style=3D"display: inline-block; padding: 13px 24px; background-color: #02=
+0617; color: #f8fafc; text-decoration: none; border-radius: 8px; margin: 0 =
+4px;">
+          View workspace build
+        </a>
+       =20
+        <a href=3D"http://test.com/templates/cern/docker/versions/angry_tor=
+valds" style=3D"display: inline-block; padding: 13px 24px; background-color=
+: #020617; color: #f8fafc; text-decoration: none; border-radius: 8px; margi=
+n: 0 4px;">
+          View template version
+        </a>
+       =20
+      </div>
+      <div style=3D"border-top: 1px solid #e2e8f0; color: #475569; font-siz=
+e: 12px; margin-top: 64px; padding-top: 24px; line-height: 1.6;">
+        <p>&copy;&nbsp;2024&nbsp;Coder. All rights reserved&nbsp;-&nbsp;<a =
+href=3D"http://test.com" style=3D"color: #2563eb; text-decoration: none;">h=
+ttp://test.com</a></p>
+        <p><a href=3D"http://test.com/settings/notifications" style=3D"colo=
+r: #2563eb; text-decoration: none;">Click here to manage your notification =
+settings</a></p>
+        <p><a href=3D"http://test.com/settings/notifications?disabled=3D89d=
+9745a-816e-4695-a17f-3d0a229e2b8d" style=3D"color: #2563eb; text-decoration=
+: none;">Stop receiving emails like this</a></p>
+      </div>
+    </div>
+  </body>
+</html>
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4--
diff --git a/...cations/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden b/...cations/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden
@@ -0,0 +1,42 @@
+{
+  "_version": "1.1",
+  "msg_id": "00000000-0000-0000-0000-000000000000",
+  "payload": {
+    "_version": "1.2",
+    "notification_name": "Prebuilt Workspace Resource Replaced",
+    "notification_template_id": "00000000-0000-0000-0000-000000000000",
+    "user_id": "00000000-0000-0000-0000-000000000000",
+    "user_email": "bobby@coder.com",
+    "user_name": "Bobby",
+    "user_username": "bobby",
+    "actions": [
+      {
+        "label": "View workspace build",
+        "url": "http://test.com/@prebuilds-claimer/my-workspace/builds/2"
+      },
+      {
+        "label": "View template version",
+        "url": "http://test.com/templates/cern/docker/versions/angry_torvalds"
+      }
+    ],
+    "labels": {
+      "claimant": "prebuilds-claimer",
+      "org": "cern",
+      "preset": "particle-accelerator",
+      "template": "docker",
+      "template_version": "angry_torvalds",
+      "workspace": "my-workspace",
+      "workspace_build_num": "2"
+    },
+    "data": {
+      "replacements": {
+        "docker_container[0]": "env, hostname"
+      }
+    },
+    "targets": null
+  },
+  "title": "There might be a problem with a recently claimed prebuilt workspace",
+  "title_markdown": "There might be a problem with a recently claimed prebuilt workspace",
+  "body": "Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-claimer.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\ndocker_container[0]  was replaced due to changes to env, hostname\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see this guide (https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: t
1241
his prebuilt workspace used the particle-accelerator preset.",
+  "body_markdown": "\nWorkspace **my-workspace** was claimed from a prebuilt workspace by **prebuilds-claimer**.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\n- _docker_container[0]_  was replaced due to changes to _env, hostname_\n\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: this prebuilt workspace used the **particle-accelerator** preset.\n"
+}
diff --git a/coderd/prebuilds/api.go b/coderd/prebuilds/api.go
@@ -7,6 +7,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )
 
 var (
@@ -27,6 +28,11 @@ type ReconciliationOrchestrator interface {
 	// Stop gracefully shuts down the orchestrator with the given cause.
 	// The cause is used for logging and error reporting.
 	Stop(ctx context.Context, cause error)
+
+	// TrackResourceReplacement handles a pathological situation whereby a terraform resource is replaced due to drift,
+	// which can obviate the whole point of pre-provisioning a prebuilt workspace.
+	// See more detail at https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement.
+	TrackResourceReplacement(ctx context.Context, workspaceID, buildID uuid.UUID, replacements []*sdkproto.ResourceReplacement)
 }
 
 type Reconciler interface {

diff --git a/coderd/prebuilds/noop.go b/coderd/prebuilds/noop.go
@@ -6,12 +6,15 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )
 
 type NoopReconciler struct{}
 
-func (NoopReconciler) Run(context.Context)                {}
-func (NoopReconciler) Stop(context.Context, error)        {}
+func (NoopReconciler) Run(context.Context)         {}
+func (NoopReconciler) Stop(context.Context, error) {}
+func (NoopReconciler) TrackResourceReplacement(context.Context, uuid.UUID, uuid.UUID, []*sdkproto.ResourceReplacement) {
+}
 func (NoopReconciler) ReconcileAll(context.Context) error { return nil }
 func (NoopReconciler) SnapshotState(context.Context, database.Store) (*GlobalSnapshot, error) {
 	return &GlobalSnapshot{}, nil
Original file line number 8000	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';