feat: notify template admins when prebuild claim results in resource …

…replacement(s) Signed-off-by: Danny Kopping <[email protected]>
coder · dannykopping · May 14, 2025 · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025
commit 61ef61a213e516bf0b525ba10669c8c8d8ccd55f
diff --git a/coderd/database/migrations/000318_resource_replacements_notification.down.sql b/coderd/database/migrations/000318_resource_replacements_notification.down.sql
@@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';
diff --git a/coderd/database/migrations/000318_resource_replacements_notification.up.sql b/coderd/database/migrations/000318_resource_replacements_notification.up.sql
@@ -0,0 +1,27 @@
+INSERT INTO notification_templates
+	(id, name, title_template, body_template, "group", actions)
+VALUES ('89d9745a-816e-4695-a17f-3d0a229e2b8d',
+		'Prebuilt Workspace Resource Replaced',
+		E'There might be a problem with a recently claimed prebuilt workspace',
+		$$
+Workspace **{{.Labels.workspace}}** was claimed from a prebuilt workspace by **{{.Labels.claimant}}**.
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+{{range $resource, $paths := .Data.replacements -}}
+- _{{ $resource }}_  was replaced due to changes to _{{ $paths }}_
+{{end}}
+
+When Terraform must change an immutable attribute, it replaces the entire resource.
+If you’re using prebuilds to speed up provisioning, unexpected replacements will slow down
+workspace startup—even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/TODO).
+$$,
+		'Workspace Events',
+		'[
+		{
+			"label": "View workspace build",
+			"url": "{{base_url}}/@{{.Labels.claimant}}/{{.Labels.workspace}}/builds/{{.Labels.workspace_build_num}}"
+		}
+	]'::jsonb);
diff --git a/coderd/notifications/events.go b/coderd/notifications/events.go
@@ -18,6 +18,7 @@ var (
 	TemplateWorkspaceManualBuildFailed = uuid.MustParse("2faeee0f-26cb-4e96-821c-85ccb9f71513")
 	TemplateWorkspaceOutOfMemory       = uuid.MustParse("a9d027b4-ac49-4fb1-9f6d-45af15f64e7a")
 	TemplateWorkspaceOutOfDisk         = uuid.MustParse("f047f6a3-5713-40f7-85aa-0394cce9fa3a")
+	TemplateWorkspaceResourceReplaced  = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
 )
 
 // Account-related events.

diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -1728,6 +1728,10 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
 			})
 		}
 
+		if resourceReplacements := completed.GetWorkspaceBuild().GetResourceReplacements(); len(resourceReplacements) > 0 {
+			s.notifyPrebuiltWorkspaceResourceReplacement(ctx, workspace, workspaceBuild, input.PrebuildClaimedByUser, resourceReplacements)
+		}
+
 		msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 			Kind:        wspubsub.WorkspaceEventKindStateChange,
 			WorkspaceID: workspace.ID,
@@ -1836,6 +1840,67 @@ func (s *server) notifyWorkspaceDeleted(ctx context.Context, workspace database.
 	}
 }
 
+func (s *server) notifyPrebuiltWorkspaceResourceReplacement(ctx context.Context, workspace database.Workspace, build database.WorkspaceBuild, claimantID uuid.UUID, replacements []*sdkproto.ResourceReplacements) {
+	if claimantID == uuid.Nil {
+		// This is not a prebuild claim.
+		return
+	}
+
+	claimant, err := s.Database.GetUserByID(ctx, claimantID)
+	if err != nil {
+		s.Logger.Warn(ctx, "failed to find claimant by ID, cannot send prebuilt workspace resource replacement notification",
+			slog.F("claimant_id", claimantID.String()), slog.Error(err))
+		return
+	}
+
+	templateAdmins, err := findTemplateAdmins(ctx, s.Database)
+	if err != nil {
+		s.Logger.Warn(ctx, "failed to find template admins, cannot send prebuilt workspace resource replacement notification",
+			slog.F("claimant_id", claimantID.String()), slog.Error(err))
+		return
+	}
+
+	repls := make(map[string]string, len(replacements))
+	for _, repl := range replacements {
+		repls[repl.GetResource()] = strings.Join(repl.GetPaths(), ", ")
+	}
+
+	for _, templateAdmin := range templateAdmins {
+		if _, err := s.NotificationsEnqueuer.EnqueueWithData(ctx, templateAdmin.ID, notifications.TemplateWorkspaceResourceReplaced,
+			map[string]string{
+				"workspace":           workspace.Name,
+				"workspace_build_num": fmt.Sprintf("%d", build.BuildNumber),
+				"claimant":            claimant.Username,
+			},
+			map[string]any{
+				"replacements": repls,
+			}, "provisionerdserver",
+			// Associate this notification with all the related entities.
+			workspace.ID, workspace.OwnerID, workspace.TemplateID, workspace.OrganizationID,
+		); err != nil {
+			s.Logger.Warn(ctx, "failed to notify of prebuilt workspace resource replacement", slog.Error(err))
+			break
+		}
+	}
+}
+
+// findTemplateAdmins fetches all users with template admin permission, including owners.
+func findTemplateAdmins(ctx context.Context, store database.Store) ([]database.GetUsersRow, error) {
+	owners, err := store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleOwner},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("get owners: %w", err)
+	}
+	templateAdmins, err := store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleTemplateAdmin},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("get template admins: %w", err)
+	}
+	return append(owners, templateAdmins...), nil
+}
+
 func (s *server) startTrace(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
 	return s.Tracer.Start(ctx, name, append(opts, trace.WithAttributes(
 		semconv.ServiceNameKey.String("coderd.provisionerd"),

diff --git a/provisioner/terraform/executor.go b/provisioner/terraform/executor.go
@@ -312,12 +312,21 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 	// the point of prebuilding if the expensive resource is replaced once claimed!
 	var (
 		isPrebuildClaimAttempt = !destroy && metadata.PrebuildClaimForUserId != ""
+		reps                   []*proto.ResourceReplacements
 	)
 	if count := len(replacements); count > 0 && isPrebuildClaimAttempt {
 		// TODO(dannyk): we should log drift always (not just during prebuild claim attempts); we're validating that this output
 		//				 will not be overwhelming for end-users, but it'll certainly be super valuable for template admins
 		//				 to diagnose this resource replacement issue, at least.
 		e.logDrift(ctx, killCtx, planfilePath, logr)
+
+		reps = make([]*proto.ResourceReplacements, 0, len(replacements))
+		for n, p := range replacements {
+			reps = append(reps, &proto.ResourceReplacements{
+				Resource: n,
+				Paths:    p,
+			})
+		}
 	}
 
 	return &proto.PlanComplete{
@@ -327,6 +336,7 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 		Timings:               append(e.timings.aggregate(), graphTimings.aggregate()...),
 		Presets:               state.Presets,
 		Plan:                  plan,
+		ResourceReplacements:  reps,
 	}, nil
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';