Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
c09c9b9
WIP: agent reinitialization
SasSwart Apr 21, 2025
476fe71
fix assignment to nil map
SasSwart Apr 21, 2025
8c8bca6
fix: ensure prebuilt workspace agent tokens are reused when a prebuil…
SasSwart Apr 23, 2025
7ce4eea
test agent reinitialization
SasSwart Apr 24, 2025
52ac64e
remove defunct metric
SasSwart Apr 24, 2025
362db7c
Remove todo
SasSwart Apr 25, 2025
dcc7379
test that we trigger workspace agent reinitialization under the right…
SasSwart Apr 28, 2025
ff66b3f
slight improvements to a test
SasSwart Apr 28, 2025
efff5d9
review notes to improve legibility
SasSwart Apr 28, 2025
cebd5db
add an integration test for prebuilt workspace agent reinitialization
SasSwart Apr 29, 2025
2679138
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart Apr 29, 2025
9feebef
enable the premium license in a prebuilds integration test
SasSwart Apr 29, 2025
b117b5c
encapsulate WaitForReinitLoop for easier testing
SasSwart Apr 30, 2025
a22b414
introduce unit testable abstraction layers
SasSwart Apr 30, 2025
9bbd2c7
test workspace claim pubsub
SasSwart May 1, 2025
5804201
add tests for agent reinitialization
SasSwart May 1, 2025
7e8dcee
review notes
SasSwart May 1, 2025
725f97b
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 1, 2025
a9b1567
make fmt lint
SasSwart May 1, 2025
21ee970
remove go mod replace
SasSwart May 1, 2025
e54d7e7
remove defunct logging
SasSwart May 1, 2025
2799858
update dependency on terraform-provider-coder
SasSwart May 2, 2025
1d93003
update dependency on terraform-provider-coder
SasSwart May 2, 2025
763fc12
go mod tidy
SasSwart May 2, 2025
0f879c7
make -B gen
SasSwart May 2, 2025
61784c9
dont require ids to InsertPresetParameters
SasSwart May 2, 2025
604eb27
dont require ids to InsertPresetParameters
SasSwart May 2, 2025
bf4d2cf
fix: set the running agent token
dannykopping May 2, 2025
38b4f0d
fix: use http client without timeout like we do in connectRPCVersion
dannykopping May 5, 2025
20df538
review notes
SasSwart May 6, 2025
4bb3b68
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 7, 2025
83972db
bump provisionerd proto version
SasSwart May 7, 2025
146b158
fix: fetch the previous agent when we need its token for prebuilt wor…
SasSwart May 12, 2025
5eb16cd
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 12, 2025
730d803
make -B lint
SasSwart May 12, 2025
150adc0
Test GetWorkspaceAgentsByBuildID
SasSwart May 12, 2025
b4ecf10
Rename GetWorkspaceAgentsByWorkspaceAndBuildNumber
SasSwart May 12, 2025
3fa3edf
make gen
SasSwart May 12, 2025
7e45919
fix a race condition
SasSwart May 12, 2025
a632508
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 12, 2025
72125ec
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 13, 2025
b65eea7
fix provisionerdserver test for prebuild claims
SasSwart May 13, 2025
e1339f3
fix race conditions
SasSwart May 13, 2025
c1a8ba6
Merge remote-tracking branch 'origin/main' into jjs/prebuilds-agent-r…
SasSwart May 13, 2025
5363dcc
Make TestReinitializeAgent more robust
SasSwart May 13, 2025
7ad9b6d
fix tests
SasSwart May 14, 2025
394571d
make -B gen
SasSwart May 14, 2025
890747b
remove a potential race in reinitialization testing in TestCompleteJob
SasSwart May 14, 2025
b3870db
fix a potential race in TestReinit
SasSwart May 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make TestReinitializeAgent more robust
Minor code quality enhancements
  • Loading branch information
SasSwart committed May 13, 2025
commit 5363dcc078c6dd2c6e83d58c42279a8fde9d9997
2 changes: 1 addition & 1 deletion cli/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ func (r *RootCmd) workspaceAgent() *serpent.Command {
mustExit = true
case event := <-reinitEvents:
logger.Info(ctx, "agent received instruction to reinitialize",
slog.F("user_id", event.UserID), slog.F("workspace_id", event.WorkspaceID), slog.F("reason", event.Reason))
slog.F("workspace_id", event.WorkspaceID), slog.F("reason", event.Reason))
}

lastErr = agnt.Close()
Expand Down
12 changes: 3 additions & 9 deletions coderd/prebuilds/claim.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type PubsubWorkspaceClaimPublisher struct {

func (p PubsubWorkspaceClaimPublisher) PublishWorkspaceClaim(claim agentsdk.ReinitializationEvent) error {
channel := agentsdk.PrebuildClaimedChannel(claim.WorkspaceID)
if err := p.ps.Publish(channel, []byte(claim.UserID.String())); err != nil {
if err := p.ps.Publish(channel, []byte(claim.Reason)); err != nil {
return xerrors.Errorf("failed to trigger prebuilt workspace agent reinitialization: %w", err)
}
return nil
Expand All @@ -48,16 +48,10 @@ func (p PubsubWorkspaceClaimListener) ListenForWorkspaceClaims(ctx context.Conte
default:
}

cancelSub, err := p.ps.Subscribe(agentsdk.PrebuildClaimedChannel(workspaceID), func(inner context.Context, id []byte) {
claimantID, err := uuid.ParseBytes(id)
if err != nil {
p.logger.Error(ctx, "invalid prebuild claimed channel payload", slog.F("input", string(id)))
return
}
cancelSub, err := p.ps.Subscribe(agentsdk.PrebuildClaimedChannel(workspaceID), func(inner context.Context, reason []byte) {
claim := agentsdk.ReinitializationEvent{
UserID: claimantID,
WorkspaceID: workspaceID,
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
Reason: agentsdk.ReinitializationReason(reason),
}

select {
Expand Down
27 changes: 10 additions & 17 deletions coderd/prebuilds/claim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ func TestPubsubWorkspaceClaimPublisher(t *testing.T) {
logger := testutil.Logger(t)
ps := pubsub.NewInMemory()
workspaceID := uuid.New()
userID := uuid.New()
reinitEvents := make(chan agentsdk.ReinitializationEvent, 1)
publisher := prebuilds.NewPubsubWorkspaceClaimPublisher(ps)
listener := prebuilds.NewPubsubWorkspaceClaimListener(ps, logger)
Expand All @@ -35,15 +34,15 @@ func TestPubsubWorkspaceClaimPublisher(t *testing.T) {
defer cancel()

claim := agentsdk.ReinitializationEvent{
UserID: userID,
WorkspaceID: workspaceID,
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
}
err = publisher.PublishWorkspaceClaim(claim)
require.NoError(t, err)

gotUserID := testutil.RequireReceive(testutil.Context(t, testutil.WaitShort), t, reinitEvents)
require.Equal(t, userID, gotUserID.UserID)
gotEvent := testutil.RequireReceive(ctx, t, reinitEvents)
require.Equal(t, workspaceID, gotEvent.WorkspaceID)
require.Equal(t, claim.Reason, gotEvent.Reason)
})

t.Run("fail to publish claim", func(t *testing.T) {
Expand All @@ -53,7 +52,6 @@ func TestPubsubWorkspaceClaimPublisher(t *testing.T) {

publisher := prebuilds.NewPubsubWorkspaceClaimPublisher(ps)
claim := agentsdk.ReinitializationEvent{
UserID: uuid.New(),
WorkspaceID: uuid.New(),
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
}
Expand All @@ -74,26 +72,21 @@ func TestPubsubWorkspaceClaimListener(t *testing.T) {
claims := make(chan agentsdk.ReinitializationEvent, 1) // Buffer to avoid messing with goroutines in the rest of the test

workspaceID := uuid.New()
userID := uuid.New()
cancelFunc, err := listener.ListenForWorkspaceClaims(context.Background(), workspaceID, claims)
require.NoError(t, err)
defer cancelFunc()

// Publish a claim
channel := agentsdk.PrebuildClaimedChannel(workspaceID)
err = ps.Publish(channel, []byte(userID.String()))
reason := agentsdk.ReinitializeReasonPrebuildClaimed
err = ps.Publish(channel, []byte(reason))
require.NoError(t, err)

// Verify we receive the claim
select {
case claim, ok := <-claims:
require.True(t, ok, "received on a closed channel")
require.Equal(t, userID, claim.UserID)
require.Equal(t, workspaceID, claim.WorkspaceID)
require.Equal(t, agentsdk.ReinitializeReasonPrebuildClaimed, claim.Reason)
case <-time.After(testutil.WaitSuperLong): // TODO: revert to waitshort
t.Fatal("timeout waiting for claim")
}
ctx := testutil.Context(t, testutil.WaitShort)
claim := testutil.RequireReceive(ctx, t, claims)
require.Equal(t, workspaceID, claim.WorkspaceID)
require.Equal(t, reason, claim.Reason)
})

t.Run("ignores claim events for other workspaces", func(t *testing.T) {
Comment thread
SasSwart marked this conversation as resolved.
Expand All @@ -111,7 +104,7 @@ func TestPubsubWorkspaceClaimListener(t *testing.T) {

// Publish a claim for a different workspace
channel := agentsdk.PrebuildClaimedChannel(otherWorkspaceID)
err = ps.Publish(channel, []byte(uuid.New().String()))
err = ps.Publish(channel, []byte(agentsdk.ReinitializeReasonPrebuildClaimed))
require.NoError(t, err)

// Verify we don't receive the claim
Expand Down
50 changes: 32 additions & 18 deletions coderd/provisionerdserver/provisionerdserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,12 @@ func TestAcquireJob(t *testing.T) {
_, err = tc.acquire(ctx, srv)
require.ErrorContains(t, err, "sql: no rows in result set")
})
for _, prebuiltWorkspace := range []bool{false, true} {
prebuiltWorkspace := prebuiltWorkspace
for _, prebuiltWorkspaceBuildStage := range []sdkproto.PrebuiltWorkspaceBuildStage{
sdkproto.PrebuiltWorkspaceBuildStage_NONE,
sdkproto.PrebuiltWorkspaceBuildStage_CREATE,
sdkproto.PrebuiltWorkspaceBuildStage_CLAIM,
} {
prebuiltWorkspaceBuildStage := prebuiltWorkspaceBuildStage
t.Run(tc.name+"_WorkspaceBuildJob", func(t *testing.T) {
t.Parallel()
// Set the max session token lifetime so we can assert we
Expand Down Expand Up @@ -294,31 +298,43 @@ func TestAcquireJob(t *testing.T) {
OwnerID: user.ID,
OrganizationID: pd.OrganizationID,
})
build := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
build := database.WorkspaceBuild{
WorkspaceID: workspace.ID,
BuildNumber: 1,
JobID: uuid.New(),
TemplateVersionID: version.ID,
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
})
var buildState sdkproto.PrebuiltWorkspaceBuildStage
if prebuiltWorkspace {
buildState = sdkproto.PrebuiltWorkspaceBuildStage_CREATE
}
_ = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
ID: build.ID,
input := provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: build.ID,
}
dbJob := database.ProvisionerJob{
ID: build.JobID,
OrganizationID: pd.OrganizationID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: build.ID,
PrebuiltWorkspaceBuildStage: buildState,
})),
})
Input: must(json.Marshal(input)),
}
if prebuiltWorkspaceBuildStage == sdkproto.PrebuiltWorkspaceBuildStage_CLAIM {
dbgen.WorkspaceBuild(t, db, build)
prebuildJob := dbgen.ProvisionerJob(t, db, ps, dbJob)
resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{
JobID: prebuildJob.ID,
})
dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{
ResourceID: resource.ID,
AuthToken: uuid.New(),
})
build.BuildNumber = 2
input.PrebuiltWorkspaceBuildStage = sdkproto.PrebuiltWorkspaceBuildStage_CLAIM
dbJob.Input = must(json.Marshal(input))
}
build = dbgen.WorkspaceBuild(t, db, build)
dbJob = dbgen.ProvisionerJob(t, db, ps, dbJob)

startPublished := make(chan struct{})
var closed bool
Expand Down Expand Up @@ -352,7 +368,7 @@ func TestAcquireJob(t *testing.T) {

<-startPublished

got, err := json.Marshal(job.Type)
got, err := json.Marshal(dbJob.Type)
require.NoError(t, err)

// Validate that a session token is generated during the job.
Expand Down Expand Up @@ -385,9 +401,7 @@ func TestAcquireJob(t *testing.T) {
WorkspaceBuildId: build.ID.String(),
WorkspaceOwnerLoginType: string(user.LoginType),
WorkspaceOwnerRbacRoles: []*sdkproto.Role{{Name: rbac.RoleOrgMember(), OrgId: pd.OrganizationID.String()}, {Name: "member", OrgId: ""}, {Name: rbac.RoleOrgAuditor(), OrgId: pd.OrganizationID.String()}},
}
if prebuiltWorkspace {
wantedMetadata.PrebuiltWorkspaceBuildStage = sdkproto.PrebuiltWorkspaceBuildStage_CREATE
PrebuiltWorkspaceBuildStage: prebuiltWorkspaceBuildStage,
}

slices.SortFunc(wantedMetadata.WorkspaceOwnerRbacRoles, func(a, b *sdkproto.Role) int {
Expand Down
88 changes: 43 additions & 45 deletions coderd/workspaceagents_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2647,62 +2647,60 @@ func TestAgentConnectionInfo(t *testing.T) {
func TestReinit(t *testing.T) {
t.Parallel()

t.Run("claimed workspaces are reinitialized", func(t *testing.T) {
t.Parallel()

db, ps := dbtestutil.NewDB(t)
pubsubSpy := pubsubReinitSpy{
Pubsub: ps,
subscriptions: make(chan string),
}
client := coderdtest.New(t, &coderdtest.Options{
Database: db,
Pubsub: pubsubSpy,
})
user := coderdtest.CreateFirstUser(t, client)

r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
}).WithAgent().Do()
db, ps := dbtestutil.NewDB(t)
pubsubSpy := pubsubReinitSpy{
Pubsub: ps,
subscribed: make(chan string),
}
client := coderdtest.New(t, &coderdtest.Options{
Database: db,
Pubsub: &pubsubSpy,
})
user := coderdtest.CreateFirstUser(t, client)

agentCtx := testutil.Context(t, testutil.WaitShort)
agentClient := agentsdk.New(client.URL)
agentClient.SetSessionToken(r.AgentToken)
r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
}).WithAgent().Do()
pubsubSpy.expectedEvent = agentsdk.PrebuildClaimedChannel(r.Workspace.ID)
Comment thread
SasSwart marked this conversation as resolved.

agentReinitializedCh := make(chan *agentsdk.ReinitializationEvent)
go func() {
reinitEvent, err := agentClient.WaitForReinit(agentCtx)
assert.NoError(t, err)
agentReinitializedCh <- reinitEvent
}()
agentCtx := testutil.Context(t, testutil.WaitShort)
agentClient := agentsdk.New(client.URL)
agentClient.SetSessionToken(r.AgentToken)

// We need to subscribe before we publish, lest we miss the event
for subscription := range pubsubSpy.subscriptions {
if subscription == agentsdk.PrebuildClaimedChannel(r.Workspace.ID) {
break
}
}
agentReinitializedCh := make(chan *agentsdk.ReinitializationEvent)
go func() {
reinitEvent, err := agentClient.WaitForReinit(agentCtx)
assert.NoError(t, err)
agentReinitializedCh <- reinitEvent
}()

err := prebuilds.NewPubsubWorkspaceClaimPublisher(ps).PublishWorkspaceClaim(agentsdk.ReinitializationEvent{
WorkspaceID: r.Workspace.ID,
UserID: user.UserID,
})
require.NoError(t, err)
// We need to subscribe before we publish, lest we miss the event
ctx := testutil.Context(t, testutil.WaitShort)
testutil.TryReceive(ctx, t, pubsubSpy.subscribed) // Wait for the appropriate subscription

ctx := testutil.Context(t, testutil.WaitShort)
reinitEvent := testutil.TryReceive(ctx, t, agentReinitializedCh)
require.NotNil(t, reinitEvent)
require.Equal(t, r.Workspace.ID, reinitEvent.WorkspaceID)
// Now that we're subscribed, publish the event
err := prebuilds.NewPubsubWorkspaceClaimPublisher(ps).PublishWorkspaceClaim(agentsdk.ReinitializationEvent{
WorkspaceID: r.Workspace.ID,
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
})
require.NoError(t, err)

ctx = testutil.Context(t, testutil.WaitShort)
reinitEvent := testutil.TryReceive(ctx, t, agentReinitializedCh)
require.NotNil(t, reinitEvent)
require.Equal(t, r.Workspace.ID, reinitEvent.WorkspaceID)
}

type pubsubReinitSpy struct {
pubsub.Pubsub
subscriptions chan string
subscribed chan string
expectedEvent string
}

func (p pubsubReinitSpy) Subscribe(event string, listener pubsub.Listener) (cancel func(), err error) {
p.subscriptions <- event
func (p *pubsubReinitSpy) Subscribe(event string, listener pubsub.Listener) (cancel func(), err error) {
if p.expectedEvent != "" && event == p.expectedEvent {
close(p.subscribed)
}
return p.Pubsub.Subscribe(event, listener)
}
4 changes: 2 additions & 2 deletions codersdk/agentsdk/agentsdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,6 @@ const (

type ReinitializationEvent struct {
WorkspaceID uuid.UUID
UserID uuid.UUID
Reason ReinitializationReason `json:"reason"`
}

Expand Down Expand Up @@ -806,8 +805,9 @@ func (s *SSEAgentReinitTransmitter) Transmit(ctx context.Context, reinitEvents <
return xerrors.Errorf("failed to create sse transmitter: %w", err)
}

// Prevent handler from returning until the sender is closed.
defer func() {
// Block returning until the ServerSentEventSender is closed
// to avoid a race condition where we might write or flush to rw after the handler returns.
<-sseSenderClosed
}()
Comment thread
SasSwart marked this conversation as resolved.

Expand Down
3 changes: 0 additions & 3 deletions codersdk/agentsdk/agentsdk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ func TestStreamAgentReinitEvents(t *testing.T) {
t.Parallel()

eventToSend := agentsdk.ReinitializationEvent{
UserID: uuid.New(),
WorkspaceID: uuid.New(),
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
}
Expand Down Expand Up @@ -56,7 +55,6 @@ func TestStreamAgentReinitEvents(t *testing.T) {
t.Parallel()

eventToSend := agentsdk.ReinitializationEvent{
UserID: uuid.New(),
WorkspaceID: uuid.New(),
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
}
Expand Down Expand Up @@ -92,7 +90,6 @@ func TestStreamAgentReinitEvents(t *testing.T) {
t.Parallel()

eventToSend := agentsdk.ReinitializationEvent{
UserID: uuid.New(),
WorkspaceID: uuid.New(),
Reason: agentsdk.ReinitializeReasonPrebuildClaimed,
}
Expand Down
Loading