Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -2059,6 +2059,13 @@ func (q *querier) DeleteApplicationConnectAPIKeysByUserID(ctx context.Context, u
return q.db.DeleteApplicationConnectAPIKeysByUserID(ctx, userID)
}

func (q *querier) DeleteCachedModuleFilesCreatedBetween(ctx context.Context, arg database.DeleteCachedModuleFilesCreatedBetweenParams) (int64, error) {
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
return 0, err
}
return q.db.DeleteCachedModuleFilesCreatedBetween(ctx, arg)
}

func (q *querier) DeleteChatContextResourcesByChatID(ctx context.Context, chatID uuid.UUID) error {
chat, err := q.db.GetChatByID(ctx, chatID)
if err != nil {
Expand Down
5 changes: 5 additions & 0 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5143,6 +5143,11 @@ func (s *MethodTestSuite) TestSystemFunctions() {
dbm.EXPECT().DeleteOldWorkspaceAgentLogs(gomock.Any(), t).Return(int64(0), nil).AnyTimes()
check.Args(t).Asserts(rbac.ResourceSystem, policy.ActionDelete)
}))
s.Run("DeleteCachedModuleFilesCreatedBetween", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
arg := database.DeleteCachedModuleFilesCreatedBetweenParams{}
dbm.EXPECT().DeleteCachedModuleFilesCreatedBetween(gomock.Any(), arg).Return(int64(0), nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourceSystem, policy.ActionDelete)
}))
s.Run("InsertWorkspaceAgentStats", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
arg := database.InsertWorkspaceAgentStatsParams{}
dbm.EXPECT().InsertWorkspaceAgentStats(gomock.Any(), arg).Return(xerrors.New("any error")).AnyTimes()
Expand Down
8 changes: 8 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

43 changes: 43 additions & 0 deletions coderd/database/dbpurge/dbpurge.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,20 @@ const (
chatDebugRunsBatchSize = 1000
)

// Terraform module archives ingested during this window may contain the
// identified upstream module.
//
// This is a one-off cleanup, not a recurring purge. It runs once per coderd
// process because the window is fixed in the past: after a successful pass
// there is nothing left to match. It lives here rather than in a migration
// because migrations cannot be backported. The version table records a single
// high-water mark, so a migration cherry-picked onto a release branch would
// cause later upgrades to skip every migration in between.
var (
identifiedModuleCacheStart = time.Date(2026, 8, 31, 8, 0, 0, 0, time.UTC)
identifiedModuleCacheEnd = time.Date(2026, 8, 31, 22, 0, 0, 0, time.UTC)
)

type Option func(*instance)

// WithClock overrides the clock used by the purger. Defaults to
Expand Down Expand Up @@ -151,6 +165,10 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.

chatConfigErr := errors.Join(chatRetentionErr, chatDebugRetentionErr)

// Latched after a successful commit so the one-off module cache cleanup
// is attempted again if the transaction rolls back.
ranModuleCachePurge := false

// Start a transaction to grab advisory lock, we don't want to run
// multiple purges at the same time (multiple replicas).
err := db.InTx(func(tx database.Store) error {
Expand Down Expand Up @@ -296,6 +314,20 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
}
}

// One-off cleanup of the identified Terraform module cache. Skipped
// once this process has completed a pass.
var purgedIdentifiedModuleFiles int64
if !i.identifiedModuleCachePurged {
purgedIdentifiedModuleFiles, err = tx.DeleteCachedModuleFilesCreatedBetween(ctx, database.DeleteCachedModuleFilesCreatedBetweenParams{
CreatedAtAfter: identifiedModuleCacheStart,
CreatedAtBefore: identifiedModuleCacheEnd,
})
if err != nil {
return xerrors.Errorf("failed to delete identified module cache files: %w", err)
}
ranModuleCachePurge = true
}

i.logger.Debug(ctx, "purged old database entries",
slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs),
slog.F("expired_api_keys", expiredAPIKeys),
Expand All @@ -307,6 +339,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
slog.F("chats", purgedChats),
slog.F("chat_files", purgedChatFiles),
slog.F("chat_debug_runs", purgedChatDebugRuns),
slog.F("identified_module_files", purgedIdentifiedModuleFiles),
slog.F("duration", i.clk.Since(start)),
)

Expand All @@ -321,6 +354,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats))
i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns))
i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles))
i.recordsPurged.WithLabelValues("identified_module_files").Add(float64(purgedIdentifiedModuleFiles))
}

// chatConfigErr is returned after the tx, so do not record this
Expand All @@ -336,6 +370,10 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
return err
}

if ranModuleCachePurge {
i.identifiedModuleCachePurged = true
}

// Surface the deferred chat-config error so doTick records
// the failed iteration metric.
if chatConfigErr != nil {
Expand All @@ -353,6 +391,11 @@ type instance struct {
clk quartz.Clock
iterationDuration *prometheus.HistogramVec
recordsPurged *prometheus.CounterVec

// identifiedModuleCachePurged latches once this process has completed a
// pass of the one-off module cache cleanup. The window is fixed in the
// past, so a completed pass leaves nothing to match on later ticks.
identifiedModuleCachePurged bool
}

func (i *instance) Close() error {
Expand Down
14 changes: 14 additions & 0 deletions coderd/database/dbpurge/dbpurge_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,17 @@ func TestDBPurgeAuthorization(t *testing.T) {
err := inst.purgeTick(ctx, db, now)
require.NoError(t, err)
}

// The behavior of the one-off module cache cleanup is covered by
// TestDeleteIdentifiedModuleCacheFiles, which supplies its own window. This
// guards the production constants themselves, which that test no longer reads.
func TestIdentifiedModuleCacheWindow(t *testing.T) {
t.Parallel()

require.True(t, identifiedModuleCacheStart.Before(identifiedModuleCacheEnd),
"window start must precede window end")
require.Equal(t, time.UTC, identifiedModuleCacheStart.Location(),
"window bounds must be UTC so they do not shift with the host timezone")
require.Equal(t, time.UTC, identifiedModuleCacheEnd.Location(),
"window bounds must be UTC so they do not shift with the host timezone")
}
176 changes: 176 additions & 0 deletions coderd/database/dbpurge/dbpurge_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -248,6 +249,7 @@ func TestMetrics(t *testing.T) {
mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteCachedModuleFilesCreatedBetween(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteCachedModuleFilesCreatedBetweenParams{})).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
Expand Down Expand Up @@ -298,6 +300,7 @@ func TestMetrics(t *testing.T) {
mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().DeleteCachedModuleFilesCreatedBetween(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteCachedModuleFilesCreatedBetweenParams{})).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
Expand Down Expand Up @@ -2714,3 +2717,176 @@ func TestDeleteOldChatFiles(t *testing.T) {
})
}
}

func awaitDoTicks(ctx context.Context, t *testing.T, clk *quartz.Mock, n int) func() {
t.Helper()
completed := make(chan struct{})
advance := make(chan struct{})
trapNow := clk.Trap().Now()
trapStop := clk.Trap().TickerStop()
trapReset := clk.Trap().TickerReset()
go func() {
defer close(completed)
defer trapReset.Close()
defer trapStop.Close()
defer trapNow.Close()
trapNow.MustWait(ctx).MustRelease(ctx)
trapReset.MustWait(ctx).MustRelease(ctx)
select {
case completed <- struct{}{}:
case <-ctx.Done():
return
}
for i := 1; i < n; i++ {
select {
case <-advance:
case <-ctx.Done():
return
}
d, w := clk.AdvanceNext()
if !assert.Equal(t, 10*time.Minute, d) {
return
}
w.MustWait(ctx)
trapStop.MustWait(ctx).MustRelease(ctx)
trapReset.MustWait(ctx).MustRelease(ctx)
select {
case completed <- struct{}{}:
case <-ctx.Done():
return
}
}
}()
first := true
return func() {
t.Helper()
if !first {
testutil.RequireSend(ctx, t, advance, struct{}{})
}
first = false
testutil.TryReceive(ctx, t, completed)
}
}

//nolint:paralleltest // It uses LockIDDBPurge.
func TestDeleteIdentifiedModuleCacheFiles(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitShort)
clk := quartz.NewMock(t)
clk.Set(dbtime.Now()).MustWait(ctx)

// The window under test is supplied explicitly rather than copied from the
// production constants, so revising the incident timestamps cannot silently
// invalidate these boundary assertions.
windowStart := time.Date(2026, 8, 31, 8, 0, 0, 0, time.UTC)
windowEnd := time.Date(2026, 8, 31, 22, 0, 0, 0, time.UTC)
inWindow := windowStart.Add(time.Minute)

db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
org := dbgen.Organization(t, db, database.Organization{})
user := dbgen.User(t, db, database.User{})
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})

logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})

mkFile := func(name string, createdBy uuid.UUID, mimetype string, createdAt time.Time) database.File {
file, err := db.InsertFile(ctx, database.InsertFileParams{
ID: uuid.New(),
Hash: fmt.Sprintf("%x", sha256.Sum256([]byte(name))),
CreatedBy: createdBy,
CreatedAt: createdAt,
Mimetype: mimetype,
Data: []byte{},
})
require.NoError(t, err, "insert file %q", name)
return file
}

// mkVersion creates a template version whose cached module files point at a
// file with the given properties. InsertFile is used directly because
// dbgen.File treats uuid.Nil as unset and substitutes a random creator,
// while uuid.Nil is exactly what identifies a provisionerd module archive.
mkVersion := func(name string, createdBy uuid.UUID, mimetype string, createdAt time.Time) (database.File, database.TemplateVersion) {
file := mkFile(name, createdBy, mimetype, createdAt)
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
Name: name,
OrganizationID: org.ID,
CreatedBy: user.ID,
})
_ = dbgen.TemplateVersionTerraformValues(t, db, database.TemplateVersionTerraformValue{
TemplateVersionID: tv.ID,
CachedModuleFiles: uuid.NullUUID{UUID: file.ID, Valid: true},
})
return file, tv
}

// Identified: a provisionerd module archive cached inside the window.
identified, identifiedTV := mkVersion("identified", uuid.Nil, "application/x-tar", inWindow)
// The lower bound is inclusive.
atStart, atStartTV := mkVersion("at-start", uuid.Nil, "application/x-tar", windowStart)
// The upper bound is exclusive, so this archive is known good.
atEnd, atEndTV := mkVersion("at-end", uuid.Nil, "application/x-tar", windowEnd)
// Cached before and after the window.
before, beforeTV := mkVersion("before", uuid.Nil, "application/x-tar", windowStart.Add(-time.Hour))
after, afterTV := mkVersion("after", uuid.Nil, "application/x-tar", windowEnd.Add(time.Hour))
// A user-uploaded template tarball shares the mimetype but has a real
// creator, so it must survive even though it is inside the window.
userUpload, userUploadTV := mkVersion("user-upload", user.ID, "application/x-tar", inWindow)

// An unreferenced archive inside the window. Only archives referenced by a
// template version are in scope.
orphan := mkFile("orphan", uuid.Nil, "application/x-tar", inWindow)

// when dbpurge runs
tick := awaitDoTicks(ctx, t, clk, 2)
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk))
defer closer.Close()
tick() // doTick() has now run.

assertFileDeleted := func(id uuid.UUID, name string) {
t.Helper()
_, err := db.GetFileByID(ctx, id)
require.ErrorIs(t, err, sql.ErrNoRows, "%s should be deleted", name)
}
assertFileExists := func(id uuid.UUID, name string) {
t.Helper()
_, err := db.GetFileByID(ctx, id)
require.NoError(t, err, "%s should be retained", name)
}
// assertCacheRef checks the template version still exists and that its
// module cache reference was cleared only when the file was deleted.
assertCacheRef := func(tv database.TemplateVersion, wantFile uuid.UUID, wantValid bool, name string) {
t.Helper()
values, err := db.GetTemplateVersionTerraformValues(ctx, tv.ID)
require.NoError(t, err, "%s: terraform values row must be retained", name)
require.Equal(t, wantValid, values.CachedModuleFiles.Valid, "%s: cache reference validity", name)
if wantValid {
require.Equal(t, wantFile, values.CachedModuleFiles.UUID, "%s: cache reference target", name)
}
}

// then the identified archives are deleted and their references cleared
assertFileDeleted(identified.ID, "archive inside the window")
assertCacheRef(identifiedTV, uuid.Nil, false, "archive inside the window")
assertFileDeleted(atStart.ID, "archive at the inclusive lower bound")
assertCacheRef(atStartTV, uuid.Nil, false, "archive at the inclusive lower bound")

// and everything else is untouched
assertFileExists(atEnd.ID, "archive at the exclusive upper bound")
assertCacheRef(atEndTV, atEnd.ID, true, "archive at the exclusive upper bound")
assertFileExists(before.ID, "archive cached before the window")
assertCacheRef(beforeTV, before.ID, true, "archive cached before the window")
assertFileExists(after.ID, "archive cached after the window")
assertCacheRef(afterTV, after.ID, true, "archive cached after the window")
assertFileExists(userUpload.ID, "user-uploaded tarball")
assertCacheRef(userUploadTV, userUpload.ID, true, "user-uploaded tarball")
assertFileExists(orphan.ID, "unreferenced archive")

// The cleanup is one-off, not a recurring purge. A second tick must not
// repeat it, so an archive inserted into the window after the first pass
// survives. This documents the latch: the window is fixed in the past and
// nothing can legitimately land in it again.
late, lateTV := mkVersion("late", uuid.Nil, "application/x-tar", inWindow)
tick()
assertFileExists(late.ID, "archive inserted after the one-off pass")
assertCacheRef(lateTV, late.ID, true, "archive inserted after the one-off pass")
}
6 changes: 6 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading