Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 91fb8e4

Browse files
mafredrimafredri
committed
feat(coderd/database): add dbrollup service to rollup insights
1 parent 46bc0f0 commit 91fb8e4

File tree

7 files changed

+348
-4
lines changed

7 files changed

+348
-4
lines changed

‎coderd/coderd.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747
"github.com/coder/coder/v2/coderd/batchstats"
4848
"github.com/coder/coder/v2/coderd/database"
4949
"github.com/coder/coder/v2/coderd/database/dbauthz"
50+
"github.com/coder/coder/v2/coderd/database/dbrollup"
5051
"github.com/coder/coder/v2/coderd/database/dbtime"
5152
"github.com/coder/coder/v2/coderd/database/pubsub"
5253
"github.com/coder/coder/v2/coderd/externalauth"
@@ -179,6 +180,7 @@ type Options struct {
179180

180181
UpdateAgentMetrics func(ctx context.Context, labels prometheusmetrics.AgentMetricLabels, metrics []*agentproto.Stats_Metric)
181182
StatsBatcher *batchstats.Batcher
183+
DBRollupInterval time.Duration
182184

183185
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
184186

@@ -338,6 +340,9 @@ func New(options *Options) *API {
338340
if options.StatsBatcher == nil {
339341
panic("developer error: options.StatsBatcher is nil")
340342
}
343+
if options.DBRollupInterval == 0 {
344+
options.DBRollupInterval = dbrollup.DefaultInterval
345+
}
341346

342347
siteCacheDir := options.CacheDir
343348
if siteCacheDir != "" {
@@ -404,7 +409,13 @@ func New(options *Options) *API {
404409
ctx,
405410
options.Logger.Named("acquirer"),
406411
options.Database,
407-
options.Pubsub),
412+
options.Pubsub,
413+
),
414+
rolluper: dbrollup.New(
415+
options.Logger,
416+
options.Database,
417+
options.DBRollupInterval,
418+
),
408419
}
409420

410421
api.AppearanceFetcher.Store(&appearance.DefaultFetcher)
@@ -1178,6 +1189,10 @@ type API struct {
11781189

11791190
statsBatcher *batchstats.Batcher
11801191

1192+
// rolluper rolls up template usage stats from raw agent and app
1193+
// stats. This is used to provide insights in the WebUI.
1194+
rolluper *dbrollup.Rolluper
1195+
11811196
Acquirer *provisionerdserver.Acquirer
11821197
}
11831198

@@ -1190,6 +1205,7 @@ func (api *API) Close() error {
11901205
api.WebsocketWaitGroup.Wait()
11911206
api.WebsocketWaitMutex.Unlock()
11921207

1208+
api.rolluper.Close()
11931209
api.metricsCache.Close()
11941210
if api.updateChecker != nil {
11951211
api.updateChecker.Close()

‎coderd/coderdtest/coderdtest.go

+2
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ type Options struct {
146146
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
147147
AllowWorkspaceRenames bool
148148
NewTicker func(duration time.Duration) (<-chan time.Time, func())
149+
DBRollupInterval time.Duration
149150
}
150151

151152
// New constructs a codersdk client connected to an in-memory API instance.
@@ -454,6 +455,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
454455
WorkspaceAppsStatsCollectorOptions: options.WorkspaceAppsStatsCollectorOptions,
455456
AllowWorkspaceRenames: options.AllowWorkspaceRenames,
456457
NewTicker: options.NewTicker,
458+
DBRollupInterval: options.DBRollupInterval,
457459
}
458460
}
459461

‎coderd/database/dbgen/dbgen.go

+32
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,38 @@ func WorkspaceApp(t testing.TB, db database.Store, orig database.WorkspaceApp) d
489489
return resource
490490
}
491491

492+
func WorkspaceAppStat(t testing.TB, db database.Store, orig database.WorkspaceAppStat) database.WorkspaceAppStat {
493+
// This is not going to be correct, but our query doesn't return the ID.
494+
id, err := cryptorand.Int63()
495+
require.NoError(t, err, "generate id")
496+
497+
scheme := database.WorkspaceAppStat{
498+
ID: takeFirst(orig.ID, id),
499+
UserID: takeFirst(orig.UserID, uuid.New()),
500+
WorkspaceID: takeFirst(orig.WorkspaceID, uuid.New()),
501+
AgentID: takeFirst(orig.AgentID, uuid.New()),
502+
AccessMethod: takeFirst(orig.AccessMethod, ""),
503+
SlugOrPort: takeFirst(orig.SlugOrPort, ""),
504+
SessionID: takeFirst(orig.SessionID, uuid.New()),
505+
SessionStartedAt: takeFirst(orig.SessionStartedAt, dbtime.Now().Add(-time.Minute)),
506+
SessionEndedAt: takeFirst(orig.SessionEndedAt, dbtime.Now()),
507+
Requests: takeFirst(orig.Requests, 1),
508+
}
509+
err = db.InsertWorkspaceAppStats(genCtx, database.InsertWorkspaceAppStatsParams{
510+
UserID: []uuid.UUID{scheme.UserID},
511+
WorkspaceID: []uuid.UUID{scheme.WorkspaceID},
512+
AgentID: []uuid.UUID{scheme.AgentID},
513+
AccessMethod: []string{scheme.AccessMethod},
514+
SlugOrPort: []string{scheme.SlugOrPort},
515+
SessionID: []uuid.UUID{scheme.SessionID},
516+
SessionStartedAt: []time.Time{scheme.SessionStartedAt},
517+
SessionEndedAt: []time.Time{scheme.SessionEndedAt},
518+
Requests: []int32{scheme.Requests},
519+
})
520+
require.NoError(t, err, "insert workspace agent stat")
521+
return scheme
522+
}
523+
492524
func WorkspaceResource(t testing.TB, db database.Store, orig database.WorkspaceResource) database.WorkspaceResource {
493525
resource, err := db.InsertWorkspaceResource(genCtx, database.InsertWorkspaceResourceParams{
494526
ID: takeFirst(orig.ID, uuid.New()),

‎coderd/database/dbpurge/dbpurge.go

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
// This is for cleaning up old, unused resources from the database that take up space.
2525
func New(ctx context.Context, logger slog.Logger, db database.Store) io.Closer {
2626
closed := make(chan struct{})
27+
logger = logger.Named("dbpurge")
2728

2829
ctx, cancelFunc := context.WithCancel(ctx)
2930
//nolint:gocritic // The system purges old db records without user input.

‎coderd/database/dbrollup/dbrollup.go

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package dbrollup
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"golang.org/x/sync/errgroup"
8+
9+
"cdr.dev/slog"
10+
11+
"github.com/coder/coder/v2/coderd/database"
12+
"github.com/coder/coder/v2/coderd/database/dbauthz"
13+
)
14+
15+
const (
16+
// DefaultInterval is the default time between rollups.
17+
// Rollups will be synchronized with the clock so that
18+
// they happen 13:00, 13:05, 13:10, etc.
19+
DefaultInterval = 5 * time.Minute
20+
)
21+
22+
type Rolluper struct {
23+
cancel context.CancelFunc
24+
closed chan struct{}
25+
db database.Store
26+
logger slog.Logger
27+
}
28+
29+
// New creates a new DB rollup service that periodically runs rollup queries.
30+
// It is the caller's responsibility to call Close on the returned instance.
31+
//
32+
// This is for e.g. generating insights data (template_usage_stats) from
33+
// raw data (workspace_agent_stats, workspace_app_stats).
34+
func New(logger slog.Logger, db database.Store, interval time.Duration) *Rolluper {
35+
ctx, cancel := context.WithCancel(context.Background())
36+
37+
r := &Rolluper{
38+
cancel: cancel,
39+
closed: make(chan struct{}),
40+
db: db,
41+
logger: logger.Named("dbrollup"),
42+
}
43+
44+
//nolint:gocritic // The system rolls up database tables without user input.
45+
ctx = dbauthz.AsSystemRestricted(ctx)
46+
go r.start(ctx, interval)
47+
48+
return r
49+
}
50+
51+
func (r *Rolluper) start(ctx context.Context, interval time.Duration) {
52+
defer close(r.closed)
53+
54+
do := func() {
55+
var eg errgroup.Group
56+
57+
r.logger.Debug(ctx, "rolling up data")
58+
now := time.Now()
59+
60+
// Track whether or not we performed a rollup (we got the advisory lock).
61+
templateUsageStats := false
62+
63+
eg.Go(func() error {
64+
return r.db.InTx(func(tx database.Store) error {
65+
// Acquire a lock to ensure that only one instance of
66+
// the rollup is running at a time.
67+
ok, err := tx.TryAcquireLock(ctx, database.LockIDDBRollup)
68+
if err != nil {
69+
return err
70+
}
71+
if !ok {
72+
return nil
73+
}
74+
75+
templateUsageStats = true
76+
return tx.UpsertTemplateUsageStats(ctx)
77+
}, nil)
78+
})
79+
80+
err := eg.Wait()
81+
if err != nil {
82+
if database.IsQueryCanceledError(err) {
83+
return
84+
}
85+
r.logger.Error(ctx, "failed to rollup data", slog.Error(err))
86+
} else {
87+
r.logger.Debug(ctx,
88+
"rolled up data",
89+
slog.F("took", time.Since(now)),
90+
slog.F("template_usage_stats", templateUsageStats),
91+
)
92+
}
93+
}
94+
95+
// Perform do immediately and on every tick of the ticker,
96+
// disregarding the execution time of do. This ensure that
97+
// the rollup is performed every interval assuming do does
98+
// not take longer than the interval to execute.
99+
t := time.NewTicker(time.Microsecond)
100+
defer t.Stop()
101+
for {
102+
select {
103+
case <-ctx.Done():
104+
return
105+
case <-t.C:
106+
// Ensure we're on the interval.
107+
now := time.Now()
108+
next := now.Add(interval).Truncate(interval) // Ensure we're on the interval and synced with the clock.
109+
d := next.Sub(now)
110+
// Safety check.
111+
if d == 0 {
112+
d = interval
113+
}
114+
t.Reset(d)
115+
116+
do()
117+
118+
r.logger.Debug(ctx, "next rollup at", slog.F("next", next))
119+
}
120+
}
121+
}
122+
123+
func (r *Rolluper) Close() error {
124+
r.cancel()
125+
<-r.closed
126+
return nil
127+
}

0 commit comments

Comments
 (0)