Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7145189

Browse files
mafredrimafredri
committed
feat(coderd/database): add dbrollup service to rollup insights
1 parent 310f545 commit 7145189

File tree

7 files changed

+350
-5
lines changed

7 files changed

+350
-5
lines changed

‎coderd/coderd.go

+16-2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747
"github.com/coder/coder/v2/coderd/batchstats"
4848
"github.com/coder/coder/v2/coderd/database"
4949
"github.com/coder/coder/v2/coderd/database/dbauthz"
50+
"github.com/coder/coder/v2/coderd/database/dbrollup"
5051
"github.com/coder/coder/v2/coderd/database/dbtime"
5152
"github.com/coder/coder/v2/coderd/database/pubsub"
5253
"github.com/coder/coder/v2/coderd/externalauth"
@@ -180,6 +181,7 @@ type Options struct {
180181

181182
UpdateAgentMetrics func(ctx context.Context, labels prometheusmetrics.AgentMetricLabels, metrics []*agentproto.Stats_Metric)
182183
StatsBatcher *batchstats.Batcher
184+
DBRollupInterval time.Duration
183185

184186
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
185187

@@ -342,6 +344,9 @@ func New(options *Options) *API {
342344
if options.StatsBatcher == nil {
343345
panic("developer error: options.StatsBatcher is nil")
344346
}
347+
if options.DBRollupInterval == 0 {
348+
options.DBRollupInterval = dbrollup.DefaultInterval
349+
}
345350

346351
siteCacheDir := options.CacheDir
347352
if siteCacheDir != "" {
@@ -414,7 +419,13 @@ func New(options *Options) *API {
414419
ctx,
415420
options.Logger.Named("acquirer"),
416421
options.Database,
417-
options.Pubsub),
422+
options.Pubsub,
423+
),
424+
rolluper: dbrollup.New(
425+
options.Logger,
426+
options.Database,
427+
options.DBRollupInterval,
428+
),
418429
workspaceUsageTracker: options.WorkspaceUsageTracker,
419430
}
420431

@@ -1190,7 +1201,9 @@ type API struct {
11901201
statsBatcher *batchstats.Batcher
11911202

11921203
Acquirer *provisionerdserver.Acquirer
1193-
1204+
// rolluper rolls up template usage stats from raw agent and app
1205+
// stats. This is used to provide insights in the WebUI.
1206+
rolluper *dbrollup.Rolluper
11941207
workspaceUsageTracker *workspaceusage.Tracker
11951208
}
11961209

@@ -1203,6 +1216,7 @@ func (api *API) Close() error {
12031216
api.WebsocketWaitGroup.Wait()
12041217
api.WebsocketWaitMutex.Unlock()
12051218

1219+
api.rolluper.Close()
12061220
api.metricsCache.Close()
12071221
if api.updateChecker != nil {
12081222
api.updateChecker.Close()

‎coderd/coderdtest/coderdtest.go

+2
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ type Options struct {
147147
WorkspaceAppsStatsCollectorOptions workspaceapps.StatsCollectorOptions
148148
AllowWorkspaceRenames bool
149149
NewTicker func(duration time.Duration) (<-chan time.Time, func())
150+
DBRollupInterval time.Duration
150151
WorkspaceUsageTrackerFlush chan int
151152
WorkspaceUsageTrackerTick chan time.Time
152153
}
@@ -487,6 +488,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
487488
WorkspaceAppsStatsCollectorOptions: options.WorkspaceAppsStatsCollectorOptions,
488489
AllowWorkspaceRenames: options.AllowWorkspaceRenames,
489490
NewTicker: options.NewTicker,
491+
DBRollupInterval: options.DBRollupInterval,
490492
WorkspaceUsageTracker: wuTracker,
491493
}
492494
}

‎coderd/database/dbgen/dbgen.go

+32
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,38 @@ func WorkspaceApp(t testing.TB, db database.Store, orig database.WorkspaceApp) d
489489
return resource
490490
}
491491

492+
func WorkspaceAppStat(t testing.TB, db database.Store, orig database.WorkspaceAppStat) database.WorkspaceAppStat {
493+
// This is not going to be correct, but our query doesn't return the ID.
494+
id, err := cryptorand.Int63()
495+
require.NoError(t, err, "generate id")
496+
497+
scheme := database.WorkspaceAppStat{
498+
ID: takeFirst(orig.ID, id),
499+
UserID: takeFirst(orig.UserID, uuid.New()),
500+
WorkspaceID: takeFirst(orig.WorkspaceID, uuid.New()),
501+
AgentID: takeFirst(orig.AgentID, uuid.New()),
502+
AccessMethod: takeFirst(orig.AccessMethod, ""),
503+
SlugOrPort: takeFirst(orig.SlugOrPort, ""),
504+
SessionID: takeFirst(orig.SessionID, uuid.New()),
505+
SessionStartedAt: takeFirst(orig.SessionStartedAt, dbtime.Now().Add(-time.Minute)),
506+
SessionEndedAt: takeFirst(orig.SessionEndedAt, dbtime.Now()),
507+
Requests: takeFirst(orig.Requests, 1),
508+
}
509+
err = db.InsertWorkspaceAppStats(genCtx, database.InsertWorkspaceAppStatsParams{
510+
UserID: []uuid.UUID{scheme.UserID},
511+
WorkspaceID: []uuid.UUID{scheme.WorkspaceID},
512+
AgentID: []uuid.UUID{scheme.AgentID},
513+
AccessMethod: []string{scheme.AccessMethod},
514+
SlugOrPort: []string{scheme.SlugOrPort},
515+
SessionID: []uuid.UUID{scheme.SessionID},
516+
SessionStartedAt: []time.Time{scheme.SessionStartedAt},
517+
SessionEndedAt: []time.Time{scheme.SessionEndedAt},
518+
Requests: []int32{scheme.Requests},
519+
})
520+
require.NoError(t, err, "insert workspace agent stat")
521+
return scheme
522+
}
523+
492524
func WorkspaceResource(t testing.TB, db database.Store, orig database.WorkspaceResource) database.WorkspaceResource {
493525
resource, err := db.InsertWorkspaceResource(genCtx, database.InsertWorkspaceResourceParams{
494526
ID: takeFirst(orig.ID, uuid.New()),

‎coderd/database/dbpurge/dbpurge.go

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
// This is for cleaning up old, unused resources from the database that take up space.
2525
func New(ctx context.Context, logger slog.Logger, db database.Store) io.Closer {
2626
closed := make(chan struct{})
27+
logger = logger.Named("dbpurge")
2728

2829
ctx, cancelFunc := context.WithCancel(ctx)
2930
//nolint:gocritic // The system purges old db records without user input.

‎coderd/database/dbrollup/dbrollup.go

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package dbrollup
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"golang.org/x/sync/errgroup"
8+
9+
"cdr.dev/slog"
10+
11+
"github.com/coder/coder/v2/coderd/database"
12+
"github.com/coder/coder/v2/coderd/database/dbauthz"
13+
)
14+
15+
const (
16+
// DefaultInterval is the default time between rollups.
17+
// Rollups will be synchronized with the clock so that
18+
// they happen 13:00, 13:05, 13:10, etc.
19+
DefaultInterval = 5 * time.Minute
20+
)
21+
22+
type Rolluper struct {
23+
cancel context.CancelFunc
24+
closed chan struct{}
25+
db database.Store
26+
logger slog.Logger
27+
}
28+
29+
// New creates a new DB rollup service that periodically runs rollup queries.
30+
// It is the caller's responsibility to call Close on the returned instance.
31+
//
32+
// This is for e.g. generating insights data (template_usage_stats) from
33+
// raw data (workspace_agent_stats, workspace_app_stats).
34+
func New(logger slog.Logger, db database.Store, interval time.Duration) *Rolluper {
35+
ctx, cancel := context.WithCancel(context.Background())
36+
37+
r := &Rolluper{
38+
cancel: cancel,
39+
closed: make(chan struct{}),
40+
db: db,
41+
logger: logger.Named("dbrollup"),
42+
}
43+
44+
//nolint:gocritic // The system rolls up database tables without user input.
45+
ctx = dbauthz.AsSystemRestricted(ctx)
46+
go r.start(ctx, interval)
47+
48+
return r
49+
}
50+
51+
func (r *Rolluper) start(ctx context.Context, interval time.Duration) {
52+
defer close(r.closed)
53+
54+
do := func() {
55+
var eg errgroup.Group
56+
57+
r.logger.Debug(ctx, "rolling up data")
58+
now := time.Now()
59+
60+
// Track whether or not we performed a rollup (we got the advisory lock).
61+
templateUsageStats := false
62+
63+
eg.Go(func() error {
64+
return r.db.InTx(func(tx database.Store) error {
65+
// Acquire a lock to ensure that only one instance of
66+
// the rollup is running at a time.
67+
ok, err := tx.TryAcquireLock(ctx, database.LockIDDBRollup)
68+
if err != nil {
69+
return err
70+
}
71+
if !ok {
72+
return nil
73+
}
74+
75+
templateUsageStats = true
76+
return tx.UpsertTemplateUsageStats(ctx)
77+
}, nil)
78+
})
79+
80+
err := eg.Wait()
81+
if err != nil {
82+
if database.IsQueryCanceledError(err) {
83+
return
84+
}
85+
// Only log if Close hasn't been called.
86+
if ctx.Err() == nil {
87+
r.logger.Error(ctx, "failed to rollup data", slog.Error(err))
88+
}
89+
} else {
90+
r.logger.Debug(ctx,
91+
"rolled up data",
92+
slog.F("took", time.Since(now)),
93+
slog.F("template_usage_stats", templateUsageStats),
94+
)
95+
}
96+
}
97+
98+
// Perform do immediately and on every tick of the ticker,
99+
// disregarding the execution time of do. This ensure that
100+
// the rollup is performed every interval assuming do does
101+
// not take longer than the interval to execute.
102+
t := time.NewTicker(time.Microsecond)
103+
defer t.Stop()
104+
for {
105+
select {
106+
case <-ctx.Done():
107+
return
108+
case <-t.C:
109+
// Ensure we're on the interval.
110+
now := time.Now()
111+
next := now.Add(interval).Truncate(interval) // Ensure we're on the interval and synced with the clock.
112+
d := next.Sub(now)
113+
// Safety check (shouldn't be possible).
114+
if d <= 0 {
115+
d = interval
116+
}
117+
t.Reset(d)
118+
119+
do()
120+
121+
r.logger.Debug(ctx, "next rollup at", slog.F("next", next))
122+
}
123+
}
124+
}
125+
126+
func (r *Rolluper) Close() error {
127+
r.cancel()
128+
<-r.closed
129+
return nil
130+
}

0 commit comments

Comments
 (0)