diff --git a/cli/server_test.go b/cli/server_test.go index fa96e192f7eb3..8ed4d89ceb970 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -964,7 +964,7 @@ func TestServer(t *testing.T) { server := httptest.NewServer(r) defer server.Close() - inv, _ := clitest.New(t, + inv, cfg := clitest.New(t, "server", "--in-memory", "--http-address", ":0", @@ -977,6 +977,25 @@ func TestServer(t *testing.T) { <-deployment <-snapshot + + accessURL := waitAccessURL(t, cfg) + + ctx := testutil.Context(t, testutil.WaitMedium) + client := codersdk.New(accessURL) + body, err := client.Request(ctx, http.MethodGet, "/", nil) + require.NoError(t, err) + require.NoError(t, body.Body.Close()) + + require.Eventually(t, func() bool { + snap := <-snapshot + htmlFirstServedFound := false + for _, item := range snap.TelemetryItems { + if item.Key == string(telemetry.TelemetryItemKeyHTMLFirstServedAt) { + htmlFirstServedFound = true + } + } + return htmlFirstServedFound + }, testutil.WaitMedium, testutil.IntervalFast, "no html_first_served telemetry item") }) t.Run("Prometheus", func(t *testing.T) { t.Parallel() diff --git a/coderd/coderd.go b/coderd/coderd.go index e273b7afdb80f..be558797389b9 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -585,6 +585,8 @@ func New(options *Options) *API { AppearanceFetcher: &api.AppearanceFetcher, BuildInfo: buildInfo, Entitlements: options.Entitlements, + Telemetry: options.Telemetry, + Logger: options.Logger.Named("site"), }) api.SiteHandler.Experiments.Store(&experiments) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 2e12cab9d33e0..0ba9e20216b41 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2096,6 +2096,20 @@ func (q *querier) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) return q.db.GetTailnetTunnelPeerIDs(ctx, srcID) } +func (q *querier) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return database.TelemetryItem{}, err + } + return q.db.GetTelemetryItem(ctx, key) +} + +func (q *querier) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return nil, err + } + return q.db.GetTelemetryItems(ctx) +} + func (q *querier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil { return nil, err @@ -3085,6 +3099,13 @@ func (q *querier) InsertReplica(ctx context.Context, arg database.InsertReplicaP return q.db.InsertReplica(ctx, arg) } +func (q *querier) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { + return err + } + return q.db.InsertTelemetryItemIfNotExists(ctx, arg) +} + func (q *querier) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { obj := rbac.ResourceTemplate.InOrg(arg.OrganizationID) if err := q.authorizeContext(ctx, policy.ActionCreate, obj); err != nil { @@ -4345,6 +4366,13 @@ func (q *querier) UpsertTailnetTunnel(ctx context.Context, arg database.UpsertTa return q.db.UpsertTailnetTunnel(ctx, arg) } +func (q *querier) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { + return err + } + return q.db.UpsertTelemetryItem(ctx, arg) +} + func (q *querier) UpsertTemplateUsageStats(ctx context.Context) error { if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { return err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index fdbbcc8b34ca6..9e784fff0bf12 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4224,6 +4224,24 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetWorkspaceModulesCreatedAfter", s.Subtest(func(db database.Store, check *expects) { check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead) })) + s.Run("GetTelemetryItem", s.Subtest(func(db database.Store, check *expects) { + check.Args("test").Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) + })) + s.Run("GetTelemetryItems", s.Subtest(func(db database.Store, check *expects) { + check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead) + })) + s.Run("InsertTelemetryItemIfNotExists", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.InsertTelemetryItemIfNotExistsParams{ + Key: "test", + Value: "value", + }).Asserts(rbac.ResourceSystem, policy.ActionCreate) + })) + s.Run("UpsertTelemetryItem", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.UpsertTelemetryItemParams{ + Key: "test", + Value: "value", + }).Asserts(rbac.ResourceSystem, policy.ActionUpdate) + })) } func (s *MethodTestSuite) TestNotifications() { diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 566540dcb2906..54e4f99959b44 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -1093,6 +1093,23 @@ func ProvisionerJobTimings(t testing.TB, db database.Store, build database.Works return timings } +func TelemetryItem(t testing.TB, db database.Store, seed database.TelemetryItem) database.TelemetryItem { + if seed.Key == "" { + seed.Key = testutil.GetRandomName(t) + } + if seed.Value == "" { + seed.Value = time.Now().Format(time.RFC3339) + } + err := db.UpsertTelemetryItem(genCtx, database.UpsertTelemetryItemParams{ + Key: seed.Key, + Value: seed.Value, + }) + require.NoError(t, err, "upsert telemetry item") + item, err := db.GetTelemetryItem(genCtx, seed.Key) + require.NoError(t, err, "get telemetry item") + return item +} + func provisionerJobTiming(t testing.TB, db database.Store, seed database.ProvisionerJobTiming) database.ProvisionerJobTiming { timing, err := db.InsertProvisionerJobTimings(genCtx, database.InsertProvisionerJobTimingsParams{ JobID: takeFirst(seed.JobID, uuid.New()), diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 6b518c7696369..103ee1e717149 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -89,6 +89,7 @@ func New() database.Store { locks: map[int64]struct{}{}, runtimeConfig: map[string]string{}, userStatusChanges: make([]database.UserStatusChange, 0), + telemetryItems: make([]database.TelemetryItem, 0), }, } // Always start with a default org. Matching migration 198. @@ -258,6 +259,7 @@ type data struct { defaultProxyDisplayName string defaultProxyIconURL string userStatusChanges []database.UserStatusChange + telemetryItems []database.TelemetryItem } func tryPercentile(fs []float64, p float64) float64 { @@ -4330,6 +4332,23 @@ func (*FakeQuerier) GetTailnetTunnelPeerIDs(context.Context, uuid.UUID) ([]datab return nil, ErrUnimplemented } +func (q *FakeQuerier) GetTelemetryItem(_ context.Context, key string) (database.TelemetryItem, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + for _, item := range q.telemetryItems { + if item.Key == key { + return item, nil + } + } + + return database.TelemetryItem{}, sql.ErrNoRows +} + +func (q *FakeQuerier) GetTelemetryItems(_ context.Context) ([]database.TelemetryItem, error) { + return q.telemetryItems, nil +} + func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { @@ -8120,6 +8139,30 @@ func (q *FakeQuerier) InsertReplica(_ context.Context, arg database.InsertReplic return replica, nil } +func (q *FakeQuerier) InsertTelemetryItemIfNotExists(_ context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + err := validateDatabaseType(arg) + if err != nil { + return err + } + + q.mutex.Lock() + defer q.mutex.Unlock() + + for _, item := range q.telemetryItems { + if item.Key == arg.Key { + return nil + } + } + + q.telemetryItems = append(q.telemetryItems, database.TelemetryItem{ + Key: arg.Key, + Value: arg.Value, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + return nil +} + func (q *FakeQuerier) InsertTemplate(_ context.Context, arg database.InsertTemplateParams) error { if err := validateDatabaseType(arg); err != nil { return err @@ -10874,6 +10917,33 @@ func (*FakeQuerier) UpsertTailnetTunnel(_ context.Context, arg database.UpsertTa return database.TailnetTunnel{}, ErrUnimplemented } +func (q *FakeQuerier) UpsertTelemetryItem(_ context.Context, arg database.UpsertTelemetryItemParams) error { + err := validateDatabaseType(arg) + if err != nil { + return err + } + + q.mutex.Lock() + defer q.mutex.Unlock() + + for i, item := range q.telemetryItems { + if item.Key == arg.Key { + q.telemetryItems[i].Value = arg.Value + q.telemetryItems[i].UpdatedAt = time.Now() + return nil + } + } + + q.telemetryItems = append(q.telemetryItems, database.TelemetryItem{ + Key: arg.Key, + Value: arg.Value, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + + return nil +} + func (q *FakeQuerier) UpsertTemplateUsageStats(ctx context.Context) error { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index ba8a1f9cdc8a6..c0d3ed4994f9c 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1134,6 +1134,20 @@ func (m queryMetricsStore) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uu return r0, r1 } +func (m queryMetricsStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + start := time.Now() + r0, r1 := m.s.GetTelemetryItem(ctx, key) + m.queryLatencies.WithLabelValues("GetTelemetryItem").Observe(time.Since(start).Seconds()) + return r0, r1 +} + +func (m queryMetricsStore) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + start := time.Now() + r0, r1 := m.s.GetTelemetryItems(ctx) + m.queryLatencies.WithLabelValues("GetTelemetryItems").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateAppInsights(ctx, arg) @@ -1911,6 +1925,13 @@ func (m queryMetricsStore) InsertReplica(ctx context.Context, arg database.Inser return replica, err } +func (m queryMetricsStore) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + start := time.Now() + r0 := m.s.InsertTelemetryItemIfNotExists(ctx, arg) + m.queryLatencies.WithLabelValues("InsertTelemetryItemIfNotExists").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { start := time.Now() err := m.s.InsertTemplate(ctx, arg) @@ -2772,6 +2793,13 @@ func (m queryMetricsStore) UpsertTailnetTunnel(ctx context.Context, arg database return r0, r1 } +func (m queryMetricsStore) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + start := time.Now() + r0 := m.s.UpsertTelemetryItem(ctx, arg) + m.queryLatencies.WithLabelValues("UpsertTelemetryItem").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) UpsertTemplateUsageStats(ctx context.Context) error { start := time.Now() r0 := m.s.UpsertTemplateUsageStats(ctx) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index d2aa8aa6fa62e..e32834a441e6d 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2346,6 +2346,36 @@ func (mr *MockStoreMockRecorder) GetTailnetTunnelPeerIDs(ctx, srcID any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTailnetTunnelPeerIDs", reflect.TypeOf((*MockStore)(nil).GetTailnetTunnelPeerIDs), ctx, srcID) } +// GetTelemetryItem mocks base method. +func (m *MockStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetryItem", ctx, key) + ret0, _ := ret[0].(database.TelemetryItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTelemetryItem indicates an expected call of GetTelemetryItem. +func (mr *MockStoreMockRecorder) GetTelemetryItem(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItem", reflect.TypeOf((*MockStore)(nil).GetTelemetryItem), ctx, key) +} + +// GetTelemetryItems mocks base method. +func (m *MockStore) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetryItems", ctx) + ret0, _ := ret[0].([]database.TelemetryItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTelemetryItems indicates an expected call of GetTelemetryItems. +func (mr *MockStoreMockRecorder) GetTelemetryItems(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItems", reflect.TypeOf((*MockStore)(nil).GetTelemetryItems), ctx) +} + // GetTemplateAppInsights mocks base method. func (m *MockStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { m.ctrl.T.Helper() @@ -4051,6 +4081,20 @@ func (mr *MockStoreMockRecorder) InsertReplica(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertReplica", reflect.TypeOf((*MockStore)(nil).InsertReplica), ctx, arg) } +// InsertTelemetryItemIfNotExists mocks base method. +func (m *MockStore) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertTelemetryItemIfNotExists", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// InsertTelemetryItemIfNotExists indicates an expected call of InsertTelemetryItemIfNotExists. +func (mr *MockStoreMockRecorder) InsertTelemetryItemIfNotExists(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertTelemetryItemIfNotExists", reflect.TypeOf((*MockStore)(nil).InsertTelemetryItemIfNotExists), ctx, arg) +} + // InsertTemplate mocks base method. func (m *MockStore) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { m.ctrl.T.Helper() @@ -5861,6 +5905,20 @@ func (mr *MockStoreMockRecorder) UpsertTailnetTunnel(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTailnetTunnel", reflect.TypeOf((*MockStore)(nil).UpsertTailnetTunnel), ctx, arg) } +// UpsertTelemetryItem mocks base method. +func (m *MockStore) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertTelemetryItem", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertTelemetryItem indicates an expected call of UpsertTelemetryItem. +func (mr *MockStoreMockRecorder) UpsertTelemetryItem(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTelemetryItem", reflect.TypeOf((*MockStore)(nil).UpsertTelemetryItem), ctx, arg) +} + // UpsertTemplateUsageStats mocks base method. func (m *MockStore) UpsertTemplateUsageStats(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c241548e166c2..9cc38adf23b6b 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1164,6 +1164,13 @@ CREATE TABLE tailnet_tunnels ( updated_at timestamp with time zone NOT NULL ); +CREATE TABLE telemetry_items ( + key text NOT NULL, + value text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + CREATE TABLE template_usage_stats ( start_time timestamp with time zone NOT NULL, end_time timestamp with time zone NOT NULL, @@ -2026,6 +2033,9 @@ ALTER TABLE ONLY tailnet_peers ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_pkey PRIMARY KEY (coordinator_id, src_id, dst_id); +ALTER TABLE ONLY telemetry_items + ADD CONSTRAINT telemetry_items_pkey PRIMARY KEY (key); + ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); diff --git a/coderd/database/migrations/000288_telemetry_items.down.sql b/coderd/database/migrations/000288_telemetry_items.down.sql new file mode 100644 index 0000000000000..118188f519e76 --- /dev/null +++ b/coderd/database/migrations/000288_telemetry_items.down.sql @@ -0,0 +1 @@ +DROP TABLE telemetry_items; diff --git a/coderd/database/migrations/000288_telemetry_items.up.sql b/coderd/database/migrations/000288_telemetry_items.up.sql new file mode 100644 index 0000000000000..40279827788d6 --- /dev/null +++ b/coderd/database/migrations/000288_telemetry_items.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE telemetry_items ( + key TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); diff --git a/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql b/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql new file mode 100644 index 0000000000000..0189558292915 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql @@ -0,0 +1,4 @@ +INSERT INTO + telemetry_items (key, value) +VALUES + ('example_key', 'example_value'); diff --git a/coderd/database/models.go b/coderd/database/models.go index b0a487c192793..9769bde33052b 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -2787,6 +2787,13 @@ type TailnetTunnel struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } +type TelemetryItem struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + // Joins in the display name information such as username, avatar, and organization name. type Template struct { ID uuid.UUID `db:"id" json:"id"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 132a7aea75bdd..1fa83208a2218 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -224,6 +224,8 @@ type sqlcQuerier interface { GetTailnetPeers(ctx context.Context, id uuid.UUID) ([]TailnetPeer, error) GetTailnetTunnelPeerBindings(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerBindingsRow, error) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerIDsRow, error) + GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) + GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error) // GetTemplateAppInsights returns the aggregate usage of each app in a given // timeframe. The result can be filtered on template_ids, meaning only user data // from workspaces based on those templates will be included. @@ -404,6 +406,7 @@ type sqlcQuerier interface { InsertProvisionerJobTimings(ctx context.Context, arg InsertProvisionerJobTimingsParams) ([]ProvisionerJobTiming, error) InsertProvisionerKey(ctx context.Context, arg InsertProvisionerKeyParams) (ProvisionerKey, error) InsertReplica(ctx context.Context, arg InsertReplicaParams) (Replica, error) + InsertTelemetryItemIfNotExists(ctx context.Context, arg InsertTelemetryItemIfNotExistsParams) error InsertTemplate(ctx context.Context, arg InsertTemplateParams) error InsertTemplateVersion(ctx context.Context, arg InsertTemplateVersionParams) error InsertTemplateVersionParameter(ctx context.Context, arg InsertTemplateVersionParameterParams) (TemplateVersionParameter, error) @@ -546,6 +549,7 @@ type sqlcQuerier interface { UpsertTailnetCoordinator(ctx context.Context, id uuid.UUID) (TailnetCoordinator, error) UpsertTailnetPeer(ctx context.Context, arg UpsertTailnetPeerParams) (TailnetPeer, error) UpsertTailnetTunnel(ctx context.Context, arg UpsertTailnetTunnelParams) (TailnetTunnel, error) + UpsertTelemetryItem(ctx context.Context, arg UpsertTelemetryItemParams) error // This query aggregates the workspace_agent_stats and workspace_app_stats data // into a single table for efficient storage and querying. Half-hour buckets are // used to store the data, and the minutes are summed for each user and template diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 38dbf1fbfd0bb..86db8fb66956a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8702,6 +8702,86 @@ func (q *sqlQuerier) UpsertTailnetTunnel(ctx context.Context, arg UpsertTailnetT return i, err } +const getTelemetryItem = `-- name: GetTelemetryItem :one +SELECT key, value, created_at, updated_at FROM telemetry_items WHERE key = $1 +` + +func (q *sqlQuerier) GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) { + row := q.db.QueryRowContext(ctx, getTelemetryItem, key) + var i TelemetryItem + err := row.Scan( + &i.Key, + &i.Value, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getTelemetryItems = `-- name: GetTelemetryItems :many +SELECT key, value, created_at, updated_at FROM telemetry_items +` + +func (q *sqlQuerier) GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error) { + rows, err := q.db.QueryContext(ctx, getTelemetryItems) + if err != nil { + return nil, err + } + defer rows.Close() + var items []TelemetryItem + for rows.Next() { + var i TelemetryItem + if err := rows.Scan( + &i.Key, + &i.Value, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertTelemetryItemIfNotExists = `-- name: InsertTelemetryItemIfNotExists :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO NOTHING +` + +type InsertTelemetryItemIfNotExistsParams struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + +func (q *sqlQuerier) InsertTelemetryItemIfNotExists(ctx context.Context, arg InsertTelemetryItemIfNotExistsParams) error { + _, err := q.db.ExecContext(ctx, insertTelemetryItemIfNotExists, arg.Key, arg.Value) + return err +} + +const upsertTelemetryItem = `-- name: UpsertTelemetryItem :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = $2, updated_at = NOW() WHERE telemetry_items.key = $1 +` + +type UpsertTelemetryItemParams struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + +func (q *sqlQuerier) UpsertTelemetryItem(ctx context.Context, arg UpsertTelemetryItemParams) error { + _, err := q.db.ExecContext(ctx, upsertTelemetryItem, arg.Key, arg.Value) + return err +} + const getTemplateAverageBuildTime = `-- name: GetTemplateAverageBuildTime :one WITH build_times AS ( SELECT diff --git a/coderd/database/queries/telemetryitems.sql b/coderd/database/queries/telemetryitems.sql new file mode 100644 index 0000000000000..7b7349db59943 --- /dev/null +++ b/coderd/database/queries/telemetryitems.sql @@ -0,0 +1,15 @@ +-- name: InsertTelemetryItemIfNotExists :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO NOTHING; + +-- name: GetTelemetryItem :one +SELECT * FROM telemetry_items WHERE key = $1; + +-- name: UpsertTelemetryItem :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = $2, updated_at = NOW() WHERE telemetry_items.key = $1; + +-- name: GetTelemetryItems :many +SELECT * FROM telemetry_items; diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index f253aa98ec266..2e4b813e438b8 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -55,6 +55,7 @@ const ( UniqueTailnetCoordinatorsPkey UniqueConstraint = "tailnet_coordinators_pkey" // ALTER TABLE ONLY tailnet_coordinators ADD CONSTRAINT tailnet_coordinators_pkey PRIMARY KEY (id); UniqueTailnetPeersPkey UniqueConstraint = "tailnet_peers_pkey" // ALTER TABLE ONLY tailnet_peers ADD CONSTRAINT tailnet_peers_pkey PRIMARY KEY (id, coordinator_id); UniqueTailnetTunnelsPkey UniqueConstraint = "tailnet_tunnels_pkey" // ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_pkey PRIMARY KEY (coordinator_id, src_id, dst_id); + UniqueTelemetryItemsPkey UniqueConstraint = "telemetry_items_pkey" // ALTER TABLE ONLY telemetry_items ADD CONSTRAINT telemetry_items_pkey PRIMARY KEY (key); UniqueTemplateUsageStatsPkey UniqueConstraint = "template_usage_stats_pkey" // ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); UniqueTemplateVersionParametersTemplateVersionIDNameKey UniqueConstraint = "template_version_parameters_template_version_id_name_key" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionVariablesTemplateVersionIDNameKey UniqueConstraint = "template_version_variables_template_version_id_name_key" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_name_key UNIQUE (template_version_id, name); diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 497a1109c7db9..3b4bcb7d15ae6 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -579,6 +579,17 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) { } return nil }) + eg.Go(func() error { + items, err := r.options.Database.GetTelemetryItems(ctx) + if err != nil { + return xerrors.Errorf("get telemetry items: %w", err) + } + snapshot.TelemetryItems = make([]TelemetryItem, 0, len(items)) + for _, item := range items { + snapshot.TelemetryItems = append(snapshot.TelemetryItems, ConvertTelemetryItem(item)) + } + return nil + }) err := eg.Wait() if err != nil { @@ -985,6 +996,15 @@ func ConvertOrganization(org database.Organization) Organization { } } +func ConvertTelemetryItem(item database.TelemetryItem) TelemetryItem { + return TelemetryItem{ + Key: item.Key, + Value: item.Value, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + } +} + // Snapshot represents a point-in-time anonymized database dump. // Data is aggregated by latest on the server-side, so partial data // can be sent without issue. @@ -1012,6 +1032,7 @@ type Snapshot struct { Workspaces []Workspace `json:"workspaces"` NetworkEvents []NetworkEvent `json:"network_events"` Organizations []Organization `json:"organizations"` + TelemetryItems []TelemetryItem `json:"telemetry_items"` } // Deployment contains information about the host running Coder. @@ -1536,6 +1557,25 @@ type Organization struct { CreatedAt time.Time `json:"created_at"` } +type telemetryItemKey string + +// The comment below gets rid of the warning that the name "TelemetryItemKey" has +// the "Telemetry" prefix, and that stutters when you use it outside the package +// (telemetry.TelemetryItemKey...). "TelemetryItem" is the name of a database table, +// so it makes sense to use the "Telemetry" prefix. +// +//revive:disable:exported +const ( + TelemetryItemKeyHTMLFirstServedAt telemetryItemKey = "html_first_served_at" +) + +type TelemetryItem struct { + Key string `json:"key"` + Value string `json:"value"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + type noopReporter struct{} func (*noopReporter) Report(_ *Snapshot) {} diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index b892e28e89d58..e81cba3ac313f 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -75,6 +75,10 @@ func TestTelemetry(t *testing.T) { Health: database.WorkspaceAppHealthDisabled, OpenIn: database.WorkspaceAppOpenInSlimWindow, }) + _ = dbgen.TelemetryItem(t, db, database.TelemetryItem{ + Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt), + Value: time.Now().Format(time.RFC3339), + }) group := dbgen.Group(t, db, database.Group{}) _ = dbgen.GroupMember(t, db, database.GroupMemberTable{UserID: user.ID, GroupID: group.ID}) wsagent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{}) @@ -127,7 +131,7 @@ func TestTelemetry(t *testing.T) { require.Len(t, snapshot.WorkspaceProxies, 1) require.Len(t, snapshot.WorkspaceModules, 1) require.Len(t, snapshot.Organizations, 1) - + require.Len(t, snapshot.TelemetryItems, 1) wsa := snapshot.WorkspaceAgents[0] require.Len(t, wsa.Subsystems, 2) require.Equal(t, string(database.WorkspaceAgentSubsystemEnvbox), wsa.Subsystems[0]) @@ -316,6 +320,47 @@ func TestTelemetryInstallSource(t *testing.T) { require.Equal(t, "aws_marketplace", deployment.InstallSource) } +func TestTelemetryItem(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + db, _ := dbtestutil.NewDB(t) + key := testutil.GetRandomName(t) + value := time.Now().Format(time.RFC3339) + + err := db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ + Key: key, + Value: value, + }) + require.NoError(t, err) + + item, err := db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Key, key) + require.Equal(t, item.Value, value) + + // Inserting a new value should not update the existing value + err = db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ + Key: key, + Value: "new_value", + }) + require.NoError(t, err) + + item, err = db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Value, value) + + // Upserting a new value should update the existing value + err = db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{ + Key: key, + Value: "new_value", + }) + require.NoError(t, err) + + item, err = db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Value, "new_value") +} + func collectSnapshot(t *testing.T, db database.Store, addOptionsFn func(opts telemetry.Options) telemetry.Options) (*telemetry.Deployment, *telemetry.Snapshot) { t.Helper() deployment := make(chan *telemetry.Deployment, 64) diff --git a/site/site.go b/site/site.go index af66c01c6f896..3a85f7b3963ad 100644 --- a/site/site.go +++ b/site/site.go @@ -34,6 +34,7 @@ import ( "golang.org/x/sync/singleflight" "golang.org/x/xerrors" + "cdr.dev/slog" "github.com/coder/coder/v2/coderd/appearance" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" @@ -41,6 +42,7 @@ import ( "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/codersdk" ) @@ -81,6 +83,8 @@ type Options struct { BuildInfo codersdk.BuildInfoResponse AppearanceFetcher *atomic.Pointer[appearance.Fetcher] Entitlements *entitlements.Set + Telemetry telemetry.Reporter + Logger slog.Logger } func New(opts *Options) *Handler { @@ -183,6 +187,8 @@ type Handler struct { Entitlements *entitlements.Set Experiments atomic.Pointer[codersdk.Experiments] + + telemetryHTMLServedOnce sync.Once } func (h *Handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -321,12 +327,51 @@ func ShouldCacheFile(reqFile string) bool { return true } +// reportHTMLFirstServedAt sends a telemetry report when the first HTML is ever served. +// The purpose is to track the first time the first user opens the site. +func (h *Handler) reportHTMLFirstServedAt() { + // nolint:gocritic // Manipulating telemetry items is system-restricted. + // TODO(hugodutka): Add a telemetry context in RBAC. + ctx := dbauthz.AsSystemRestricted(context.Background()) + itemKey := string(telemetry.TelemetryItemKeyHTMLFirstServedAt) + _, err := h.opts.Database.GetTelemetryItem(ctx, itemKey) + if err == nil { + // If the value is already set, then we reported it before. + // We don't need to report it again. + return + } + if !errors.Is(err, sql.ErrNoRows) { + h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) + return + } + if err := h.opts.Database.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ + Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt), + Value: time.Now().Format(time.RFC3339), + }); err != nil { + h.opts.Logger.Debug(ctx, "failed to set telemetry html first served at", slog.Error(err)) + return + } + item, err := h.opts.Database.GetTelemetryItem(ctx, itemKey) + if err != nil { + h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) + return + } + h.opts.Telemetry.Report(&telemetry.Snapshot{ + TelemetryItems: []telemetry.TelemetryItem{telemetry.ConvertTelemetryItem(item)}, + }) +} + func (h *Handler) serveHTML(resp http.ResponseWriter, request *http.Request, reqPath string, state htmlState) bool { if data, err := h.renderHTMLWithState(request, reqPath, state); err == nil { if reqPath == "" { // Pass "index.html" to the ServeContent so the ServeContent sets the right content headers. reqPath = "index.html" } + // `Once` is used to reduce the volume of db calls and telemetry reports. + // It's fine to run the enclosed function multiple times, but it's unnecessary. + h.telemetryHTMLServedOnce.Do(func() { + go h.reportHTMLFirstServedAt() + }) http.ServeContent(resp, request, reqPath, time.Time{}, bytes.NewReader(data)) return true } diff --git a/site/site_test.go b/site/site_test.go index 8bee665a56ae3..63f3f9aa17226 100644 --- a/site/site_test.go +++ b/site/site_test.go @@ -27,8 +27,10 @@ import ( "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbmem" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/site" "github.com/coder/coder/v2/testutil" @@ -45,9 +47,10 @@ func TestInjection(t *testing.T) { binFs := http.FS(fstest.MapFS{}) db := dbmem.New() handler := site.New(&site.Options{ - BinFS: binFs, - Database: db, - SiteFS: siteFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFs, + Database: db, + SiteFS: siteFS, }) user := dbgen.User(t, db, database.User{}) @@ -101,9 +104,10 @@ func TestInjectionFailureProducesCleanHTML(t *testing.T) { }, } handler := site.New(&site.Options{ - BinFS: binFs, - Database: db, - SiteFS: siteFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFs, + Database: db, + SiteFS: siteFS, // No OAuth2 configs, refresh will fail. OAuth2Configs: &httpmw.OAuth2Configs{ @@ -147,9 +151,12 @@ func TestCaching(t *testing.T) { } binFS := http.FS(fstest.MapFS{}) + db, _ := dbtestutil.NewDB(t) srv := httptest.NewServer(site.New(&site.Options{ - BinFS: binFS, - SiteFS: rootFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFS, + SiteFS: rootFS, + Database: db, })) defer srv.Close() @@ -213,9 +220,12 @@ func TestServingFiles(t *testing.T) { } binFS := http.FS(fstest.MapFS{}) + db, _ := dbtestutil.NewDB(t) srv := httptest.NewServer(site.New(&site.Options{ - BinFS: binFS, - SiteFS: rootFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFS, + SiteFS: rootFS, + Database: db, })) defer srv.Close() @@ -473,6 +483,7 @@ func TestServingBin(t *testing.T) { } srv := httptest.NewServer(site.New(&site.Options{ + Telemetry: telemetry.NewNoop(), BinFS: binFS, BinHashes: binHashes, SiteFS: rootFS,