From 63246e84e8a432426e041b02847b8626519dee2c Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Wed, 29 Jan 2025 18:56:55 +0000 Subject: [PATCH 01/16] track the first time html is served in telemetry --- coderd/coderd.go | 2 + coderd/database/dbauthz/dbauthz.go | 14 +++++ coderd/database/dbauthz/dbauthz_test.go | 6 ++ coderd/database/dbmem/dbmem.go | 21 +++++++ coderd/database/dbmetrics/querymetrics.go | 14 +++++ coderd/database/dbmock/dbmock.go | 29 ++++++++++ coderd/database/querier.go | 2 + coderd/database/queries.sql.go | 22 ++++++++ coderd/database/queries/siteconfig.sql | 7 +++ coderd/telemetry/telemetry.go | 68 ++++++++++++++++------- coderd/telemetry/telemetry_test.go | 19 +++++++ site/site.go | 36 ++++++++++++ 12 files changed, 220 insertions(+), 20 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index e273b7afdb80f..be558797389b9 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -585,6 +585,8 @@ func New(options *Options) *API { AppearanceFetcher: &api.AppearanceFetcher, BuildInfo: buildInfo, Entitlements: options.Entitlements, + Telemetry: options.Telemetry, + Logger: options.Logger.Named("site"), }) api.SiteHandler.Experiments.Store(&experiments) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 2e12cab9d33e0..abb174bbca61b 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2096,6 +2096,13 @@ func (q *querier) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) return q.db.GetTailnetTunnelPeerIDs(ctx, srcID) } +func (q *querier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return "", err + } + return q.db.GetTelemetryHTMLFirstServedAt(ctx) +} + func (q *querier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil { return nil, err @@ -3428,6 +3435,13 @@ func (q *querier) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) return q.db.RevokeDBCryptKey(ctx, activeKeyDigest) } +func (q *querier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { + return err + } + return q.db.SetTelemetryHTMLFirstServedAt(ctx, value) +} + func (q *querier) TryAcquireLock(ctx context.Context, id int64) (bool, error) { return q.db.TryAcquireLock(ctx, id) } diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index fdbbcc8b34ca6..047b33168f0ae 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4224,6 +4224,12 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetWorkspaceModulesCreatedAfter", s.Subtest(func(db database.Store, check *expects) { check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead) })) + s.Run("GetTelemetryHTMLFirstServedAt", s.Subtest(func(db database.Store, check *expects) { + check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) + })) + s.Run("SetTelemetryHTMLFirstServedAt", s.Subtest(func(db database.Store, check *expects) { + check.Args(time.Now().Format(time.RFC3339)).Asserts(rbac.ResourceSystem, policy.ActionUpdate) + })) } func (s *MethodTestSuite) TestNotifications() { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 6b518c7696369..93c90c5b61b87 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -258,6 +258,7 @@ type data struct { defaultProxyDisplayName string defaultProxyIconURL string userStatusChanges []database.UserStatusChange + htmlFirstServedAt string } func tryPercentile(fs []float64, p float64) float64 { @@ -4330,6 +4331,16 @@ func (*FakeQuerier) GetTailnetTunnelPeerIDs(context.Context, uuid.UUID) ([]datab return nil, ErrUnimplemented } +func (q *FakeQuerier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + if q.htmlFirstServedAt == "" { + return "", sql.ErrNoRows + } + + return q.htmlFirstServedAt, nil +} + func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { @@ -9123,6 +9134,16 @@ func (q *FakeQuerier) RevokeDBCryptKey(_ context.Context, activeKeyDigest string return sql.ErrNoRows } +func (q *FakeQuerier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { + q.mutex.Lock() + defer q.mutex.Unlock() + if q.htmlFirstServedAt != "" { + return nil + } + q.htmlFirstServedAt = value + return nil +} + func (*FakeQuerier) TryAcquireLock(_ context.Context, _ int64) (bool, error) { return false, xerrors.New("TryAcquireLock must only be called within a transaction") } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index ba8a1f9cdc8a6..786dbbbc12839 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1134,6 +1134,13 @@ func (m queryMetricsStore) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uu return r0, r1 } +func (m queryMetricsStore) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { + start := time.Now() + r0, r1 := m.s.GetTelemetryHTMLFirstServedAt(ctx) + m.queryLatencies.WithLabelValues("GetTelemetryHTMLFirstServedAt").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateAppInsights(ctx, arg) @@ -2163,6 +2170,13 @@ func (m queryMetricsStore) RevokeDBCryptKey(ctx context.Context, activeKeyDigest return r0 } +func (m queryMetricsStore) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { + start := time.Now() + r0 := m.s.SetTelemetryHTMLFirstServedAt(ctx, value) + m.queryLatencies.WithLabelValues("SetTelemetryHTMLFirstServedAt").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { start := time.Now() ok, err := m.s.TryAcquireLock(ctx, pgTryAdvisoryXactLock) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index d2aa8aa6fa62e..0dace92efd68b 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2346,6 +2346,21 @@ func (mr *MockStoreMockRecorder) GetTailnetTunnelPeerIDs(ctx, srcID any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTailnetTunnelPeerIDs", reflect.TypeOf((*MockStore)(nil).GetTailnetTunnelPeerIDs), ctx, srcID) } +// GetTelemetryHTMLFirstServedAt mocks base method. +func (m *MockStore) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetryHTMLFirstServedAt", ctx) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTelemetryHTMLFirstServedAt indicates an expected call of GetTelemetryHTMLFirstServedAt. +func (mr *MockStoreMockRecorder) GetTelemetryHTMLFirstServedAt(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryHTMLFirstServedAt", reflect.TypeOf((*MockStore)(nil).GetTelemetryHTMLFirstServedAt), ctx) +} + // GetTemplateAppInsights mocks base method. func (m *MockStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { m.ctrl.T.Helper() @@ -4610,6 +4625,20 @@ func (mr *MockStoreMockRecorder) RevokeDBCryptKey(ctx, activeKeyDigest any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RevokeDBCryptKey", reflect.TypeOf((*MockStore)(nil).RevokeDBCryptKey), ctx, activeKeyDigest) } +// SetTelemetryHTMLFirstServedAt mocks base method. +func (m *MockStore) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetTelemetryHTMLFirstServedAt", ctx, value) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetTelemetryHTMLFirstServedAt indicates an expected call of SetTelemetryHTMLFirstServedAt. +func (mr *MockStoreMockRecorder) SetTelemetryHTMLFirstServedAt(ctx, value any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTelemetryHTMLFirstServedAt", reflect.TypeOf((*MockStore)(nil).SetTelemetryHTMLFirstServedAt), ctx, value) +} + // TryAcquireLock mocks base method. func (m *MockStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 132a7aea75bdd..138d5df2c7625 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -224,6 +224,7 @@ type sqlcQuerier interface { GetTailnetPeers(ctx context.Context, id uuid.UUID) ([]TailnetPeer, error) GetTailnetTunnelPeerBindings(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerBindingsRow, error) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerIDsRow, error) + GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) // GetTemplateAppInsights returns the aggregate usage of each app in a given // timeframe. The result can be filtered on template_ids, meaning only user data // from workspaces based on those templates will be included. @@ -449,6 +450,7 @@ type sqlcQuerier interface { RemoveUserFromAllGroups(ctx context.Context, userID uuid.UUID) error RemoveUserFromGroups(ctx context.Context, arg RemoveUserFromGroupsParams) ([]uuid.UUID, error) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) error + SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error // Non blocking lock. Returns true if the lock was acquired, false otherwise. // // This must be called from within a transaction. The lock will be automatically diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 38dbf1fbfd0bb..3a052666d674a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -7815,6 +7815,17 @@ func (q *sqlQuerier) GetRuntimeConfig(ctx context.Context, key string) (string, return value, err } +const getTelemetryHTMLFirstServedAt = `-- name: GetTelemetryHTMLFirstServedAt :one +SELECT value FROM site_configs WHERE key = 'telemetry_html_first_served_at' +` + +func (q *sqlQuerier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { + row := q.db.QueryRowContext(ctx, getTelemetryHTMLFirstServedAt) + var value string + err := row.Scan(&value) + return value, err +} + const insertDERPMeshKey = `-- name: InsertDERPMeshKey :exec INSERT INTO site_configs (key, value) VALUES ('derp_mesh_key', $1) ` @@ -7833,6 +7844,17 @@ func (q *sqlQuerier) InsertDeploymentID(ctx context.Context, value string) error return err } +const setTelemetryHTMLFirstServedAt = `-- name: SetTelemetryHTMLFirstServedAt :exec +INSERT INTO site_configs (key, value) +VALUES ('telemetry_html_first_served_at', $1) +ON CONFLICT (key) DO NOTHING +` + +func (q *sqlQuerier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { + _, err := q.db.ExecContext(ctx, setTelemetryHTMLFirstServedAt, value) + return err +} + const upsertAnnouncementBanners = `-- name: UpsertAnnouncementBanners :exec INSERT INTO site_configs (key, value) VALUES ('announcement_banners', $1) ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'announcement_banners' diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index e8d02372e5a4f..f400ce3610f18 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -107,3 +107,10 @@ ON CONFLICT (key) DO UPDATE SET value = $2 WHERE site_configs.key = $1; DELETE FROM site_configs WHERE site_configs.key = $1; +-- name: SetTelemetryHTMLFirstServedAt :exec +INSERT INTO site_configs (key, value) +VALUES ('telemetry_html_first_served_at', $1) +ON CONFLICT (key) DO NOTHING; + +-- name: GetTelemetryHTMLFirstServedAt :one +SELECT value FROM site_configs WHERE key = 'telemetry_html_first_served_at'; diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 497a1109c7db9..2ac9719332f08 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -99,6 +99,8 @@ type Reporter interface { // database. For example, if a new user is added, a snapshot can // contain just that user entry. Report(snapshot *Snapshot) + // ReportDeployment sends deployment information to the telemetry server. + ReportDeployment() Enabled() bool Close() } @@ -222,6 +224,12 @@ func (r *remoteReporter) reportWithDeployment() { r.reportSync(snapshot) } +func (r *remoteReporter) ReportDeployment() { + if err := r.deployment(); err != nil { + r.options.Logger.Debug(r.ctx, "failed to report deployment", slog.Error(err)) + } +} + // deployment collects host information and reports it to the telemetry server. func (r *remoteReporter) deployment() error { sysInfoHost, err := sysinfo.Host() @@ -250,26 +258,32 @@ func (r *remoteReporter) deployment() error { r.options.Logger.Debug(r.ctx, "check IDP org sync", slog.Error(err)) } + htmlFirstServedAt, err := getHTMLFirstServedAt(r.ctx, r.options.Database) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + r.options.Logger.Debug(r.ctx, "get telemetry html first served at", slog.Error(err)) + } + data, err := json.Marshal(&Deployment{ - ID: r.options.DeploymentID, - Architecture: sysInfo.Architecture, - BuiltinPostgres: r.options.BuiltinPostgres, - Containerized: containerized, - Config: r.options.DeploymentConfig, - Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - InstallSource: installSource, - Tunnel: r.options.Tunnel, - OSType: sysInfo.OS.Type, - OSFamily: sysInfo.OS.Family, - OSPlatform: sysInfo.OS.Platform, - OSName: sysInfo.OS.Name, - OSVersion: sysInfo.OS.Version, - CPUCores: runtime.NumCPU(), - MemoryTotal: mem.Total, - MachineID: sysInfo.UniqueID, - StartedAt: r.startedAt, - ShutdownAt: r.shutdownAt, - IDPOrgSync: &idpOrgSync, + ID: r.options.DeploymentID, + Architecture: sysInfo.Architecture, + BuiltinPostgres: r.options.BuiltinPostgres, + Containerized: containerized, + Config: r.options.DeploymentConfig, + Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + InstallSource: installSource, + Tunnel: r.options.Tunnel, + OSType: sysInfo.OS.Type, + OSFamily: sysInfo.OS.Family, + OSPlatform: sysInfo.OS.Platform, + OSName: sysInfo.OS.Name, + OSVersion: sysInfo.OS.Version, + CPUCores: runtime.NumCPU(), + MemoryTotal: mem.Total, + MachineID: sysInfo.UniqueID, + StartedAt: r.startedAt, + ShutdownAt: r.shutdownAt, + IDPOrgSync: &idpOrgSync, + HTMLFirstServedAt: htmlFirstServedAt, }) if err != nil { return xerrors.Errorf("marshal deployment: %w", err) @@ -330,6 +344,18 @@ func checkIDPOrgSync(ctx context.Context, db database.Store, values *codersdk.De return syncConfig.Field != "", nil } +func getHTMLFirstServedAt(ctx context.Context, db database.Store) (*time.Time, error) { + htmlFirstServedAtStr, err := db.GetTelemetryHTMLFirstServedAt(ctx) + if err != nil { + return nil, xerrors.Errorf("get telemetry html first served at: %w", err) + } + t, err := time.Parse(time.RFC3339, htmlFirstServedAtStr) + if err != nil { + return nil, xerrors.Errorf("parse telemetry html first served at: %w", err) + } + return &t, nil +} + // createSnapshot collects a full snapshot from the database. func (r *remoteReporter) createSnapshot() (*Snapshot, error) { var ( @@ -1036,7 +1062,8 @@ type Deployment struct { ShutdownAt *time.Time `json:"shutdown_at"` // While IDPOrgSync will always be set, it's nullable to make // the struct backwards compatible with older coder versions. - IDPOrgSync *bool `json:"idp_org_sync"` + IDPOrgSync *bool `json:"idp_org_sync"` + HTMLFirstServedAt *time.Time `json:"html_first_served_at"` } type APIKey struct { @@ -1541,3 +1568,4 @@ type noopReporter struct{} func (*noopReporter) Report(_ *Snapshot) {} func (*noopReporter) Enabled() bool { return false } func (*noopReporter) Close() {} +func (*noopReporter) ReportDeployment() {} diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index b892e28e89d58..e866b0415de6e 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -306,6 +306,25 @@ func TestTelemetry(t *testing.T) { deployment, _ = collectSnapshot(t, db, nil) require.True(t, *deployment.IDPOrgSync) }) + t.Run("HTMLFirstServedAt", func(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + deployment, _ := collectSnapshot(t, db, nil) + require.Nil(t, deployment.HTMLFirstServedAt) + + ctx := testutil.Context(t, testutil.WaitMedium) + now := time.Now().Format(time.RFC3339) + parsedNow, err := time.Parse(time.RFC3339, now) + require.NoError(t, err) + require.NoError(t, db.SetTelemetryHTMLFirstServedAt(ctx, now)) + deployment, _ = collectSnapshot(t, db, nil) + require.Equal(t, *deployment.HTMLFirstServedAt, parsedNow) + + // Test idempotency + require.NoError(t, db.SetTelemetryHTMLFirstServedAt(ctx, time.Now().Add(time.Hour).Format(time.RFC3339))) + deployment, _ = collectSnapshot(t, db, nil) + require.Equal(t, *deployment.HTMLFirstServedAt, parsedNow) + }) } // nolint:paralleltest diff --git a/site/site.go b/site/site.go index af66c01c6f896..ef97fbe7201ca 100644 --- a/site/site.go +++ b/site/site.go @@ -34,6 +34,7 @@ import ( "golang.org/x/sync/singleflight" "golang.org/x/xerrors" + "cdr.dev/slog" "github.com/coder/coder/v2/coderd/appearance" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" @@ -41,6 +42,7 @@ import ( "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/codersdk" ) @@ -81,6 +83,8 @@ type Options struct { BuildInfo codersdk.BuildInfoResponse AppearanceFetcher *atomic.Pointer[appearance.Fetcher] Entitlements *entitlements.Set + Telemetry telemetry.Reporter + Logger slog.Logger } func New(opts *Options) *Handler { @@ -183,6 +187,8 @@ type Handler struct { Entitlements *entitlements.Set Experiments atomic.Pointer[codersdk.Experiments] + + TelemetryHTMLServedOnce sync.Once } func (h *Handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -321,12 +327,42 @@ func ShouldCacheFile(reqFile string) bool { return true } +// reportHTMLFirstServedAt sends a telemetry report when the first HTML is ever served. +// The purpose is to track the first time the first user opens the site. +func (h *Handler) reportHTMLFirstServedAt() { + // `Once` is used to reduce the volume of db calls and telemetry reports. + // It's fine to run this multiple times, but it's unnecessary. + h.TelemetryHTMLServedOnce.Do(func() { + ctx := context.Background() + // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. + _, err := h.opts.Database.GetTelemetryHTMLFirstServedAt(dbauthz.AsSystemRestricted(ctx)) + if err == nil { + // If the value is already set, then we reported it before. + // We don't need to report it again. + return + } + if !errors.Is(err, sql.ErrNoRows) { + h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) + return + } + // SetTelemetryHTMLFirstServedAt is idempotent, so there's no harm in calling it multiple times, + // even across restarts. Once it's set for the first time, it will never be changed. + // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. + if err := h.opts.Database.SetTelemetryHTMLFirstServedAt(dbauthz.AsSystemRestricted(ctx), time.Now().Format(time.RFC3339)); err != nil { + h.opts.Logger.Debug(ctx, "failed to set telemetry html first served at", slog.Error(err)) + return + } + h.opts.Telemetry.ReportDeployment() + }) +} + func (h *Handler) serveHTML(resp http.ResponseWriter, request *http.Request, reqPath string, state htmlState) bool { if data, err := h.renderHTMLWithState(request, reqPath, state); err == nil { if reqPath == "" { // Pass "index.html" to the ServeContent so the ServeContent sets the right content headers. reqPath = "index.html" } + go h.reportHTMLFirstServedAt() http.ServeContent(resp, request, reqPath, time.Time{}, bytes.NewReader(data)) return true } From 32876a2aed4ff36b720aba675c032f1f8a430c91 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 12:58:00 +0000 Subject: [PATCH 02/16] add the telemetry_items table --- coderd/database/dbauthz/dbauthz.go | 21 ++++++ coderd/database/dbauthz/dbauthz_test.go | 15 +++++ coderd/database/dbmem/dbmem.go | 66 +++++++++++++++++++ coderd/database/dbmetrics/querymetrics.go | 21 ++++++ coderd/database/dbmock/dbmock.go | 43 ++++++++++++ coderd/database/dump.sql | 9 +++ .../000288_telemetry_items.down.sql | 1 + .../migrations/000288_telemetry_items.up.sql | 8 +++ coderd/database/models.go | 7 ++ coderd/database/querier.go | 3 + coderd/database/queries.sql.go | 48 ++++++++++++++ coderd/database/queries/telemetryitems.sql | 12 ++++ coderd/database/unique_constraint.go | 1 + 13 files changed, 255 insertions(+) create mode 100644 coderd/database/migrations/000288_telemetry_items.down.sql create mode 100644 coderd/database/migrations/000288_telemetry_items.up.sql create mode 100644 coderd/database/queries/telemetryitems.sql diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index abb174bbca61b..a1857dd0eac1f 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2103,6 +2103,13 @@ func (q *querier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, er return q.db.GetTelemetryHTMLFirstServedAt(ctx) } +func (q *querier) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return database.TelemetryItem{}, err + } + return q.db.GetTelemetryItem(ctx, key) +} + func (q *querier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil { return nil, err @@ -3092,6 +3099,13 @@ func (q *querier) InsertReplica(ctx context.Context, arg database.InsertReplicaP return q.db.InsertReplica(ctx, arg) } +func (q *querier) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil { + return err + } + return q.db.InsertTelemetryItemIfNotExists(ctx, arg) +} + func (q *querier) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { obj := rbac.ResourceTemplate.InOrg(arg.OrganizationID) if err := q.authorizeContext(ctx, policy.ActionCreate, obj); err != nil { @@ -4359,6 +4373,13 @@ func (q *querier) UpsertTailnetTunnel(ctx context.Context, arg database.UpsertTa return q.db.UpsertTailnetTunnel(ctx, arg) } +func (q *querier) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { + return err + } + return q.db.UpsertTelemetryItem(ctx, arg) +} + func (q *querier) UpsertTemplateUsageStats(ctx context.Context) error { if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { return err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 047b33168f0ae..4f6bd467aeadb 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4230,6 +4230,21 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("SetTelemetryHTMLFirstServedAt", s.Subtest(func(db database.Store, check *expects) { check.Args(time.Now().Format(time.RFC3339)).Asserts(rbac.ResourceSystem, policy.ActionUpdate) })) + s.Run("GetTelemetryItem", s.Subtest(func(db database.Store, check *expects) { + check.Args("test").Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) + })) + s.Run("InsertTelemetryItemIfNotExists", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.InsertTelemetryItemIfNotExistsParams{ + Key: "test", + Value: "value", + }).Asserts(rbac.ResourceSystem, policy.ActionCreate) + })) + s.Run("UpsertTelemetryItem", s.Subtest(func(db database.Store, check *expects) { + check.Args(database.UpsertTelemetryItemParams{ + Key: "test", + Value: "value", + }).Asserts(rbac.ResourceSystem, policy.ActionUpdate) + })) } func (s *MethodTestSuite) TestNotifications() { diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 93c90c5b61b87..b3efd24e2f78c 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -89,6 +89,7 @@ func New() database.Store { locks: map[int64]struct{}{}, runtimeConfig: map[string]string{}, userStatusChanges: make([]database.UserStatusChange, 0), + telemetryItems: make([]database.TelemetryItem, 0), }, } // Always start with a default org. Matching migration 198. @@ -259,6 +260,7 @@ type data struct { defaultProxyIconURL string userStatusChanges []database.UserStatusChange htmlFirstServedAt string + telemetryItems []database.TelemetryItem } func tryPercentile(fs []float64, p float64) float64 { @@ -4341,6 +4343,19 @@ func (q *FakeQuerier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string return q.htmlFirstServedAt, nil } +func (q *FakeQuerier) GetTelemetryItem(_ context.Context, key string) (database.TelemetryItem, error) { + q.mutex.RLock() + defer q.mutex.RUnlock() + + for _, item := range q.telemetryItems { + if item.Key == key { + return item, nil + } + } + + return database.TelemetryItem{}, sql.ErrNoRows +} + func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { @@ -8131,6 +8146,30 @@ func (q *FakeQuerier) InsertReplica(_ context.Context, arg database.InsertReplic return replica, nil } +func (q *FakeQuerier) InsertTelemetryItemIfNotExists(_ context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + err := validateDatabaseType(arg) + if err != nil { + return err + } + + q.mutex.Lock() + defer q.mutex.Unlock() + + for _, item := range q.telemetryItems { + if item.Key == arg.Key { + return nil + } + } + + q.telemetryItems = append(q.telemetryItems, database.TelemetryItem{ + Key: arg.Key, + Value: arg.Value, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + return nil +} + func (q *FakeQuerier) InsertTemplate(_ context.Context, arg database.InsertTemplateParams) error { if err := validateDatabaseType(arg); err != nil { return err @@ -10895,6 +10934,33 @@ func (*FakeQuerier) UpsertTailnetTunnel(_ context.Context, arg database.UpsertTa return database.TailnetTunnel{}, ErrUnimplemented } +func (q *FakeQuerier) UpsertTelemetryItem(_ context.Context, arg database.UpsertTelemetryItemParams) error { + err := validateDatabaseType(arg) + if err != nil { + return err + } + + q.mutex.Lock() + defer q.mutex.Unlock() + + for i, item := range q.telemetryItems { + if item.Key == arg.Key { + q.telemetryItems[i].Value = arg.Value + q.telemetryItems[i].UpdatedAt = time.Now() + return nil + } + } + + q.telemetryItems = append(q.telemetryItems, database.TelemetryItem{ + Key: arg.Key, + Value: arg.Value, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + + return nil +} + func (q *FakeQuerier) UpsertTemplateUsageStats(ctx context.Context) error { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 786dbbbc12839..51d6572e63e21 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1141,6 +1141,13 @@ func (m queryMetricsStore) GetTelemetryHTMLFirstServedAt(ctx context.Context) (s return r0, r1 } +func (m queryMetricsStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + start := time.Now() + r0, r1 := m.s.GetTelemetryItem(ctx, key) + m.queryLatencies.WithLabelValues("GetTelemetryItem").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateAppInsights(ctx, arg) @@ -1918,6 +1925,13 @@ func (m queryMetricsStore) InsertReplica(ctx context.Context, arg database.Inser return replica, err } +func (m queryMetricsStore) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + start := time.Now() + r0 := m.s.InsertTelemetryItemIfNotExists(ctx, arg) + m.queryLatencies.WithLabelValues("InsertTelemetryItemIfNotExists").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { start := time.Now() err := m.s.InsertTemplate(ctx, arg) @@ -2786,6 +2800,13 @@ func (m queryMetricsStore) UpsertTailnetTunnel(ctx context.Context, arg database return r0, r1 } +func (m queryMetricsStore) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + start := time.Now() + r0 := m.s.UpsertTelemetryItem(ctx, arg) + m.queryLatencies.WithLabelValues("UpsertTelemetryItem").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) UpsertTemplateUsageStats(ctx context.Context) error { start := time.Now() r0 := m.s.UpsertTemplateUsageStats(ctx) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 0dace92efd68b..2167c9834da81 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2361,6 +2361,21 @@ func (mr *MockStoreMockRecorder) GetTelemetryHTMLFirstServedAt(ctx any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryHTMLFirstServedAt", reflect.TypeOf((*MockStore)(nil).GetTelemetryHTMLFirstServedAt), ctx) } +// GetTelemetryItem mocks base method. +func (m *MockStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetryItem", ctx, key) + ret0, _ := ret[0].(database.TelemetryItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTelemetryItem indicates an expected call of GetTelemetryItem. +func (mr *MockStoreMockRecorder) GetTelemetryItem(ctx, key any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItem", reflect.TypeOf((*MockStore)(nil).GetTelemetryItem), ctx, key) +} + // GetTemplateAppInsights mocks base method. func (m *MockStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { m.ctrl.T.Helper() @@ -4066,6 +4081,20 @@ func (mr *MockStoreMockRecorder) InsertReplica(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertReplica", reflect.TypeOf((*MockStore)(nil).InsertReplica), ctx, arg) } +// InsertTelemetryItemIfNotExists mocks base method. +func (m *MockStore) InsertTelemetryItemIfNotExists(ctx context.Context, arg database.InsertTelemetryItemIfNotExistsParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertTelemetryItemIfNotExists", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// InsertTelemetryItemIfNotExists indicates an expected call of InsertTelemetryItemIfNotExists. +func (mr *MockStoreMockRecorder) InsertTelemetryItemIfNotExists(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertTelemetryItemIfNotExists", reflect.TypeOf((*MockStore)(nil).InsertTelemetryItemIfNotExists), ctx, arg) +} + // InsertTemplate mocks base method. func (m *MockStore) InsertTemplate(ctx context.Context, arg database.InsertTemplateParams) error { m.ctrl.T.Helper() @@ -5890,6 +5919,20 @@ func (mr *MockStoreMockRecorder) UpsertTailnetTunnel(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTailnetTunnel", reflect.TypeOf((*MockStore)(nil).UpsertTailnetTunnel), ctx, arg) } +// UpsertTelemetryItem mocks base method. +func (m *MockStore) UpsertTelemetryItem(ctx context.Context, arg database.UpsertTelemetryItemParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertTelemetryItem", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertTelemetryItem indicates an expected call of UpsertTelemetryItem. +func (mr *MockStoreMockRecorder) UpsertTelemetryItem(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTelemetryItem", reflect.TypeOf((*MockStore)(nil).UpsertTelemetryItem), ctx, arg) +} + // UpsertTemplateUsageStats mocks base method. func (m *MockStore) UpsertTemplateUsageStats(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c241548e166c2..c9f44058dd889 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1164,6 +1164,13 @@ CREATE TABLE tailnet_tunnels ( updated_at timestamp with time zone NOT NULL ); +CREATE TABLE telemetry_items ( + key text NOT NULL, + value text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + CREATE TABLE template_usage_stats ( start_time timestamp with time zone NOT NULL, end_time timestamp with time zone NOT NULL, @@ -2184,6 +2191,8 @@ CREATE INDEX provisioner_jobs_started_at_idx ON provisioner_jobs USING btree (st CREATE UNIQUE INDEX provisioner_keys_organization_id_name_idx ON provisioner_keys USING btree (organization_id, lower((name)::text)); +CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items USING btree (key); + CREATE INDEX template_usage_stats_start_time_idx ON template_usage_stats USING btree (start_time DESC); COMMENT ON INDEX template_usage_stats_start_time_idx IS 'Index for querying MAX(start_time).'; diff --git a/coderd/database/migrations/000288_telemetry_items.down.sql b/coderd/database/migrations/000288_telemetry_items.down.sql new file mode 100644 index 0000000000000..118188f519e76 --- /dev/null +++ b/coderd/database/migrations/000288_telemetry_items.down.sql @@ -0,0 +1 @@ +DROP TABLE telemetry_items; diff --git a/coderd/database/migrations/000288_telemetry_items.up.sql b/coderd/database/migrations/000288_telemetry_items.up.sql new file mode 100644 index 0000000000000..aefe921f611e9 --- /dev/null +++ b/coderd/database/migrations/000288_telemetry_items.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE telemetry_items ( + key TEXT NOT NULL, + value TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items (key); diff --git a/coderd/database/models.go b/coderd/database/models.go index b0a487c192793..9769bde33052b 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -2787,6 +2787,13 @@ type TailnetTunnel struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } +type TelemetryItem struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + // Joins in the display name information such as username, avatar, and organization name. type Template struct { ID uuid.UUID `db:"id" json:"id"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 138d5df2c7625..bd5aec10c57ef 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -225,6 +225,7 @@ type sqlcQuerier interface { GetTailnetTunnelPeerBindings(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerBindingsRow, error) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerIDsRow, error) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) + GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) // GetTemplateAppInsights returns the aggregate usage of each app in a given // timeframe. The result can be filtered on template_ids, meaning only user data // from workspaces based on those templates will be included. @@ -405,6 +406,7 @@ type sqlcQuerier interface { InsertProvisionerJobTimings(ctx context.Context, arg InsertProvisionerJobTimingsParams) ([]ProvisionerJobTiming, error) InsertProvisionerKey(ctx context.Context, arg InsertProvisionerKeyParams) (ProvisionerKey, error) InsertReplica(ctx context.Context, arg InsertReplicaParams) (Replica, error) + InsertTelemetryItemIfNotExists(ctx context.Context, arg InsertTelemetryItemIfNotExistsParams) error InsertTemplate(ctx context.Context, arg InsertTemplateParams) error InsertTemplateVersion(ctx context.Context, arg InsertTemplateVersionParams) error InsertTemplateVersionParameter(ctx context.Context, arg InsertTemplateVersionParameterParams) (TemplateVersionParameter, error) @@ -548,6 +550,7 @@ type sqlcQuerier interface { UpsertTailnetCoordinator(ctx context.Context, id uuid.UUID) (TailnetCoordinator, error) UpsertTailnetPeer(ctx context.Context, arg UpsertTailnetPeerParams) (TailnetPeer, error) UpsertTailnetTunnel(ctx context.Context, arg UpsertTailnetTunnelParams) (TailnetTunnel, error) + UpsertTelemetryItem(ctx context.Context, arg UpsertTelemetryItemParams) error // This query aggregates the workspace_agent_stats and workspace_app_stats data // into a single table for efficient storage and querying. Half-hour buckets are // used to store the data, and the minutes are summed for each user and template diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 3a052666d674a..45cf84205f439 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8724,6 +8724,54 @@ func (q *sqlQuerier) UpsertTailnetTunnel(ctx context.Context, arg UpsertTailnetT return i, err } +const getTelemetryItem = `-- name: GetTelemetryItem :one +SELECT key, value, created_at, updated_at FROM telemetry_items WHERE key = $1 +` + +func (q *sqlQuerier) GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) { + row := q.db.QueryRowContext(ctx, getTelemetryItem, key) + var i TelemetryItem + err := row.Scan( + &i.Key, + &i.Value, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const insertTelemetryItemIfNotExists = `-- name: InsertTelemetryItemIfNotExists :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO NOTHING +` + +type InsertTelemetryItemIfNotExistsParams struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + +func (q *sqlQuerier) InsertTelemetryItemIfNotExists(ctx context.Context, arg InsertTelemetryItemIfNotExistsParams) error { + _, err := q.db.ExecContext(ctx, insertTelemetryItemIfNotExists, arg.Key, arg.Value) + return err +} + +const upsertTelemetryItem = `-- name: UpsertTelemetryItem :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = $2, updated_at = NOW() WHERE telemetry_items.key = $1 +` + +type UpsertTelemetryItemParams struct { + Key string `db:"key" json:"key"` + Value string `db:"value" json:"value"` +} + +func (q *sqlQuerier) UpsertTelemetryItem(ctx context.Context, arg UpsertTelemetryItemParams) error { + _, err := q.db.ExecContext(ctx, upsertTelemetryItem, arg.Key, arg.Value) + return err +} + const getTemplateAverageBuildTime = `-- name: GetTemplateAverageBuildTime :one WITH build_times AS ( SELECT diff --git a/coderd/database/queries/telemetryitems.sql b/coderd/database/queries/telemetryitems.sql new file mode 100644 index 0000000000000..1513b9701a4ff --- /dev/null +++ b/coderd/database/queries/telemetryitems.sql @@ -0,0 +1,12 @@ +-- name: InsertTelemetryItemIfNotExists :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO NOTHING; + +-- name: GetTelemetryItem :one +SELECT * FROM telemetry_items WHERE key = $1; + +-- name: UpsertTelemetryItem :exec +INSERT INTO telemetry_items (key, value) +VALUES ($1, $2) +ON CONFLICT (key) DO UPDATE SET value = $2, updated_at = NOW() WHERE telemetry_items.key = $1; diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index f253aa98ec266..5d3a365c15cf4 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -97,6 +97,7 @@ const ( UniqueNotificationMessagesDedupeHashIndex UniqueConstraint = "notification_messages_dedupe_hash_idx" // CREATE UNIQUE INDEX notification_messages_dedupe_hash_idx ON notification_messages USING btree (dedupe_hash); UniqueOrganizationsSingleDefaultOrg UniqueConstraint = "organizations_single_default_org" // CREATE UNIQUE INDEX organizations_single_default_org ON organizations USING btree (is_default) WHERE (is_default = true); UniqueProvisionerKeysOrganizationIDNameIndex UniqueConstraint = "provisioner_keys_organization_id_name_idx" // CREATE UNIQUE INDEX provisioner_keys_organization_id_name_idx ON provisioner_keys USING btree (organization_id, lower((name)::text)); + UniqueTelemetryItemsKeyIndex UniqueConstraint = "telemetry_items_key_idx" // CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items USING btree (key); UniqueTemplateUsageStatsStartTimeTemplateIDUserIDIndex UniqueConstraint = "template_usage_stats_start_time_template_id_user_id_idx" // CREATE UNIQUE INDEX template_usage_stats_start_time_template_id_user_id_idx ON template_usage_stats USING btree (start_time, template_id, user_id); UniqueTemplatesOrganizationIDNameIndex UniqueConstraint = "templates_organization_id_name_idx" // CREATE UNIQUE INDEX templates_organization_id_name_idx ON templates USING btree (organization_id, lower((name)::text)) WHERE (deleted = false); UniqueUserLinksLinkedIDLoginTypeIndex UniqueConstraint = "user_links_linked_id_login_type_idx" // CREATE UNIQUE INDEX user_links_linked_id_login_type_idx ON user_links USING btree (linked_id, login_type) WHERE (linked_id <> ''::text); From f68a163a81306c90bbe45c359e01cfda3c596b9b Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:05:40 +0000 Subject: [PATCH 03/16] add the GetTelemetryItems query --- coderd/database/dbauthz/dbauthz.go | 7 +++++ coderd/database/dbauthz/dbauthz_test.go | 3 ++ coderd/database/dbmem/dbmem.go | 4 +++ coderd/database/dbmetrics/querymetrics.go | 7 +++++ coderd/database/dbmock/dbmock.go | 15 ++++++++++ coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 32 ++++++++++++++++++++++ coderd/database/queries/telemetryitems.sql | 3 ++ 8 files changed, 72 insertions(+) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index a1857dd0eac1f..804cdab6656a3 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2110,6 +2110,13 @@ func (q *querier) GetTelemetryItem(ctx context.Context, key string) (database.Te return q.db.GetTelemetryItem(ctx, key) } +func (q *querier) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return nil, err + } + return q.db.GetTelemetryItems(ctx) +} + func (q *querier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil { return nil, err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 4f6bd467aeadb..90ba26eab9ad4 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4233,6 +4233,9 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetTelemetryItem", s.Subtest(func(db database.Store, check *expects) { check.Args("test").Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) })) + s.Run("GetTelemetryItems", s.Subtest(func(db database.Store, check *expects) { + check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead) + })) s.Run("InsertTelemetryItemIfNotExists", s.Subtest(func(db database.Store, check *expects) { check.Args(database.InsertTelemetryItemIfNotExistsParams{ Key: "test", diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index b3efd24e2f78c..931e836a9acc7 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -4356,6 +4356,10 @@ func (q *FakeQuerier) GetTelemetryItem(_ context.Context, key string) (database. return database.TelemetryItem{}, sql.ErrNoRows } +func (q *FakeQuerier) GetTelemetryItems(_ context.Context) ([]database.TelemetryItem, error) { + return q.telemetryItems, nil +} + func (q *FakeQuerier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { err := validateDatabaseType(arg) if err != nil { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 51d6572e63e21..434a124eca0d2 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1148,6 +1148,13 @@ func (m queryMetricsStore) GetTelemetryItem(ctx context.Context, key string) (da return r0, r1 } +func (m queryMetricsStore) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + start := time.Now() + r0, r1 := m.s.GetTelemetryItems(ctx) + m.queryLatencies.WithLabelValues("GetTelemetryItems").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { start := time.Now() r0, r1 := m.s.GetTemplateAppInsights(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 2167c9834da81..84bedc470ea9c 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2376,6 +2376,21 @@ func (mr *MockStoreMockRecorder) GetTelemetryItem(ctx, key any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItem", reflect.TypeOf((*MockStore)(nil).GetTelemetryItem), ctx, key) } +// GetTelemetryItems mocks base method. +func (m *MockStore) GetTelemetryItems(ctx context.Context) ([]database.TelemetryItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetryItems", ctx) + ret0, _ := ret[0].([]database.TelemetryItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTelemetryItems indicates an expected call of GetTelemetryItems. +func (mr *MockStoreMockRecorder) GetTelemetryItems(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItems", reflect.TypeOf((*MockStore)(nil).GetTelemetryItems), ctx) +} + // GetTemplateAppInsights mocks base method. func (m *MockStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index bd5aec10c57ef..5e72b64b7480e 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -226,6 +226,7 @@ type sqlcQuerier interface { GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerIDsRow, error) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) + GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error) // GetTemplateAppInsights returns the aggregate usage of each app in a given // timeframe. The result can be filtered on template_ids, meaning only user data // from workspaces based on those templates will be included. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 45cf84205f439..66f5f6a98c73c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8740,6 +8740,38 @@ func (q *sqlQuerier) GetTelemetryItem(ctx context.Context, key string) (Telemetr return i, err } +const getTelemetryItems = `-- name: GetTelemetryItems :many +SELECT key, value, created_at, updated_at FROM telemetry_items +` + +func (q *sqlQuerier) GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error) { + rows, err := q.db.QueryContext(ctx, getTelemetryItems) + if err != nil { + return nil, err + } + defer rows.Close() + var items []TelemetryItem + for rows.Next() { + var i TelemetryItem + if err := rows.Scan( + &i.Key, + &i.Value, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertTelemetryItemIfNotExists = `-- name: InsertTelemetryItemIfNotExists :exec INSERT INTO telemetry_items (key, value) VALUES ($1, $2) diff --git a/coderd/database/queries/telemetryitems.sql b/coderd/database/queries/telemetryitems.sql index 1513b9701a4ff..7b7349db59943 100644 --- a/coderd/database/queries/telemetryitems.sql +++ b/coderd/database/queries/telemetryitems.sql @@ -10,3 +10,6 @@ SELECT * FROM telemetry_items WHERE key = $1; INSERT INTO telemetry_items (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = $2, updated_at = NOW() WHERE telemetry_items.key = $1; + +-- name: GetTelemetryItems :many +SELECT * FROM telemetry_items; From 6554aa31eface2a15ebee877972b66f1255ca96e Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:12:21 +0000 Subject: [PATCH 04/16] change TelemetryItem's key to PK --- coderd/database/dump.sql | 5 +++-- coderd/database/migrations/000288_telemetry_items.up.sql | 4 +--- coderd/database/unique_constraint.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c9f44058dd889..9cc38adf23b6b 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -2033,6 +2033,9 @@ ALTER TABLE ONLY tailnet_peers ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_pkey PRIMARY KEY (coordinator_id, src_id, dst_id); +ALTER TABLE ONLY telemetry_items + ADD CONSTRAINT telemetry_items_pkey PRIMARY KEY (key); + ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); @@ -2191,8 +2194,6 @@ CREATE INDEX provisioner_jobs_started_at_idx ON provisioner_jobs USING btree (st CREATE UNIQUE INDEX provisioner_keys_organization_id_name_idx ON provisioner_keys USING btree (organization_id, lower((name)::text)); -CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items USING btree (key); - CREATE INDEX template_usage_stats_start_time_idx ON template_usage_stats USING btree (start_time DESC); COMMENT ON INDEX template_usage_stats_start_time_idx IS 'Index for querying MAX(start_time).'; diff --git a/coderd/database/migrations/000288_telemetry_items.up.sql b/coderd/database/migrations/000288_telemetry_items.up.sql index aefe921f611e9..40279827788d6 100644 --- a/coderd/database/migrations/000288_telemetry_items.up.sql +++ b/coderd/database/migrations/000288_telemetry_items.up.sql @@ -1,8 +1,6 @@ CREATE TABLE telemetry_items ( - key TEXT NOT NULL, + key TEXT NOT NULL PRIMARY KEY, value TEXT NOT NULL, created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() ); - -CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items (key); diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index 5d3a365c15cf4..2e4b813e438b8 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -55,6 +55,7 @@ const ( UniqueTailnetCoordinatorsPkey UniqueConstraint = "tailnet_coordinators_pkey" // ALTER TABLE ONLY tailnet_coordinators ADD CONSTRAINT tailnet_coordinators_pkey PRIMARY KEY (id); UniqueTailnetPeersPkey UniqueConstraint = "tailnet_peers_pkey" // ALTER TABLE ONLY tailnet_peers ADD CONSTRAINT tailnet_peers_pkey PRIMARY KEY (id, coordinator_id); UniqueTailnetTunnelsPkey UniqueConstraint = "tailnet_tunnels_pkey" // ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_pkey PRIMARY KEY (coordinator_id, src_id, dst_id); + UniqueTelemetryItemsPkey UniqueConstraint = "telemetry_items_pkey" // ALTER TABLE ONLY telemetry_items ADD CONSTRAINT telemetry_items_pkey PRIMARY KEY (key); UniqueTemplateUsageStatsPkey UniqueConstraint = "template_usage_stats_pkey" // ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); UniqueTemplateVersionParametersTemplateVersionIDNameKey UniqueConstraint = "template_version_parameters_template_version_id_name_key" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionVariablesTemplateVersionIDNameKey UniqueConstraint = "template_version_variables_template_version_id_name_key" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_name_key UNIQUE (template_version_id, name); @@ -97,7 +98,6 @@ const ( UniqueNotificationMessagesDedupeHashIndex UniqueConstraint = "notification_messages_dedupe_hash_idx" // CREATE UNIQUE INDEX notification_messages_dedupe_hash_idx ON notification_messages USING btree (dedupe_hash); UniqueOrganizationsSingleDefaultOrg UniqueConstraint = "organizations_single_default_org" // CREATE UNIQUE INDEX organizations_single_default_org ON organizations USING btree (is_default) WHERE (is_default = true); UniqueProvisionerKeysOrganizationIDNameIndex UniqueConstraint = "provisioner_keys_organization_id_name_idx" // CREATE UNIQUE INDEX provisioner_keys_organization_id_name_idx ON provisioner_keys USING btree (organization_id, lower((name)::text)); - UniqueTelemetryItemsKeyIndex UniqueConstraint = "telemetry_items_key_idx" // CREATE UNIQUE INDEX telemetry_items_key_idx ON telemetry_items USING btree (key); UniqueTemplateUsageStatsStartTimeTemplateIDUserIDIndex UniqueConstraint = "template_usage_stats_start_time_template_id_user_id_idx" // CREATE UNIQUE INDEX template_usage_stats_start_time_template_id_user_id_idx ON template_usage_stats USING btree (start_time, template_id, user_id); UniqueTemplatesOrganizationIDNameIndex UniqueConstraint = "templates_organization_id_name_idx" // CREATE UNIQUE INDEX templates_organization_id_name_idx ON templates USING btree (organization_id, lower((name)::text)) WHERE (deleted = false); UniqueUserLinksLinkedIDLoginTypeIndex UniqueConstraint = "user_links_linked_id_login_type_idx" // CREATE UNIQUE INDEX user_links_linked_id_login_type_idx ON user_links USING btree (linked_id, login_type) WHERE (linked_id <> ''::text); From 0a95dcd2aba52cba452f92cd69ba7cf33620af38 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:33:02 +0000 Subject: [PATCH 05/16] report html first served at with a TelemetryItem --- coderd/database/dbgen/dbgen.go | 17 +++++++ coderd/telemetry/telemetry.go | 82 ++++++++++++++++++++---------- coderd/telemetry/telemetry_test.go | 66 ++++++++++++++++-------- site/site.go | 58 ++++++++++++--------- 4 files changed, 152 insertions(+), 71 deletions(-) diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 566540dcb2906..54e4f99959b44 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -1093,6 +1093,23 @@ func ProvisionerJobTimings(t testing.TB, db database.Store, build database.Works return timings } +func TelemetryItem(t testing.TB, db database.Store, seed database.TelemetryItem) database.TelemetryItem { + if seed.Key == "" { + seed.Key = testutil.GetRandomName(t) + } + if seed.Value == "" { + seed.Value = time.Now().Format(time.RFC3339) + } + err := db.UpsertTelemetryItem(genCtx, database.UpsertTelemetryItemParams{ + Key: seed.Key, + Value: seed.Value, + }) + require.NoError(t, err, "upsert telemetry item") + item, err := db.GetTelemetryItem(genCtx, seed.Key) + require.NoError(t, err, "get telemetry item") + return item +} + func provisionerJobTiming(t testing.TB, db database.Store, seed database.ProvisionerJobTiming) database.ProvisionerJobTiming { timing, err := db.InsertProvisionerJobTimings(genCtx, database.InsertProvisionerJobTimingsParams{ JobID: takeFirst(seed.JobID, uuid.New()), diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 2ac9719332f08..fe26f0b3aa65c 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -258,32 +258,26 @@ func (r *remoteReporter) deployment() error { r.options.Logger.Debug(r.ctx, "check IDP org sync", slog.Error(err)) } - htmlFirstServedAt, err := getHTMLFirstServedAt(r.ctx, r.options.Database) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - r.options.Logger.Debug(r.ctx, "get telemetry html first served at", slog.Error(err)) - } - data, err := json.Marshal(&Deployment{ - ID: r.options.DeploymentID, - Architecture: sysInfo.Architecture, - BuiltinPostgres: r.options.BuiltinPostgres, - Containerized: containerized, - Config: r.options.DeploymentConfig, - Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - InstallSource: installSource, - Tunnel: r.options.Tunnel, - OSType: sysInfo.OS.Type, - OSFamily: sysInfo.OS.Family, - OSPlatform: sysInfo.OS.Platform, - OSName: sysInfo.OS.Name, - OSVersion: sysInfo.OS.Version, - CPUCores: runtime.NumCPU(), - MemoryTotal: mem.Total, - MachineID: sysInfo.UniqueID, - StartedAt: r.startedAt, - ShutdownAt: r.shutdownAt, - IDPOrgSync: &idpOrgSync, - HTMLFirstServedAt: htmlFirstServedAt, + ID: r.options.DeploymentID, + Architecture: sysInfo.Architecture, + BuiltinPostgres: r.options.BuiltinPostgres, + Containerized: containerized, + Config: r.options.DeploymentConfig, + Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + InstallSource: installSource, + Tunnel: r.options.Tunnel, + OSType: sysInfo.OS.Type, + OSFamily: sysInfo.OS.Family, + OSPlatform: sysInfo.OS.Platform, + OSName: sysInfo.OS.Name, + OSVersion: sysInfo.OS.Version, + CPUCores: runtime.NumCPU(), + MemoryTotal: mem.Total, + MachineID: sysInfo.UniqueID, + StartedAt: r.startedAt, + ShutdownAt: r.shutdownAt, + IDPOrgSync: &idpOrgSync, }) if err != nil { return xerrors.Errorf("marshal deployment: %w", err) @@ -605,6 +599,17 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) { } return nil }) + eg.Go(func() error { + items, err := r.options.Database.GetTelemetryItems(ctx) + if err != nil { + return xerrors.Errorf("get telemetry items: %w", err) + } + snapshot.TelemetryItems = make([]TelemetryItem, 0, len(items)) + for _, item := range items { + snapshot.TelemetryItems = append(snapshot.TelemetryItems, ConvertTelemetryItem(item)) + } + return nil + }) err := eg.Wait() if err != nil { @@ -1011,6 +1016,15 @@ func ConvertOrganization(org database.Organization) Organization { } } +func ConvertTelemetryItem(item database.TelemetryItem) TelemetryItem { + return TelemetryItem{ + Key: item.Key, + Value: item.Value, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, + } +} + // Snapshot represents a point-in-time anonymized database dump. // Data is aggregated by latest on the server-side, so partial data // can be sent without issue. @@ -1038,6 +1052,7 @@ type Snapshot struct { Workspaces []Workspace `json:"workspaces"` NetworkEvents []NetworkEvent `json:"network_events"` Organizations []Organization `json:"organizations"` + TelemetryItems []TelemetryItem `json:"telemetry_items"` } // Deployment contains information about the host running Coder. @@ -1062,8 +1077,7 @@ type Deployment struct { ShutdownAt *time.Time `json:"shutdown_at"` // While IDPOrgSync will always be set, it's nullable to make // the struct backwards compatible with older coder versions. - IDPOrgSync *bool `json:"idp_org_sync"` - HTMLFirstServedAt *time.Time `json:"html_first_served_at"` + IDPOrgSync *bool `json:"idp_org_sync"` } type APIKey struct { @@ -1563,6 +1577,20 @@ type Organization struct { CreatedAt time.Time `json:"created_at"` } +//revive:disable:exported +type TelemetryItemKey string + +const ( + TelemetryItemKeyHTMLFirstServedAt TelemetryItemKey = "html_first_served_at" +) + +type TelemetryItem struct { + Key string `json:"key"` + Value string `json:"value"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + type noopReporter struct{} func (*noopReporter) Report(_ *Snapshot) {} diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index e866b0415de6e..e81cba3ac313f 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -75,6 +75,10 @@ func TestTelemetry(t *testing.T) { Health: database.WorkspaceAppHealthDisabled, OpenIn: database.WorkspaceAppOpenInSlimWindow, }) + _ = dbgen.TelemetryItem(t, db, database.TelemetryItem{ + Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt), + Value: time.Now().Format(time.RFC3339), + }) group := dbgen.Group(t, db, database.Group{}) _ = dbgen.GroupMember(t, db, database.GroupMemberTable{UserID: user.ID, GroupID: group.ID}) wsagent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{}) @@ -127,7 +131,7 @@ func TestTelemetry(t *testing.T) { require.Len(t, snapshot.WorkspaceProxies, 1) require.Len(t, snapshot.WorkspaceModules, 1) require.Len(t, snapshot.Organizations, 1) - + require.Len(t, snapshot.TelemetryItems, 1) wsa := snapshot.WorkspaceAgents[0] require.Len(t, wsa.Subsystems, 2) require.Equal(t, string(database.WorkspaceAgentSubsystemEnvbox), wsa.Subsystems[0]) @@ -306,25 +310,6 @@ func TestTelemetry(t *testing.T) { deployment, _ = collectSnapshot(t, db, nil) require.True(t, *deployment.IDPOrgSync) }) - t.Run("HTMLFirstServedAt", func(t *testing.T) { - t.Parallel() - db, _ := dbtestutil.NewDB(t) - deployment, _ := collectSnapshot(t, db, nil) - require.Nil(t, deployment.HTMLFirstServedAt) - - ctx := testutil.Context(t, testutil.WaitMedium) - now := time.Now().Format(time.RFC3339) - parsedNow, err := time.Parse(time.RFC3339, now) - require.NoError(t, err) - require.NoError(t, db.SetTelemetryHTMLFirstServedAt(ctx, now)) - deployment, _ = collectSnapshot(t, db, nil) - require.Equal(t, *deployment.HTMLFirstServedAt, parsedNow) - - // Test idempotency - require.NoError(t, db.SetTelemetryHTMLFirstServedAt(ctx, time.Now().Add(time.Hour).Format(time.RFC3339))) - deployment, _ = collectSnapshot(t, db, nil) - require.Equal(t, *deployment.HTMLFirstServedAt, parsedNow) - }) } // nolint:paralleltest @@ -335,6 +320,47 @@ func TestTelemetryInstallSource(t *testing.T) { require.Equal(t, "aws_marketplace", deployment.InstallSource) } +func TestTelemetryItem(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + db, _ := dbtestutil.NewDB(t) + key := testutil.GetRandomName(t) + value := time.Now().Format(time.RFC3339) + + err := db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ + Key: key, + Value: value, + }) + require.NoError(t, err) + + item, err := db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Key, key) + require.Equal(t, item.Value, value) + + // Inserting a new value should not update the existing value + err = db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ + Key: key, + Value: "new_value", + }) + require.NoError(t, err) + + item, err = db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Value, value) + + // Upserting a new value should update the existing value + err = db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{ + Key: key, + Value: "new_value", + }) + require.NoError(t, err) + + item, err = db.GetTelemetryItem(ctx, key) + require.NoError(t, err) + require.Equal(t, item.Value, "new_value") +} + func collectSnapshot(t *testing.T, db database.Store, addOptionsFn func(opts telemetry.Options) telemetry.Options) (*telemetry.Deployment, *telemetry.Snapshot) { t.Helper() deployment := make(chan *telemetry.Deployment, 64) diff --git a/site/site.go b/site/site.go index ef97fbe7201ca..c78ed59dc6353 100644 --- a/site/site.go +++ b/site/site.go @@ -330,29 +330,35 @@ func ShouldCacheFile(reqFile string) bool { // reportHTMLFirstServedAt sends a telemetry report when the first HTML is ever served. // The purpose is to track the first time the first user opens the site. func (h *Handler) reportHTMLFirstServedAt() { - // `Once` is used to reduce the volume of db calls and telemetry reports. - // It's fine to run this multiple times, but it's unnecessary. - h.TelemetryHTMLServedOnce.Do(func() { - ctx := context.Background() - // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. - _, err := h.opts.Database.GetTelemetryHTMLFirstServedAt(dbauthz.AsSystemRestricted(ctx)) - if err == nil { - // If the value is already set, then we reported it before. - // We don't need to report it again. - return - } - if !errors.Is(err, sql.ErrNoRows) { - h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) - return - } - // SetTelemetryHTMLFirstServedAt is idempotent, so there's no harm in calling it multiple times, - // even across restarts. Once it's set for the first time, it will never be changed. - // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. - if err := h.opts.Database.SetTelemetryHTMLFirstServedAt(dbauthz.AsSystemRestricted(ctx), time.Now().Format(time.RFC3339)); err != nil { - h.opts.Logger.Debug(ctx, "failed to set telemetry html first served at", slog.Error(err)) - return - } - h.opts.Telemetry.ReportDeployment() + ctx := context.Background() + itemKey := string(telemetry.TelemetryItemKeyHTMLFirstServedAt) + // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. + _, err := h.opts.Database.GetTelemetryItem(dbauthz.AsSystemRestricted(ctx), itemKey) + if err == nil { + // If the value is already set, then we reported it before. + // We don't need to report it again. + return + } + if !errors.Is(err, sql.ErrNoRows) { + h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) + return + } + // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. + if err := h.opts.Database.InsertTelemetryItemIfNotExists(dbauthz.AsSystemRestricted(ctx), database.InsertTelemetryItemIfNotExistsParams{ + Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt), + Value: time.Now().Format(time.RFC3339), + }); err != nil { + h.opts.Logger.Debug(ctx, "failed to set telemetry html first served at", slog.Error(err)) + return + } + // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. + item, err := h.opts.Database.GetTelemetryItem(dbauthz.AsSystemRestricted(ctx), itemKey) + if err != nil { + h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) + return + } + h.opts.Telemetry.Report(&telemetry.Snapshot{ + TelemetryItems: []telemetry.TelemetryItem{telemetry.ConvertTelemetryItem(item)}, }) } @@ -362,7 +368,11 @@ func (h *Handler) serveHTML(resp http.ResponseWriter, request *http.Request, req // Pass "index.html" to the ServeContent so the ServeContent sets the right content headers. reqPath = "index.html" } - go h.reportHTMLFirstServedAt() + // `Once` is used to reduce the volume of db calls and telemetry reports. + // It's fine to run the enclosed function multiple times, but it's unnecessary. + h.TelemetryHTMLServedOnce.Do(func() { + go h.reportHTMLFirstServedAt() + }) http.ServeContent(resp, request, reqPath, time.Time{}, bytes.NewReader(data)) return true } From 5118aeb398c407988fda181b9ea5bdc5336dede7 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:34:19 +0000 Subject: [PATCH 06/16] remove ReportDeployment from Reporter --- coderd/telemetry/telemetry.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index fe26f0b3aa65c..d1123cb3b6d0c 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -99,8 +99,6 @@ type Reporter interface { // database. For example, if a new user is added, a snapshot can // contain just that user entry. Report(snapshot *Snapshot) - // ReportDeployment sends deployment information to the telemetry server. - ReportDeployment() Enabled() bool Close() } @@ -224,12 +222,6 @@ func (r *remoteReporter) reportWithDeployment() { r.reportSync(snapshot) } -func (r *remoteReporter) ReportDeployment() { - if err := r.deployment(); err != nil { - r.options.Logger.Debug(r.ctx, "failed to report deployment", slog.Error(err)) - } -} - // deployment collects host information and reports it to the telemetry server. func (r *remoteReporter) deployment() error { sysInfoHost, err := sysinfo.Host() @@ -1596,4 +1588,3 @@ type noopReporter struct{} func (*noopReporter) Report(_ *Snapshot) {} func (*noopReporter) Enabled() bool { return false } func (*noopReporter) Close() {} -func (*noopReporter) ReportDeployment() {} From a6a984f67441d1a40c22592d60c06134f1e181c7 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:40:58 +0000 Subject: [PATCH 07/16] remove GetHTMLFirstServedAt and SetHTMLFirstServedAt --- coderd/database/dbauthz/dbauthz.go | 14 ----------- coderd/database/dbauthz/dbauthz_test.go | 6 ----- coderd/database/dbmem/dbmem.go | 20 ---------------- coderd/database/dbmetrics/querymetrics.go | 14 ----------- coderd/database/dbmock/dbmock.go | 29 ----------------------- coderd/database/querier.go | 2 -- coderd/database/queries.sql.go | 22 ----------------- coderd/database/queries/siteconfig.sql | 8 ------- coderd/telemetry/telemetry.go | 12 ---------- 9 files changed, 127 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 804cdab6656a3..0ba9e20216b41 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2096,13 +2096,6 @@ func (q *querier) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) return q.db.GetTailnetTunnelPeerIDs(ctx, srcID) } -func (q *querier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { - if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { - return "", err - } - return q.db.GetTelemetryHTMLFirstServedAt(ctx) -} - func (q *querier) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { return database.TelemetryItem{}, err @@ -3456,13 +3449,6 @@ func (q *querier) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) return q.db.RevokeDBCryptKey(ctx, activeKeyDigest) } -func (q *querier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { - if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { - return err - } - return q.db.SetTelemetryHTMLFirstServedAt(ctx, value) -} - func (q *querier) TryAcquireLock(ctx context.Context, id int64) (bool, error) { return q.db.TryAcquireLock(ctx, id) } diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 90ba26eab9ad4..9e784fff0bf12 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4224,12 +4224,6 @@ func (s *MethodTestSuite) TestSystemFunctions() { s.Run("GetWorkspaceModulesCreatedAfter", s.Subtest(func(db database.Store, check *expects) { check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead) })) - s.Run("GetTelemetryHTMLFirstServedAt", s.Subtest(func(db database.Store, check *expects) { - check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) - })) - s.Run("SetTelemetryHTMLFirstServedAt", s.Subtest(func(db database.Store, check *expects) { - check.Args(time.Now().Format(time.RFC3339)).Asserts(rbac.ResourceSystem, policy.ActionUpdate) - })) s.Run("GetTelemetryItem", s.Subtest(func(db database.Store, check *expects) { check.Args("test").Asserts(rbac.ResourceSystem, policy.ActionRead).Errors(sql.ErrNoRows) })) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 931e836a9acc7..8e074e19e2645 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -4333,16 +4333,6 @@ func (*FakeQuerier) GetTailnetTunnelPeerIDs(context.Context, uuid.UUID) ([]datab return nil, ErrUnimplemented } -func (q *FakeQuerier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { - q.mutex.RLock() - defer q.mutex.RUnlock() - if q.htmlFirstServedAt == "" { - return "", sql.ErrNoRows - } - - return q.htmlFirstServedAt, nil -} - func (q *FakeQuerier) GetTelemetryItem(_ context.Context, key string) (database.TelemetryItem, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -9177,16 +9167,6 @@ func (q *FakeQuerier) RevokeDBCryptKey(_ context.Context, activeKeyDigest string return sql.ErrNoRows } -func (q *FakeQuerier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { - q.mutex.Lock() - defer q.mutex.Unlock() - if q.htmlFirstServedAt != "" { - return nil - } - q.htmlFirstServedAt = value - return nil -} - func (*FakeQuerier) TryAcquireLock(_ context.Context, _ int64) (bool, error) { return false, xerrors.New("TryAcquireLock must only be called within a transaction") } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 434a124eca0d2..c0d3ed4994f9c 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1134,13 +1134,6 @@ func (m queryMetricsStore) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uu return r0, r1 } -func (m queryMetricsStore) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { - start := time.Now() - r0, r1 := m.s.GetTelemetryHTMLFirstServedAt(ctx) - m.queryLatencies.WithLabelValues("GetTelemetryHTMLFirstServedAt").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m queryMetricsStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { start := time.Now() r0, r1 := m.s.GetTelemetryItem(ctx, key) @@ -2191,13 +2184,6 @@ func (m queryMetricsStore) RevokeDBCryptKey(ctx context.Context, activeKeyDigest return r0 } -func (m queryMetricsStore) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { - start := time.Now() - r0 := m.s.SetTelemetryHTMLFirstServedAt(ctx, value) - m.queryLatencies.WithLabelValues("SetTelemetryHTMLFirstServedAt").Observe(time.Since(start).Seconds()) - return r0 -} - func (m queryMetricsStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { start := time.Now() ok, err := m.s.TryAcquireLock(ctx, pgTryAdvisoryXactLock) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 84bedc470ea9c..e32834a441e6d 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2346,21 +2346,6 @@ func (mr *MockStoreMockRecorder) GetTailnetTunnelPeerIDs(ctx, srcID any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTailnetTunnelPeerIDs", reflect.TypeOf((*MockStore)(nil).GetTailnetTunnelPeerIDs), ctx, srcID) } -// GetTelemetryHTMLFirstServedAt mocks base method. -func (m *MockStore) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTelemetryHTMLFirstServedAt", ctx) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTelemetryHTMLFirstServedAt indicates an expected call of GetTelemetryHTMLFirstServedAt. -func (mr *MockStoreMockRecorder) GetTelemetryHTMLFirstServedAt(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryHTMLFirstServedAt", reflect.TypeOf((*MockStore)(nil).GetTelemetryHTMLFirstServedAt), ctx) -} - // GetTelemetryItem mocks base method. func (m *MockStore) GetTelemetryItem(ctx context.Context, key string) (database.TelemetryItem, error) { m.ctrl.T.Helper() @@ -4669,20 +4654,6 @@ func (mr *MockStoreMockRecorder) RevokeDBCryptKey(ctx, activeKeyDigest any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RevokeDBCryptKey", reflect.TypeOf((*MockStore)(nil).RevokeDBCryptKey), ctx, activeKeyDigest) } -// SetTelemetryHTMLFirstServedAt mocks base method. -func (m *MockStore) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetTelemetryHTMLFirstServedAt", ctx, value) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetTelemetryHTMLFirstServedAt indicates an expected call of SetTelemetryHTMLFirstServedAt. -func (mr *MockStoreMockRecorder) SetTelemetryHTMLFirstServedAt(ctx, value any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTelemetryHTMLFirstServedAt", reflect.TypeOf((*MockStore)(nil).SetTelemetryHTMLFirstServedAt), ctx, value) -} - // TryAcquireLock mocks base method. func (m *MockStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 5e72b64b7480e..1fa83208a2218 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -224,7 +224,6 @@ type sqlcQuerier interface { GetTailnetPeers(ctx context.Context, id uuid.UUID) ([]TailnetPeer, error) GetTailnetTunnelPeerBindings(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerBindingsRow, error) GetTailnetTunnelPeerIDs(ctx context.Context, srcID uuid.UUID) ([]GetTailnetTunnelPeerIDsRow, error) - GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error) GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error) // GetTemplateAppInsights returns the aggregate usage of each app in a given @@ -453,7 +452,6 @@ type sqlcQuerier interface { RemoveUserFromAllGroups(ctx context.Context, userID uuid.UUID) error RemoveUserFromGroups(ctx context.Context, arg RemoveUserFromGroupsParams) ([]uuid.UUID, error) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) error - SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error // Non blocking lock. Returns true if the lock was acquired, false otherwise. // // This must be called from within a transaction. The lock will be automatically diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 66f5f6a98c73c..86db8fb66956a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -7815,17 +7815,6 @@ func (q *sqlQuerier) GetRuntimeConfig(ctx context.Context, key string) (string, return value, err } -const getTelemetryHTMLFirstServedAt = `-- name: GetTelemetryHTMLFirstServedAt :one -SELECT value FROM site_configs WHERE key = 'telemetry_html_first_served_at' -` - -func (q *sqlQuerier) GetTelemetryHTMLFirstServedAt(ctx context.Context) (string, error) { - row := q.db.QueryRowContext(ctx, getTelemetryHTMLFirstServedAt) - var value string - err := row.Scan(&value) - return value, err -} - const insertDERPMeshKey = `-- name: InsertDERPMeshKey :exec INSERT INTO site_configs (key, value) VALUES ('derp_mesh_key', $1) ` @@ -7844,17 +7833,6 @@ func (q *sqlQuerier) InsertDeploymentID(ctx context.Context, value string) error return err } -const setTelemetryHTMLFirstServedAt = `-- name: SetTelemetryHTMLFirstServedAt :exec -INSERT INTO site_configs (key, value) -VALUES ('telemetry_html_first_served_at', $1) -ON CONFLICT (key) DO NOTHING -` - -func (q *sqlQuerier) SetTelemetryHTMLFirstServedAt(ctx context.Context, value string) error { - _, err := q.db.ExecContext(ctx, setTelemetryHTMLFirstServedAt, value) - return err -} - const upsertAnnouncementBanners = `-- name: UpsertAnnouncementBanners :exec INSERT INTO site_configs (key, value) VALUES ('announcement_banners', $1) ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'announcement_banners' diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index f400ce3610f18..02c648302bf4f 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -106,11 +106,3 @@ ON CONFLICT (key) DO UPDATE SET value = $2 WHERE site_configs.key = $1; -- name: DeleteRuntimeConfig :exec DELETE FROM site_configs WHERE site_configs.key = $1; - --- name: SetTelemetryHTMLFirstServedAt :exec -INSERT INTO site_configs (key, value) -VALUES ('telemetry_html_first_served_at', $1) -ON CONFLICT (key) DO NOTHING; - --- name: GetTelemetryHTMLFirstServedAt :one -SELECT value FROM site_configs WHERE key = 'telemetry_html_first_served_at'; diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index d1123cb3b6d0c..f29cca67c5871 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -330,18 +330,6 @@ func checkIDPOrgSync(ctx context.Context, db database.Store, values *codersdk.De return syncConfig.Field != "", nil } -func getHTMLFirstServedAt(ctx context.Context, db database.Store) (*time.Time, error) { - htmlFirstServedAtStr, err := db.GetTelemetryHTMLFirstServedAt(ctx) - if err != nil { - return nil, xerrors.Errorf("get telemetry html first served at: %w", err) - } - t, err := time.Parse(time.RFC3339, htmlFirstServedAtStr) - if err != nil { - return nil, xerrors.Errorf("parse telemetry html first served at: %w", err) - } - return &t, nil -} - // createSnapshot collects a full snapshot from the database. func (r *remoteReporter) createSnapshot() (*Snapshot, error) { var ( From 3f6615b119c15d27396da87629e2163c8e2ac16a Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:44:13 +0000 Subject: [PATCH 08/16] remove htmlFirstServedAt from dbmem --- coderd/database/dbmem/dbmem.go | 1 - 1 file changed, 1 deletion(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 8e074e19e2645..103ee1e717149 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -259,7 +259,6 @@ type data struct { defaultProxyDisplayName string defaultProxyIconURL string userStatusChanges []database.UserStatusChange - htmlFirstServedAt string telemetryItems []database.TelemetryItem } From e338c8fc153fc4a0065f9aba6c53b2d9b5be2dbd Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 13:47:03 +0000 Subject: [PATCH 09/16] revert change to siteconfig.sql --- coderd/database/queries/siteconfig.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index 02c648302bf4f..e8d02372e5a4f 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -106,3 +106,4 @@ ON CONFLICT (key) DO UPDATE SET value = $2 WHERE site_configs.key = $1; -- name: DeleteRuntimeConfig :exec DELETE FROM site_configs WHERE site_configs.key = $1; + From 0828c1d7aab43c3fe9b8e47fc733f3913ccf0129 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 14:10:29 +0000 Subject: [PATCH 10/16] fix tests --- site/site_test.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/site/site_test.go b/site/site_test.go index 8bee665a56ae3..63f3f9aa17226 100644 --- a/site/site_test.go +++ b/site/site_test.go @@ -27,8 +27,10 @@ import ( "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbmem" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/site" "github.com/coder/coder/v2/testutil" @@ -45,9 +47,10 @@ func TestInjection(t *testing.T) { binFs := http.FS(fstest.MapFS{}) db := dbmem.New() handler := site.New(&site.Options{ - BinFS: binFs, - Database: db, - SiteFS: siteFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFs, + Database: db, + SiteFS: siteFS, }) user := dbgen.User(t, db, database.User{}) @@ -101,9 +104,10 @@ func TestInjectionFailureProducesCleanHTML(t *testing.T) { }, } handler := site.New(&site.Options{ - BinFS: binFs, - Database: db, - SiteFS: siteFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFs, + Database: db, + SiteFS: siteFS, // No OAuth2 configs, refresh will fail. OAuth2Configs: &httpmw.OAuth2Configs{ @@ -147,9 +151,12 @@ func TestCaching(t *testing.T) { } binFS := http.FS(fstest.MapFS{}) + db, _ := dbtestutil.NewDB(t) srv := httptest.NewServer(site.New(&site.Options{ - BinFS: binFS, - SiteFS: rootFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFS, + SiteFS: rootFS, + Database: db, })) defer srv.Close() @@ -213,9 +220,12 @@ func TestServingFiles(t *testing.T) { } binFS := http.FS(fstest.MapFS{}) + db, _ := dbtestutil.NewDB(t) srv := httptest.NewServer(site.New(&site.Options{ - BinFS: binFS, - SiteFS: rootFS, + Telemetry: telemetry.NewNoop(), + BinFS: binFS, + SiteFS: rootFS, + Database: db, })) defer srv.Close() @@ -473,6 +483,7 @@ func TestServingBin(t *testing.T) { } srv := httptest.NewServer(site.New(&site.Options{ + Telemetry: telemetry.NewNoop(), BinFS: binFS, BinHashes: binHashes, SiteFS: rootFS, From 35e79bcb571d68123c6b5fdb76e90c7daaa8b74f Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 14:19:14 +0000 Subject: [PATCH 11/16] add a test for reporting html first served --- cli/server_test.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cli/server_test.go b/cli/server_test.go index fa96e192f7eb3..31d3610b8866d 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -964,7 +964,7 @@ func TestServer(t *testing.T) { server := httptest.NewServer(r) defer server.Close() - inv, _ := clitest.New(t, + inv, cfg := clitest.New(t, "server", "--in-memory", "--http-address", ":0", @@ -977,6 +977,26 @@ func TestServer(t *testing.T) { <-deployment <-snapshot + + accessURL := waitAccessURL(t, cfg) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium) + defer cancel() + client := codersdk.New(accessURL) + body, err := client.Request(ctx, http.MethodGet, "/", nil) + require.NoError(t, err) + require.NoError(t, body.Body.Close()) + + snap := <-snapshot + require.Condition(t, func() bool { + htmlFirstServedFound := false + for _, item := range snap.TelemetryItems { + if item.Key == string(telemetry.TelemetryItemKeyHTMLFirstServedAt) { + htmlFirstServedFound = true + } + } + return htmlFirstServedFound + }, "no html_first_served telemetry item") }) t.Run("Prometheus", func(t *testing.T) { t.Parallel() From 89c8371f4d760ff0ea9e397cc1a0b0e0a247138c Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 14:23:25 +0000 Subject: [PATCH 12/16] make the snapshot check more robust --- cli/server_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/server_test.go b/cli/server_test.go index 31d3610b8866d..7531c665c7fa4 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -987,8 +987,8 @@ func TestServer(t *testing.T) { require.NoError(t, err) require.NoError(t, body.Body.Close()) - snap := <-snapshot - require.Condition(t, func() bool { + require.Eventually(t, func() bool { + snap := <-snapshot htmlFirstServedFound := false for _, item := range snap.TelemetryItems { if item.Key == string(telemetry.TelemetryItemKeyHTMLFirstServedAt) { @@ -996,7 +996,7 @@ func TestServer(t *testing.T) { } } return htmlFirstServedFound - }, "no html_first_served telemetry item") + }, testutil.WaitMedium, testutil.IntervalFast, "no html_first_served telemetry item") }) t.Run("Prometheus", func(t *testing.T) { t.Parallel() From ff86e66a810b93ca0847f421042d8a781decabe3 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 16:06:58 +0000 Subject: [PATCH 13/16] add telemetry_items fixture --- .../testdata/fixtures/000288_telemetry_items.up.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql b/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql new file mode 100644 index 0000000000000..0189558292915 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000288_telemetry_items.up.sql @@ -0,0 +1,4 @@ +INSERT INTO + telemetry_items (key, value) +VALUES + ('example_key', 'example_value'); From d2ca62d93691529a8f411f6ce3da4ea93896d1bb Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 30 Jan 2025 16:11:05 +0000 Subject: [PATCH 14/16] Simplify telemetry item reporting context handling --- site/site.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/site/site.go b/site/site.go index c78ed59dc6353..f4831e4f224ff 100644 --- a/site/site.go +++ b/site/site.go @@ -330,10 +330,10 @@ func ShouldCacheFile(reqFile string) bool { // reportHTMLFirstServedAt sends a telemetry report when the first HTML is ever served. // The purpose is to track the first time the first user opens the site. func (h *Handler) reportHTMLFirstServedAt() { - ctx := context.Background() + // nolint:gocritic // Manipulating telemetry items is system-restricted. + ctx := dbauthz.AsSystemRestricted(context.Background()) itemKey := string(telemetry.TelemetryItemKeyHTMLFirstServedAt) - // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. - _, err := h.opts.Database.GetTelemetryItem(dbauthz.AsSystemRestricted(ctx), itemKey) + _, err := h.opts.Database.GetTelemetryItem(ctx, itemKey) if err == nil { // If the value is already set, then we reported it before. // We don't need to report it again. @@ -343,16 +343,14 @@ func (h *Handler) reportHTMLFirstServedAt() { h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) return } - // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. - if err := h.opts.Database.InsertTelemetryItemIfNotExists(dbauthz.AsSystemRestricted(ctx), database.InsertTelemetryItemIfNotExistsParams{ + if err := h.opts.Database.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{ Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt), Value: time.Now().Format(time.RFC3339), }); err != nil { h.opts.Logger.Debug(ctx, "failed to set telemetry html first served at", slog.Error(err)) return } - // nolint:gocritic // Only used for telemetry, so AsSystemRestricted is fine. - item, err := h.opts.Database.GetTelemetryItem(dbauthz.AsSystemRestricted(ctx), itemKey) + item, err := h.opts.Database.GetTelemetryItem(ctx, itemKey) if err != nil { h.opts.Logger.Debug(ctx, "failed to get telemetry html first served at", slog.Error(err)) return From 00c27c50e414707f8ac8e1edaeab1413cafd11ec Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Fri, 31 Jan 2025 12:01:00 +0000 Subject: [PATCH 15/16] address the PR review --- cli/server_test.go | 3 +-- coderd/telemetry/telemetry.go | 11 ++++++++--- site/site.go | 5 +++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cli/server_test.go b/cli/server_test.go index 7531c665c7fa4..8ed4d89ceb970 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -980,8 +980,7 @@ func TestServer(t *testing.T) { accessURL := waitAccessURL(t, cfg) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium) - defer cancel() + ctx := testutil.Context(t, testutil.WaitMedium) client := codersdk.New(accessURL) body, err := client.Request(ctx, http.MethodGet, "/", nil) require.NoError(t, err) diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index f29cca67c5871..3b4bcb7d15ae6 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -1557,11 +1557,16 @@ type Organization struct { CreatedAt time.Time `json:"created_at"` } -//revive:disable:exported -type TelemetryItemKey string +type telemetryItemKey string +// The comment below gets rid of the warning that the name "TelemetryItemKey" has +// the "Telemetry" prefix, and that stutters when you use it outside the package +// (telemetry.TelemetryItemKey...). "TelemetryItem" is the name of a database table, +// so it makes sense to use the "Telemetry" prefix. +// +//revive:disable:exported const ( - TelemetryItemKeyHTMLFirstServedAt TelemetryItemKey = "html_first_served_at" + TelemetryItemKeyHTMLFirstServedAt telemetryItemKey = "html_first_served_at" ) type TelemetryItem struct { diff --git a/site/site.go b/site/site.go index f4831e4f224ff..d6ab14ebc8e88 100644 --- a/site/site.go +++ b/site/site.go @@ -188,7 +188,7 @@ type Handler struct { Entitlements *entitlements.Set Experiments atomic.Pointer[codersdk.Experiments] - TelemetryHTMLServedOnce sync.Once + elemetryHTMLServedOnce sync.Once } func (h *Handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -331,6 +331,7 @@ func ShouldCacheFile(reqFile string) bool { // The purpose is to track the first time the first user opens the site. func (h *Handler) reportHTMLFirstServedAt() { // nolint:gocritic // Manipulating telemetry items is system-restricted. + // TODO(hugodutka): Add a telemetry context in RBAC. ctx := dbauthz.AsSystemRestricted(context.Background()) itemKey := string(telemetry.TelemetryItemKeyHTMLFirstServedAt) _, err := h.opts.Database.GetTelemetryItem(ctx, itemKey) @@ -368,7 +369,7 @@ func (h *Handler) serveHTML(resp http.ResponseWriter, request *http.Request, req } // `Once` is used to reduce the volume of db calls and telemetry reports. // It's fine to run the enclosed function multiple times, but it's unnecessary. - h.TelemetryHTMLServedOnce.Do(func() { + h.elemetryHTMLServedOnce.Do(func() { go h.reportHTMLFirstServedAt() }) http.ServeContent(resp, request, reqPath, time.Time{}, bytes.NewReader(data)) From e2977404f73f70ff7ac7d553d68f1442907da808 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Fri, 31 Jan 2025 12:34:45 +0000 Subject: [PATCH 16/16] fix typo --- site/site.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/site.go b/site/site.go index d6ab14ebc8e88..3a85f7b3963ad 100644 --- a/site/site.go +++ b/site/site.go @@ -188,7 +188,7 @@ type Handler struct { Entitlements *entitlements.Set Experiments atomic.Pointer[codersdk.Experiments] - elemetryHTMLServedOnce sync.Once + telemetryHTMLServedOnce sync.Once } func (h *Handler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { @@ -369,7 +369,7 @@ func (h *Handler) serveHTML(resp http.ResponseWriter, request *http.Request, req } // `Once` is used to reduce the volume of db calls and telemetry reports. // It's fine to run the enclosed function multiple times, but it's unnecessary. - h.elemetryHTMLServedOnce.Do(func() { + h.telemetryHTMLServedOnce.Do(func() { go h.reportHTMLFirstServedAt() }) http.ServeContent(resp, request, reqPath, time.Time{}, bytes.NewReader(data))