From 21a32dbc3fdbe3ed4640ff516780370a304a6c2f Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Wed, 9 Sep 2026 14:29:50 +0000 Subject: [PATCH 1/7] feat: store session usage per app and family in template usage stats Replace the fixed per-family minute columns on template_usage_stats with template_usage_stats_session_families and template_usage_stats_session_apps, one row per bucket and name, so a session reported under a new app name is stored and reported instead of being dropped. Family attribution comes from the single codersdk registry, which the rollup and the insights reads take as one jsonb parameter of app name to family, so registering an app or a family needs no SQL change, no column, and no probe. Migration 000591 backfills the existing family totals into the child table and drops the fixed columns. GetTemplateInsights keeps the per-user aggregate as its only large grouping and applies the 30 minute cross-template cap only to the buckets that need it, and the rollup scans workspace_agent_stats once. At 302k history rows the 30 day read goes from 2.4x the fixed-column baseline to about 1.1x on typical data, and the (start_time, user_id) child indexes keep cross-template heavy data within about 2x. --- coderd/database/dbauthz/dbauthz_test.go | 92 +-- coderd/database/dbauthz/sessioncountparams.go | 60 +- coderd/database/dbrollup/dbrollup_test.go | 31 +- coderd/database/dump.sql | 47 +- coderd/database/foreign_key_constraint.go | 2 + ...emplate_usage_stats_session_usage.down.sql | 61 ++ ..._template_usage_stats_session_usage.up.sql | 73 ++ .../migrations/migration000591_test.go | 365 ++++++++++ ..._template_usage_stats_session_usage.up.sql | 48 ++ coderd/database/models.go | 32 +- coderd/database/querier.go | 11 + coderd/database/querier_test.go | 48 +- coderd/database/queries.sql.go | 640 ++++++++++++------ coderd/database/queries/insights.sql | 589 +++++++++++----- coderd/database/session_usage_history_test.go | 76 +++ coderd/database/session_usage_test.go | 141 ++++ coderd/database/types.go | 1 + coderd/database/unique_constraint.go | 2 + coderd/insights.go | 105 ++- .../insights_session_family_internal_test.go | 79 +++ .../insights/metricscollector.go | 92 ++- .../metricscollector_internal_test.go | 50 ++ codersdk/appname.go | 76 +-- codersdk/appname_internal_test.go | 15 +- codersdk/appname_test.go | 94 ++- 25 files changed, 2188 insertions(+), 642 deletions(-) create mode 100644 coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql create mode 100644 coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql create mode 100644 coderd/database/migrations/migration000591_test.go create mode 100644 coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql create mode 100644 coderd/database/session_usage_history_test.go create mode 100644 coderd/database/session_usage_test.go create mode 100644 coderd/insights_session_family_internal_test.go create mode 100644 coderd/prometheusmetrics/insights/metricscollector_internal_test.go diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 9cea5e8d4f9..352f008460b 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "encoding/json" "fmt" - "maps" "net" "reflect" "testing" @@ -7842,10 +7841,11 @@ func TestAsExternalAuthChecker(t *testing.T) { }) } -// TestSessionCountAppFamiliesRequired ensures the queries that take the app -// family registry fail loudly when it is empty, so a forgotten parameter -// surfaces as an error instead of silently dropping every family's sessions -// from usage reporting. +// TestSessionCountAppFamiliesRequired ensures the queries that take the +// app-to-family registry fail loudly when it is missing. The queries fall +// back to the unknown family, so a forgotten parameter would misattribute +// known activity and undercount the fixed per-family compatibility fields +// rather than surfacing an error. func TestSessionCountAppFamiliesRequired(t *testing.T) { t.Parallel() @@ -7862,45 +7862,28 @@ func TestSessionCountAppFamiliesRequired(t *testing.T) { require.ErrorContains(t, err, "developer error") } -// TestSessionCountAppFamiliesMustMatchQueries covers registries that are -// present but wrong. Each query hardcodes one probe per family, so a registry -// whose keys drifted from codersdk.AttributedAppFamilies would report zero -// for the affected family instead of failing. -func TestSessionCountAppFamiliesMustMatchQueries(t *testing.T) { +// TestSessionCountAppFamiliesShape covers registries that are present but +// unusable. The queries join the registry by normalized app name and fall +// back to the unknown family, so an entry the join can never match, or one +// with no family to attribute to, would misattribute known activity and +// report a plausible but undercounted result instead of failing. +func TestSessionCountAppFamiliesShape(t *testing.T) { t.Parallel() - valid := map[codersdk.AppFamilyName][]string{} - for _, family := range codersdk.AttributedAppFamilies() { - valid[family] = []string{string(family)} - } - without := func(drop codersdk.AppFamilyName) json.RawMessage { - families := maps.Clone(valid) - delete(families, drop) - return mustMarshalAppFamilies(t, families) - } - for _, tc := range []struct { name string appFamilies json.RawMessage errContains string }{ - {"EmptyObject", json.RawMessage(`{}`), `missing family "vscode"`}, - {"JSONNull", json.RawMessage(`null`), `missing family "vscode"`}, + {"EmptyObject", json.RawMessage(`{}`), "must not be empty"}, + {"JSONNull", json.RawMessage(`null`), "must not be empty"}, {"NotAnObject", json.RawMessage(`["vscode"]`), "must be a JSON object"}, - {"MissingFamily", without(codersdk.AppFamilySSH), `missing family "ssh"`}, - {"EmptyAppNames", mustMarshalAppFamilies(t, map[codersdk.AppFamilyName][]string{ - codersdk.AppFamilyVSCode: {"vscode"}, - codersdk.AppFamilyJetBrains: {"jetbrains"}, - codersdk.AppFamilySSH: {}, - codersdk.AppFamilyReconnectingPTY: {"reconnecting_pty"}, - }), `no app names for family "ssh"`}, - {"UnknownFamily", mustMarshalAppFamilies(t, map[codersdk.AppFamilyName][]string{ - codersdk.AppFamilyVSCode: {"vscode"}, - codersdk.AppFamilyJetBrains: {"jetbrains"}, - codersdk.AppFamilySSH: {"ssh"}, - codersdk.AppFamilyReconnectingPTY: {"reconnecting_pty"}, - "emacs": {"emacs"}, - }), `has family "emacs"`}, + {"FamilyToAppNames", json.RawMessage(`{"vscode":["cursor"]}`), "must be a JSON object"}, + {"UnnormalizedAppName", json.RawMessage(`{"VSCode-Insiders":"vscode"}`), "is not normalized"}, + {"EmptyAppName", json.RawMessage(`{"":"vscode"}`), "empty app name"}, + {"EmptyFamily", json.RawMessage(`{"vscode":""}`), `no family for app "vscode"`}, + {"WhitespaceFamily", json.RawMessage(`{"vscode":" "}`), `no family for app "vscode"`}, + {"NullFamily", json.RawMessage(`{"vscode":null}`), `no family for app "vscode"`}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -7920,9 +7903,36 @@ func TestSessionCountAppFamiliesMustMatchQueries(t *testing.T) { } } -func mustMarshalAppFamilies(t *testing.T, families map[codersdk.AppFamilyName][]string) json.RawMessage { - t.Helper() - raw, err := json.Marshal(families) - require.NoError(t, err) - return raw +// No query names a family, so registering an app under a family that has +// never been seen before is valid without any SQL change. Validation must not +// reintroduce a hardcoded family list. +func TestSessionCountAppFamiliesAcceptsNewFamily(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + appFamilies json.RawMessage + }{ + {"Registry", codersdk.SessionCountAppFamiliesJSON()}, + {"NewFamily", json.RawMessage(`{"emacs":"emacs","vscode":"vscode"}`)}, + {"SingleEntry", json.RawMessage(`{"ssh":"ssh"}`)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + dbm := dbmock.NewMockStore(ctrl) + dbm.EXPECT().Wrappers().Return([]string{}).AnyTimes() + arg := database.GetTemplateInsightsByTemplateParams{AppFamilies: tc.appFamilies} + dbm.EXPECT().GetTemplateInsightsByTemplate(gomock.Any(), arg).Return([]database.GetTemplateInsightsByTemplateRow{}, nil).AnyTimes() + dbm.EXPECT().UpsertTemplateUsageStats(gomock.Any(), tc.appFamilies).Return(nil).AnyTimes() + q := dbauthz.New(dbm, &coderdtest.RecordingAuthorizer{Wrapped: &coderdtest.FakeAuthorizer{}}, slog.Make(), coderdtest.AccessControlStorePointer()) + ctx := dbauthz.As(context.Background(), coderdtest.RandomRBACSubject()) + + _, err := q.GetTemplateInsightsByTemplate(ctx, arg) + require.NoError(t, err) + require.NoError(t, q.UpsertTemplateUsageStats(ctx, tc.appFamilies)) + }) + } } diff --git a/coderd/database/dbauthz/sessioncountparams.go b/coderd/database/dbauthz/sessioncountparams.go index 70c0c7ae07d..71aa8ed34e7 100644 --- a/coderd/database/dbauthz/sessioncountparams.go +++ b/coderd/database/dbauthz/sessioncountparams.go @@ -3,7 +3,7 @@ package dbauthz import ( "context" "encoding/json" - "slices" + "strings" "golang.org/x/xerrors" @@ -13,42 +13,50 @@ import ( "github.com/coder/coder/v2/codersdk" ) -// The template insights read query and the usage stats rollup take the app -// family attribution registry as a single jsonb parameter. Both hardcode one -// probe per family, so a registry that is empty, malformed, or keyed -// differently from codersdk.AttributedAppFamilies would still run and return -// zero counts for the affected families, silently dropping sessions from -// insights and Prometheus. dbauthz wraps every production store, including -// the transaction stores used by the rollup, so validating here makes a wrong -// registry fail the call loudly instead of failing the data quietly. These -// methods override the generated ones in dbauthz.go; scripts/dbgen preserves -// methods defined outside that file. +// The minute aggregation queries take the app-to-family attribution registry +// as a single jsonb parameter and join it by app name, falling back to the +// unknown family for any app the registry does not cover. A registry that is +// empty, malformed, or keyed by something other than normalized app names +// therefore still runs and still totals every session: it misattributes known +// activity to the unknown family instead. The raw per-app data survives, so +// nothing is lost, but the fixed per-family compatibility fields reported to +// insights, Prometheus, and telemetry undercount for as long as it goes +// unnoticed. dbauthz wraps every production store, including the transaction +// stores used by the rollup, so validating here makes a wrong registry fail +// the call loudly instead of quietly skewing the attribution. These methods +// override the generated ones in dbauthz.go; scripts/dbgen preserves methods +// defined outside that file. -// validateSessionCountAppFamilies checks that the registry has exactly the -// families the queries probe, each with at least one app name. +// validateSessionCountAppFamilies checks that the registry is a non-empty +// jsonb object of normalized app name to family name. It deliberately does +// not check which families appear: no query names a family, so a registry +// entry for a new family is valid without any SQL change. func validateSessionCountAppFamilies(appFamilies json.RawMessage) error { if len(appFamilies) == 0 { return xerrors.New("developer error: session count app families must not be empty, populate them with codersdk.SessionCountAppFamiliesJSON()") } - var families map[codersdk.AppFamilyName][]string + var families map[string]codersdk.AppFamilyName if err := json.Unmarshal(appFamilies, &families); err != nil { - return xerrors.Errorf("developer error: session count app families must be a JSON object of family to app names, populate them with codersdk.SessionCountAppFamiliesJSON(): %w", err) + return xerrors.Errorf("developer error: session count app families must be a JSON object of app name to family, populate them with codersdk.SessionCountAppFamiliesJSON(): %w", err) + } + if len(families) == 0 { + return xerrors.New("developer error: session count app families must not be empty, populate them with codersdk.SessionCountAppFamiliesJSON()") } - required := codersdk.AttributedAppFamilies() - for _, family := range required { - appNames, ok := families[family] - if !ok { - return xerrors.Errorf("developer error: session count app families is missing family %q, which the queries probe; populate them with codersdk.SessionCountAppFamiliesJSON()", family) + for appName, family := range families { + if appName == "" { + return xerrors.New("developer error: session count app families has an empty app name, which no session can match") } - if len(appNames) == 0 { - return xerrors.Errorf("developer error: session count app families has no app names for family %q, so its sessions would go uncounted", family) + // Stored app names are normalized, so an unnormalized key matches no + // session and that app's activity falls back to the unknown family. + if normalized := codersdk.NormalizeAppName(appName); normalized != appName { + return xerrors.Errorf("developer error: session count app families app name %q is not normalized, expected %q", appName, normalized) } - } - for family := range families { - if !slices.Contains(required, family) { - return xerrors.Errorf("developer error: session count app families has family %q, which no query probes; add a probe per query or drop it from codersdk.AttributedAppFamilies", family) + // A blank family is not an attribution, so its apps would report under + // no usable name at all. + if strings.TrimSpace(string(family)) == "" { + return xerrors.Errorf("developer error: session count app families has no family for app %q, so its sessions would be misattributed", appName) } } return nil diff --git a/coderd/database/dbrollup/dbrollup_test.go b/coderd/database/dbrollup/dbrollup_test.go index 03ff495a04d..cb81abac366 100644 --- a/coderd/database/dbrollup/dbrollup_test.go +++ b/coderd/database/dbrollup/dbrollup_test.go @@ -136,7 +136,7 @@ func TestRollup_TwoInstancesUseLocking(t *testing.T) { func TestRollupTemplateUsageStats(t *testing.T) { t.Parallel() - db, ps := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure()) + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) anHourAgo := dbtime.Now().Add(-time.Hour).Truncate(time.Hour).UTC() @@ -245,15 +245,30 @@ func TestRollupTemplateUsageStats(t *testing.T) { stats[0].StartTime = stats[0].StartTime.UTC() require.Equal(t, database.TemplateUsageStat{ - TemplateID: tpl.ID, - UserID: user.ID, - StartTime: wags1.CreatedAt, - EndTime: wags1.CreatedAt.Add(30 * time.Minute), - MedianLatencyMs: sql.NullFloat64{Float64: 1, Valid: true}, - UsageMins: 3, - ReconnectingPtyMins: 2, + TemplateID: tpl.ID, + UserID: user.ID, + StartTime: wags1.CreatedAt, + EndTime: wags1.CreatedAt.Add(30 * time.Minute), + MedianLatencyMs: sql.NullFloat64{Float64: 1, Valid: true}, + UsageMins: 3, AppUsageMins: database.StringMapOfInt{ app.Slug: 2, }, }, stats[0]) + + // Session minutes live in the child tables, keyed by the reported app + // name and by its family. + for _, tc := range []struct { + table, nameColumn, name string + }{ + {"template_usage_stats_session_apps", "app_name", "reconnecting_pty"}, + {"template_usage_stats_session_families", "family", "reconnecting_pty"}, + } { + var usageMins int64 + //nolint:gosec // Table and column names are constants in this test. + err := sqlDB.QueryRowContext(ctx, "SELECT usage_mins FROM "+tc.table+" WHERE start_time = $1 AND template_id = $2 AND user_id = $3 AND "+tc.nameColumn+" = $4", + wags1.CreatedAt, tpl.ID, user.ID, tc.name).Scan(&usageMins) + require.NoError(t, err, tc.table) + require.EqualValues(t, 2, usageMins, tc.table) + } } diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index edeae3f00b8..3858e958e6a 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -3056,11 +3056,6 @@ CREATE TABLE template_usage_stats ( user_id uuid NOT NULL, median_latency_ms real, usage_mins smallint NOT NULL, - ssh_mins smallint NOT NULL, - sftp_mins smallint NOT NULL, - reconnecting_pty_mins smallint NOT NULL, - vscode_mins smallint NOT NULL, - jetbrains_mins smallint NOT NULL, app_usage_mins jsonb ); @@ -3078,17 +3073,35 @@ COMMENT ON COLUMN template_usage_stats.median_latency_ms IS 'Median latency the COMMENT ON COLUMN template_usage_stats.usage_mins IS 'Total minutes the user has been using the template.'; -COMMENT ON COLUMN template_usage_stats.ssh_mins IS 'Total minutes the user has been using SSH.'; +COMMENT ON COLUMN template_usage_stats.app_usage_mins IS 'Object with app names as keys and total minutes used as values. Null means no app usage was recorded.'; -COMMENT ON COLUMN template_usage_stats.sftp_mins IS 'Total minutes the user has been using SFTP.'; +CREATE TABLE template_usage_stats_session_apps ( + start_time timestamp with time zone NOT NULL, + template_id uuid NOT NULL, + user_id uuid NOT NULL, + app_name text NOT NULL, + usage_mins smallint NOT NULL +); -COMMENT ON COLUMN template_usage_stats.reconnecting_pty_mins IS 'Total minutes the user has been using the reconnecting PTY.'; +COMMENT ON TABLE template_usage_stats_session_apps IS 'Session usage of each template_usage_stats bucket, split by app name. A bucket with family rows but no rows here predates per-app recording, so its per-app usage is unknown rather than zero.'; -COMMENT ON COLUMN template_usage_stats.vscode_mins IS 'Total minutes the user has been using VSCode.'; +COMMENT ON COLUMN template_usage_stats_session_apps.app_name IS 'App name as the agent reported it, so it is a source label rather than a curated identity. An agent that reports only the fixed session counts reports family names here, as does history converted by migration 000590.'; -COMMENT ON COLUMN template_usage_stats.jetbrains_mins IS 'Total minutes the user has been using JetBrains.'; +COMMENT ON COLUMN template_usage_stats_session_apps.usage_mins IS 'Total minutes the user has been using the app.'; -COMMENT ON COLUMN template_usage_stats.app_usage_mins IS 'Object with app names as keys and total minutes used as values. Null means no app usage was recorded.'; +CREATE TABLE template_usage_stats_session_families ( + start_time timestamp with time zone NOT NULL, + template_id uuid NOT NULL, + user_id uuid NOT NULL, + family text NOT NULL, + usage_mins smallint NOT NULL +); + +COMMENT ON TABLE template_usage_stats_session_families IS 'Session usage of each template_usage_stats bucket, split by app family. A bucket with no row here recorded no session usage.'; + +COMMENT ON COLUMN template_usage_stats_session_families.family IS 'Family name the registry attributed the session to when the bucket was last rolled up, including ''unknown'' for an app name the registry did not know. Buckets the rollup no longer revisits keep their recorded attribution.'; + +COMMENT ON COLUMN template_usage_stats_session_families.usage_mins IS 'Total minutes the user has been using the family. Minutes shared by two apps of the family count once.'; CREATE TABLE template_version_parameters ( template_version_id uuid NOT NULL, @@ -4486,6 +4499,12 @@ ALTER TABLE ONLY telemetry_locks ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); +ALTER TABLE ONLY template_usage_stats_session_apps + ADD CONSTRAINT template_usage_stats_session_apps_pkey PRIMARY KEY (start_time, user_id, template_id, app_name); + +ALTER TABLE ONLY template_usage_stats_session_families + ADD CONSTRAINT template_usage_stats_session_families_pkey PRIMARY KEY (start_time, user_id, template_id, family); + ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); @@ -5360,6 +5379,12 @@ ALTER TABLE ONLY tailnet_peers ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_coordinator_id_fkey FOREIGN KEY (coordinator_id) REFERENCES tailnet_coordinators(id) ON DELETE CASCADE; +ALTER TABLE ONLY template_usage_stats_session_families + ADD CONSTRAINT template_usage_stats_session__start_time_template_id_user__fkey FOREIGN KEY (start_time, template_id, user_id) REFERENCES template_usage_stats(start_time, template_id, user_id) ON DELETE CASCADE; + +ALTER TABLE ONLY template_usage_stats_session_apps + ADD CONSTRAINT template_usage_stats_session_start_time_template_id_user__fkey1 FOREIGN KEY (start_time, template_id, user_id) REFERENCES template_usage_stats(start_time, template_id, user_id) ON DELETE CASCADE; + ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; diff --git a/coderd/database/foreign_key_constraint.go b/coderd/database/foreign_key_constraint.go index 876a407ae75..b8681fa67f0 100644 --- a/coderd/database/foreign_key_constraint.go +++ b/coderd/database/foreign_key_constraint.go @@ -93,6 +93,8 @@ const ( ForeignKeyProvisionerKeysOrganizationID ForeignKeyConstraint = "provisioner_keys_organization_id_fkey" // ALTER TABLE ONLY provisioner_keys ADD CONSTRAINT provisioner_keys_organization_id_fkey FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; ForeignKeyTailnetPeersCoordinatorID ForeignKeyConstraint = "tailnet_peers_coordinator_id_fkey" // ALTER TABLE ONLY tailnet_peers ADD CONSTRAINT tailnet_peers_coordinator_id_fkey FOREIGN KEY (coordinator_id) REFERENCES tailnet_coordinators(id) ON DELETE CASCADE; ForeignKeyTailnetTunnelsCoordinatorID ForeignKeyConstraint = "tailnet_tunnels_coordinator_id_fkey" // ALTER TABLE ONLY tailnet_tunnels ADD CONSTRAINT tailnet_tunnels_coordinator_id_fkey FOREIGN KEY (coordinator_id) REFERENCES tailnet_coordinators(id) ON DELETE CASCADE; + ForeignKeyTemplateUsageStatsSessionStartTimeTemplateIDUser ForeignKeyConstraint = "template_usage_stats_session__start_time_template_id_user__fkey" // ALTER TABLE ONLY template_usage_stats_session_families ADD CONSTRAINT template_usage_stats_session__start_time_template_id_user__fkey FOREIGN KEY (start_time, template_id, user_id) REFERENCES template_usage_stats(start_time, template_id, user_id) ON DELETE CASCADE; + ForeignKeyTemplateUsageStatsSessionStartTimeTemplateIDUserFkey1 ForeignKeyConstraint = "template_usage_stats_session_start_time_template_id_user__fkey1" // ALTER TABLE ONLY template_usage_stats_session_apps ADD CONSTRAINT template_usage_stats_session_start_time_template_id_user__fkey1 FOREIGN KEY (start_time, template_id, user_id) REFERENCES template_usage_stats(start_time, template_id, user_id) ON DELETE CASCADE; ForeignKeyTemplateVersionParametersTemplateVersionID ForeignKeyConstraint = "template_version_parameters_template_version_id_fkey" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetParametTemplateVersionPresetID ForeignKeyConstraint = "template_version_preset_paramet_template_version_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; ForeignKeyTemplateVersionPresetPrebuildSchedulesPresetID ForeignKeyConstraint = "template_version_preset_prebuild_schedules_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_prebuild_schedules ADD CONSTRAINT template_version_preset_prebuild_schedules_preset_id_fkey FOREIGN KEY (preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE; diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql b/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql new file mode 100644 index 00000000000..c74189aca4a --- /dev/null +++ b/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql @@ -0,0 +1,61 @@ +-- The fixed columns are NOT NULL with no default, so add them with a default +-- to fill existing rows, then drop the default to restore the original schema. +ALTER TABLE template_usage_stats + ADD COLUMN ssh_mins smallint DEFAULT 0 NOT NULL, + ADD COLUMN sftp_mins smallint DEFAULT 0 NOT NULL, + ADD COLUMN reconnecting_pty_mins smallint DEFAULT 0 NOT NULL, + ADD COLUMN vscode_mins smallint DEFAULT 0 NOT NULL, + ADD COLUMN jetbrains_mins smallint DEFAULT 0 NOT NULL; + +ALTER TABLE template_usage_stats + ALTER COLUMN ssh_mins DROP DEFAULT, + ALTER COLUMN sftp_mins DROP DEFAULT, + ALTER COLUMN reconnecting_pty_mins DROP DEFAULT, + ALTER COLUMN vscode_mins DROP DEFAULT, + ALTER COLUMN jetbrains_mins DROP DEFAULT; + +COMMENT ON COLUMN template_usage_stats.ssh_mins IS 'Total minutes the user has been using SSH.'; + +COMMENT ON COLUMN template_usage_stats.sftp_mins IS 'Total minutes the user has been using SFTP.'; + +COMMENT ON COLUMN template_usage_stats.reconnecting_pty_mins IS 'Total minutes the user has been using the reconnecting PTY.'; + +COMMENT ON COLUMN template_usage_stats.vscode_mins IS 'Total minutes the user has been using VSCode.'; + +COMMENT ON COLUMN template_usage_stats.jetbrains_mins IS 'Total minutes the user has been using JetBrains.'; + +-- Restore the five families the fixed columns have room for. Usage attributed +-- to any other family is discarded, and so is all per-app session usage: the +-- fixed columns have nowhere to put either. +UPDATE template_usage_stats AS tus +SET + ssh_mins = families.ssh_mins, + sftp_mins = families.sftp_mins, + reconnecting_pty_mins = families.reconnecting_pty_mins, + vscode_mins = families.vscode_mins, + jetbrains_mins = families.jetbrains_mins +FROM ( + SELECT + start_time, + template_id, + user_id, + COALESCE(MAX(usage_mins) FILTER (WHERE family = 'ssh'), 0)::smallint AS ssh_mins, + COALESCE(MAX(usage_mins) FILTER (WHERE family = 'sftp'), 0)::smallint AS sftp_mins, + COALESCE(MAX(usage_mins) FILTER (WHERE family = 'reconnecting_pty'), 0)::smallint AS reconnecting_pty_mins, + COALESCE(MAX(usage_mins) FILTER (WHERE family = 'vscode'), 0)::smallint AS vscode_mins, + COALESCE(MAX(usage_mins) FILTER (WHERE family = 'jetbrains'), 0)::smallint AS jetbrains_mins + FROM + template_usage_stats_session_families + WHERE + family IN ('ssh', 'sftp', 'reconnecting_pty', 'vscode', 'jetbrains') + GROUP BY + start_time, template_id, user_id +) AS families +WHERE + tus.start_time = families.start_time + AND tus.template_id = families.template_id + AND tus.user_id = families.user_id; + +DROP TABLE template_usage_stats_session_apps; + +DROP TABLE template_usage_stats_session_families; diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql new file mode 100644 index 00000000000..1f3201fbf53 --- /dev/null +++ b/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql @@ -0,0 +1,73 @@ +-- The primary keys put user_id before template_id: the insights read caps a +-- user's minutes per half hour across templates, and looks up one user's rows +-- in a half hour through this prefix. The upsert's conflict target names the +-- same columns in the parent's order, which the unique index satisfies. +CREATE TABLE template_usage_stats_session_families ( + start_time timestamptz NOT NULL, + template_id uuid NOT NULL, + user_id uuid NOT NULL, + family text NOT NULL, + usage_mins smallint NOT NULL, + + PRIMARY KEY (start_time, user_id, template_id, family), + FOREIGN KEY (start_time, template_id, user_id) + REFERENCES template_usage_stats (start_time, template_id, user_id) + ON DELETE CASCADE +); + +COMMENT ON TABLE template_usage_stats_session_families IS 'Session usage of each template_usage_stats bucket, split by app family. A bucket with no row here recorded no session usage.'; + +COMMENT ON COLUMN template_usage_stats_session_families.family IS 'Family name the registry attributed the session to when the bucket was last rolled up, including ''unknown'' for an app name the registry did not know. Buckets the rollup no longer revisits keep their recorded attribution.'; + +COMMENT ON COLUMN template_usage_stats_session_families.usage_mins IS 'Total minutes the user has been using the family. Minutes shared by two apps of the family count once.'; + +CREATE TABLE template_usage_stats_session_apps ( + start_time timestamptz NOT NULL, + template_id uuid NOT NULL, + user_id uuid NOT NULL, + app_name text NOT NULL, + usage_mins smallint NOT NULL, + + PRIMARY KEY (start_time, user_id, template_id, app_name), + FOREIGN KEY (start_time, template_id, user_id) + REFERENCES template_usage_stats (start_time, template_id, user_id) + ON DELETE CASCADE +); + +COMMENT ON TABLE template_usage_stats_session_apps IS 'Session usage of each template_usage_stats bucket, split by app name. A bucket with family rows but no rows here predates per-app recording, so its per-app usage is unknown rather than zero.'; + +COMMENT ON COLUMN template_usage_stats_session_apps.app_name IS 'App name as the agent reported it, so it is a source label rather than a curated identity. An agent that reports only the fixed session counts reports family names here, as does history converted by migration 000590.'; + +COMMENT ON COLUMN template_usage_stats_session_apps.usage_mins IS 'Total minutes the user has been using the app.'; + +-- Carry every family the fixed columns recorded, sftp included: the rollup has +-- never written it, but a row that has a value must not lose it. Zero minutes +-- are skipped so a bucket has rows only for the families it saw, which is what +-- the rollup writes from now on. No app rows are written: the fixed columns +-- only ever recorded the family, so per-app usage stays unknown for these +-- buckets rather than being invented from family totals. +INSERT INTO template_usage_stats_session_families (start_time, template_id, user_id, family, usage_mins) +SELECT + tus.start_time, + tus.template_id, + tus.user_id, + families.family, + families.usage_mins +FROM + template_usage_stats AS tus, + LATERAL (VALUES + ('ssh', tus.ssh_mins), + ('sftp', tus.sftp_mins), + ('reconnecting_pty', tus.reconnecting_pty_mins), + ('vscode', tus.vscode_mins), + ('jetbrains', tus.jetbrains_mins) + ) AS families(family, usage_mins) +WHERE + families.usage_mins > 0; + +ALTER TABLE template_usage_stats + DROP COLUMN ssh_mins, + DROP COLUMN sftp_mins, + DROP COLUMN reconnecting_pty_mins, + DROP COLUMN vscode_mins, + DROP COLUMN jetbrains_mins; diff --git a/coderd/database/migrations/migration000591_test.go b/coderd/database/migrations/migration000591_test.go new file mode 100644 index 00000000000..e6ab1cbdf03 --- /dev/null +++ b/coderd/database/migrations/migration000591_test.go @@ -0,0 +1,365 @@ +package migrations_test + +import ( + "database/sql" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database/migrations" + "github.com/coder/coder/v2/testutil" +) + +// stepTo advances the migrations to version and fails if it is never reached. +func stepTo(t *testing.T, sqlDB *sql.DB, version uint) { + t.Helper() + + next, err := migrations.Stepper(sqlDB) + require.NoError(t, err) + for { + got, more, err := next() + require.NoError(t, err) + if !more { + t.Fatalf("migration %d not found", version) + } + if got == version { + return + } + } +} + +// familyRow is one row of template_usage_stats_session_families, or of +// template_usage_stats_session_apps, which has the same shape. +type familyRow struct { + name string + usageMins int64 +} + +// sessionRows reads a child table ordered by name. +func sessionRows(t *testing.T, tx *sql.Tx, table, nameColumn string) []familyRow { + t.Helper() + + //nolint:gosec // Table and column names are constants in this test. + rows, err := tx.Query("SELECT " + nameColumn + ", usage_mins FROM " + table + " ORDER BY " + nameColumn) + require.NoError(t, err) + defer rows.Close() + + var got []familyRow + for rows.Next() { + var row familyRow + require.NoError(t, rows.Scan(&row.name, &row.usageMins)) + got = append(got, row) + } + require.NoError(t, rows.Err()) + return got +} + +// TestMigration000591TemplateUsageStatsSessionUsage covers the conversion of +// the fixed per-family minute columns into the family child table, which the +// testdata/fixtures run does not reach: its template_usage_stats rows record +// no session minutes, so the backfill matches zero rows in CI. +// +//nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. +func TestMigration000591TemplateUsageStatsSessionUsage(t *testing.T) { + t.Parallel() + + sqlDB := testSQLDB(t) + stepTo(t, sqlDB, 590) + + ctx := testutil.Context(t, testutil.WaitSuperLong) + migrationSQL, err := os.ReadFile("000591_template_usage_stats_session_usage.up.sql") + require.NoError(t, err) + // insertUsageStats writes one row per minute set, keyed by + // (ssh, sftp, reconnecting_pty, vscode, jetbrains). + insertUsageStats := func(t *testing.T, tx *sql.Tx, mins ...[5]int) { + t.Helper() + + for i, m := range mins { + _, err := tx.ExecContext(ctx, ` + INSERT INTO template_usage_stats ( + start_time, end_time, template_id, user_id, median_latency_ms, + usage_mins, ssh_mins, sftp_mins, reconnecting_pty_mins, + vscode_mins, jetbrains_mins, app_usage_mins + ) VALUES ( + date_trunc('hour', statement_timestamp()) + $1::bigint * interval '30 minutes', + date_trunc('hour', statement_timestamp()) + $1::bigint * interval '30 minutes' + interval '30 minutes', + gen_random_uuid(), gen_random_uuid(), NULL, 30, $2, $3, $4, $5, $6, NULL + ) + `, i, m[0], m[1], m[2], m[3], m[4]) + require.NoError(t, err) + } + } + + t.Run("up", func(t *testing.T) { + // A bucket gets one family row per family it saw, and no app rows at + // all: the fixed columns only ever recorded the family, so per-app + // usage stays unknown rather than being invented from family totals. + tests := []struct { + name string + mins [][5]int + wantFamily []familyRow + }{ + {name: "no rows"}, + { + name: "every family", + mins: [][5]int{{1, 2, 3, 4, 5}}, + wantFamily: []familyRow{ + {"jetbrains", 5}, {"reconnecting_pty", 3}, {"sftp", 2}, {"ssh", 1}, {"vscode", 4}, + }, + }, + { + // The rollup has never written sftp_mins, but a row that has + // a value must not lose it. + name: "sftp only", + mins: [][5]int{{0, 7, 0, 0, 0}}, + wantFamily: []familyRow{{"sftp", 7}}, + }, + { + name: "zero minutes produce no row", + mins: [][5]int{{5, 0, 0, 6, 0}}, + wantFamily: []familyRow{{"ssh", 5}, {"vscode", 6}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + + insertUsageStats(t, tx, tt.mins...) + + _, err = tx.ExecContext(ctx, string(migrationSQL)) + require.NoError(t, err) + + require.Equal(t, tt.wantFamily, sessionRows(t, tx, "template_usage_stats_session_families", "family")) + require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_apps", "app_name")) + }) + } + }) + + // The child tables only hold session usage of buckets that exist, and they + // follow their bucket when it is deleted, which is how dbpurge and the + // retention deletes stay correct without knowing about them. + t.Run("child rows require and follow their bucket", func(t *testing.T) { + tx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + + _, err = tx.ExecContext(ctx, string(migrationSQL)) + require.NoError(t, err) + + for _, table := range []string{ + "template_usage_stats_session_families", + "template_usage_stats_session_apps", + } { + _, err = tx.ExecContext(ctx, `SAVEPOINT before_orphan`) + require.NoError(t, err) + //nolint:gosec // The table name is a constant in this test. + _, err = tx.ExecContext(ctx, ` + INSERT INTO `+table+` VALUES ( + date_trunc('hour', statement_timestamp()), + gen_random_uuid(), gen_random_uuid(), 'ssh', 1 + ) + `) + require.ErrorContains(t, err, "violates foreign key constraint", "%s", table) + _, err = tx.ExecContext(ctx, `ROLLBACK TO SAVEPOINT before_orphan`) + require.NoError(t, err) + } + + _, err = tx.ExecContext(ctx, ` + INSERT INTO template_usage_stats ( + start_time, end_time, template_id, user_id, median_latency_ms, + usage_mins, app_usage_mins + ) VALUES ( + date_trunc('hour', statement_timestamp()), + date_trunc('hour', statement_timestamp()) + interval '30 minutes', + '22222222-2222-2222-2222-222222222222'::uuid, + '11111111-1111-1111-1111-111111111111'::uuid, + NULL, 30, NULL + ) + `) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, ` + INSERT INTO template_usage_stats_session_families ( + start_time, template_id, user_id, family, usage_mins + ) VALUES ( + date_trunc('hour', statement_timestamp()), + '22222222-2222-2222-2222-222222222222'::uuid, + '11111111-1111-1111-1111-111111111111'::uuid, 'ssh', 1 + ) + `) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, ` + INSERT INTO template_usage_stats_session_apps ( + start_time, template_id, user_id, app_name, usage_mins + ) VALUES ( + date_trunc('hour', statement_timestamp()), + '22222222-2222-2222-2222-222222222222'::uuid, + '11111111-1111-1111-1111-111111111111'::uuid, 'zed', 1 + ) + `) + require.NoError(t, err) + + _, err = tx.ExecContext(ctx, `DELETE FROM template_usage_stats`) + require.NoError(t, err) + require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_families", "family")) + require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_apps", "app_name")) + }) +} + +// TestMigration000591ChainFrom589 walks the whole window this change spans, +// 589 up to 591 and back down to 589, with data present at every step. The +// isolated 591 tests start at 590, so they never see 590 converting raw +// session counts the rollup has not consumed, which is the state an upgrade +// actually finds. +// +//nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. +func TestMigration000591ChainFrom589(t *testing.T) { + t.Parallel() + + sqlDB := testSQLDB(t) + stepTo(t, sqlDB, 589) + + ctx := testutil.Context(t, testutil.WaitSuperLong) + up590, err := os.ReadFile("000590_workspace_agent_session_counts.up.sql") + require.NoError(t, err) + down590, err := os.ReadFile("000590_workspace_agent_session_counts.down.sql") + require.NoError(t, err) + up591, err := os.ReadFile("000591_template_usage_stats_session_usage.up.sql") + require.NoError(t, err) + down591, err := os.ReadFile("000591_template_usage_stats_session_usage.down.sql") + require.NoError(t, err) + + // backlogHours spans more than a day, and two backlogged rows sit inside + // that span, so 590's backlog warning branch runs. That is the slow path an + // upgrade of a stalled deployment takes. + const backlogHours = 48 + + // chainStep names a migration file so a failure says which step broke. + type chainStep struct { + name string + sql []byte + } + + tx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + + // One rolled-up half hour, so 590 has a watermark to measure the backlog + // against, and so 591 has a row whose fixed family minutes must convert. + _, err = tx.ExecContext(ctx, ` + INSERT INTO template_usage_stats ( + start_time, end_time, template_id, user_id, median_latency_ms, + usage_mins, ssh_mins, sftp_mins, reconnecting_pty_mins, + vscode_mins, jetbrains_mins, app_usage_mins + ) VALUES ( + date_trunc('hour', statement_timestamp()) - $1::bigint * interval '1 hour', + date_trunc('hour', statement_timestamp()) - $1::bigint * interval '1 hour' + interval '30 minutes', + '22222222-2222-2222-2222-222222222222'::uuid, + '11111111-1111-1111-1111-111111111111'::uuid, + NULL, 30, 3, 2, 0, 4, 0, NULL + ) + `, backlogHours) + require.NoError(t, err) + + // Raw agent stats the rollup has not consumed: two inside the backlog, + // spanning more than a day so 590 measures a wide backlog and takes its + // warning path, and one older than the window 590 converts, which is the + // watermark less the day DeleteOldWorkspaceAgentStats retains. 590 leaves + // that one as an empty map because template_usage_stats accounts for it. + _, err = tx.ExecContext(ctx, ` + INSERT INTO workspace_agent_stats ( + id, created_at, user_id, agent_id, workspace_id, template_id, + connection_count, session_count_vscode, session_count_ssh + ) VALUES ( + gen_random_uuid(), statement_timestamp() - interval '5 minutes', + '11111111-1111-1111-1111-111111111111'::uuid, gen_random_uuid(), + gen_random_uuid(), '22222222-2222-2222-2222-222222222222'::uuid, 1, 2, 1 + ), ( + gen_random_uuid(), statement_timestamp() - ($1::bigint - 1) * interval '1 hour', + '11111111-1111-1111-1111-111111111111'::uuid, gen_random_uuid(), + gen_random_uuid(), '22222222-2222-2222-2222-222222222222'::uuid, 1, 4, 2 + ), ( + gen_random_uuid(), statement_timestamp() - $1::bigint * interval '1 hour' - interval '30 hours', + '11111111-1111-1111-1111-111111111111'::uuid, gen_random_uuid(), + gen_random_uuid(), '22222222-2222-2222-2222-222222222222'::uuid, 1, 5, 5 + ) + `, backlogHours) + require.NoError(t, err) + + for _, step := range []chainStep{{"590 up", up590}, {"591 up", up591}} { + _, err = tx.ExecContext(ctx, string(step.sql)) + require.NoError(t, err, "%s", step.name) + } + + // 590 converted both backlogged rows under the canonical family names, and + // zeroed the row the rollup had already consumed. + rows, err := tx.QueryContext(ctx, ` + SELECT session_counts FROM workspace_agent_stats ORDER BY created_at + `) + require.NoError(t, err) + var gotCounts []string + for rows.Next() { + var counts []byte + require.NoError(t, rows.Scan(&counts)) + gotCounts = append(gotCounts, string(counts)) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Len(t, gotCounts, 3) + require.JSONEq(t, `{}`, gotCounts[0], "older than the converted window") + require.JSONEq(t, `{"vscode": 4, "ssh": 2}`, gotCounts[1], "backlogged, over a day old") + require.JSONEq(t, `{"vscode": 2, "ssh": 1}`, gotCounts[2], "backlogged, recent") + + // 591 converted the fixed family minutes and recorded no per-app usage. + require.Equal(t, []familyRow{{"sftp", 2}, {"ssh", 3}, {"vscode", 4}}, + sessionRows(t, tx, "template_usage_stats_session_families", "family")) + require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_apps", "app_name")) + + // Back down: 591 restores the fixed columns, then 590 restores the fixed + // session counts, landing on the 589 schema. + for _, step := range []chainStep{{"591 down", down591}, {"590 down", down590}} { + _, err = tx.ExecContext(ctx, string(step.sql)) + require.NoError(t, err, "%s", step.name) + } + + var ssh, sftp, reconnectingPTY, vscode, jetbrains int64 + err = tx.QueryRowContext(ctx, ` + SELECT ssh_mins, sftp_mins, reconnecting_pty_mins, vscode_mins, jetbrains_mins + FROM template_usage_stats + `).Scan(&ssh, &sftp, &reconnectingPTY, &vscode, &jetbrains) + require.NoError(t, err) + require.EqualValues(t, 3, ssh) + require.EqualValues(t, 2, sftp) + require.EqualValues(t, 0, reconnectingPTY) + require.EqualValues(t, 4, vscode) + require.EqualValues(t, 0, jetbrains) + + var backloggedVSCode, backloggedSSH int64 + err = tx.QueryRowContext(ctx, ` + SELECT session_count_vscode, session_count_ssh + FROM workspace_agent_stats + ORDER BY created_at DESC + LIMIT 1 + `).Scan(&backloggedVSCode, &backloggedSSH) + require.NoError(t, err) + require.EqualValues(t, 2, backloggedVSCode) + require.EqualValues(t, 1, backloggedSSH) + + // Everything both migrations added is gone, so the 589 schema is back. + var leftovers int + err = tx.QueryRowContext(ctx, ` + SELECT + (SELECT COUNT(*) FROM information_schema.columns + WHERE table_name = 'workspace_agent_stats' AND column_name = 'session_counts') + + (SELECT COUNT(*) FROM information_schema.tables + WHERE table_name IN ( + 'template_usage_stats_session_families', + 'template_usage_stats_session_apps' + )) + `).Scan(&leftovers) + require.NoError(t, err) + require.Zero(t, leftovers) +} diff --git a/coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql new file mode 100644 index 00000000000..f268426f33e --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql @@ -0,0 +1,48 @@ +-- The 000591 backfill produces family rows only, because the fixed columns it +-- converts never recorded an app name. App rows exist only for buckets the +-- rollup wrote after this migration, so the fixture seeds one directly. +INSERT INTO template_usage_stats ( + start_time, + end_time, + template_id, + user_id, + median_latency_ms, + usage_mins, + app_usage_mins +) VALUES ( + date_trunc('hour', NOW()) + '30 minute'::interval, + date_trunc('hour', NOW()) + '60 minute'::interval, + '8f2b1c9e-6d4a-4f3b-8a7c-1e5d9b3a2c40', + '2c7e5a1b-9d38-4c6f-b2e4-7a1f8c3d5b90', + 1, + 2, + NULL +); + +INSERT INTO template_usage_stats_session_families ( + start_time, + template_id, + user_id, + family, + usage_mins +) VALUES ( + date_trunc('hour', NOW()) + '30 minute'::interval, + '8f2b1c9e-6d4a-4f3b-8a7c-1e5d9b3a2c40', + '2c7e5a1b-9d38-4c6f-b2e4-7a1f8c3d5b90', + 'vscode', + 2 +); + +INSERT INTO template_usage_stats_session_apps ( + start_time, + template_id, + user_id, + app_name, + usage_mins +) VALUES ( + date_trunc('hour', NOW()) + '30 minute'::interval, + '8f2b1c9e-6d4a-4f3b-8a7c-1e5d9b3a2c40', + '2c7e5a1b-9d38-4c6f-b2e4-7a1f8c3d5b90', + 'cursor', + 2 +); diff --git a/coderd/database/models.go b/coderd/database/models.go index 91b561d91d9..64296545280 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -6048,20 +6048,32 @@ type TemplateUsageStat struct { MedianLatencyMs sql.NullFloat64 `db:"median_latency_ms" json:"median_latency_ms"` // Total minutes the user has been using the template. UsageMins int16 `db:"usage_mins" json:"usage_mins"` - // Total minutes the user has been using SSH. - SshMins int16 `db:"ssh_mins" json:"ssh_mins"` - // Total minutes the user has been using SFTP. - SftpMins int16 `db:"sftp_mins" json:"sftp_mins"` - // Total minutes the user has been using the reconnecting PTY. - ReconnectingPtyMins int16 `db:"reconnecting_pty_mins" json:"reconnecting_pty_mins"` - // Total minutes the user has been using VSCode. - VscodeMins int16 `db:"vscode_mins" json:"vscode_mins"` - // Total minutes the user has been using JetBrains. - JetbrainsMins int16 `db:"jetbrains_mins" json:"jetbrains_mins"` // Object with app names as keys and total minutes used as values. Null means no app usage was recorded. AppUsageMins StringMapOfInt `db:"app_usage_mins" json:"app_usage_mins"` } +// Session usage of each template_usage_stats bucket, split by app name. A bucket with family rows but no rows here predates per-app recording, so its per-app usage is unknown rather than zero. +type TemplateUsageStatsSessionApp struct { + StartTime time.Time `db:"start_time" json:"start_time"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + UserID uuid.UUID `db:"user_id" json:"user_id"` + // App name as the agent reported it, so it is a source label rather than a curated identity. An agent that reports only the fixed session counts reports family names here, as does history converted by migration 000590. + AppName string `db:"app_name" json:"app_name"` + // Total minutes the user has been using the app. + UsageMins int16 `db:"usage_mins" json:"usage_mins"` +} + +// Session usage of each template_usage_stats bucket, split by app family. A bucket with no row here recorded no session usage. +type TemplateUsageStatsSessionFamily struct { + StartTime time.Time `db:"start_time" json:"start_time"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + UserID uuid.UUID `db:"user_id" json:"user_id"` + // Family name the registry attributed the session to when the bucket was last rolled up, including 'unknown' for an app name the registry did not know. Buckets the rollup no longer revisits keep their recorded attribution. + Family string `db:"family" json:"family"` + // Total minutes the user has been using the family. Minutes shared by two apps of the family count once. + UsageMins int16 `db:"usage_mins" json:"usage_mins"` +} + // Joins in the username + avatar url of the created by user. type TemplateVersion struct { ID uuid.UUID `db:"id" json:"id"` diff --git a/coderd/database/querier.go b/coderd/database/querier.go index e63f9826f7b..3db413ee51d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -841,6 +841,10 @@ type sqlcQuerier interface { // workspaces in a given timeframe. The template IDs, active users, and // usage_seconds all reflect any usage in the template, including apps. // + // Session usage comes from the family child table exactly as the rollup + // recorded it, so a family the rollup learns about later is reported without a + // change here. + // // When combining data from multiple templates, we must make a guess at // how the user behaved for the 30 minute interval. In this case we make // the assumption that if the user used two workspaces for 15 minutes, @@ -1718,6 +1722,13 @@ type sqlcQuerier interface { // into a single table for efficient storage and querying. Half-hour buckets are // used to store the data, and the minutes are summed for each user and template // combination. The result is stored in the template_usage_stats table. + // + // Session usage is stored per app name and per app family in the child tables, + // so the main row carries no session columns at all. Every recomputed bucket + // rewrites its own child rows: names that disappeared are deleted, the rest + // are upserted. The keys come from the computed set rather than from the main + // upsert, because the no-op guard below suppresses main rows whose columns did + // not change while their session usage still has to be corrected. UpsertTemplateUsageStats(ctx context.Context, appFamilies json.RawMessage) error UpsertUserAIBudgetOverride(ctx context.Context, arg UpsertUserAIBudgetOverrideParams) (UserAIBudgetOverride, error) // UpsertUserAIProviderKey preserves the original id and created_at when the diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 23a9c0c2f36..7a6ccbb46bd 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -864,22 +864,15 @@ func TestGetTemplateInsightsByTemplate(t *testing.T) { AppFamilies: appFamilies, }) require.NoError(t, err) - // The query does not order its rows. - require.ElementsMatch(t, []database.GetTemplateInsightsByTemplateRow{ - { - TemplateID: templateID, - ActiveUsers: 2, - UsageVscodeSeconds: 120, - UsageJetbrainsSeconds: 60, - UsageReconnectingPtySeconds: 60, - UsageSshSeconds: 120, - }, - { - TemplateID: sharedConnectionTemplateID, - ActiveUsers: 1, - UsageVscodeSeconds: 60, - }, - }, insights) + byTemplate := make(map[uuid.UUID]database.GetTemplateInsightsByTemplateRow) + for _, row := range insights { + byTemplate[row.TemplateID] = row + } + require.Len(t, byTemplate, 2) + require.EqualValues(t, 2, byTemplate[templateID].ActiveUsers) + require.JSONEq(t, `{"vscode":120,"jetbrains":60,"reconnecting_pty":60,"ssh":120,"unknown":60}`, string(byTemplate[templateID].SessionFamilyUsageSeconds)) + require.EqualValues(t, 1, byTemplate[sharedConnectionTemplateID].ActiveUsers) + require.JSONEq(t, `{"vscode":60,"unknown":60}`, string(byTemplate[sharedConnectionTemplateID].SessionFamilyUsageSeconds)) } func TestGetWorkspaceAgentUsageStats(t *testing.T) { @@ -19012,15 +19005,14 @@ func TestSessionCountsAttributeByFamily(t *testing.T) { for _, row := range insights { byTemplate[row.TemplateID] = row } - require.Equal(t, int64(60), byTemplate[cursorTemplate].UsageVscodeSeconds) - require.Equal(t, int64(60), byTemplate[zedTemplate].UsageSshSeconds) + require.JSONEq(t, `{"vscode":60}`, string(byTemplate[cursorTemplate].SessionFamilyUsageSeconds)) + require.JSONEq(t, `{"ssh":60}`, string(byTemplate[zedTemplate].SessionFamilyUsageSeconds)) // An app with no family is still activity, so the user is not counted idle. unknown, ok := byTemplate[unknownTemplate] require.True(t, ok, "a session with no family must still appear as usage") require.Equal(t, int64(1), unknown.ActiveUsers) - require.Zero(t, unknown.UsageVscodeSeconds) - require.Zero(t, unknown.UsageSshSeconds) + require.JSONEq(t, `{"unknown":60}`, string(unknown.SessionFamilyUsageSeconds)) } // The rollup attributes session counts the same way the read queries do, so a @@ -19080,24 +19072,22 @@ func TestUpsertTemplateUsageStatsAttributesSessionCountsByFamily(t *testing.T) { cursor, ok := byTemplate[cursorTemplate] require.True(t, ok, "a VS Code fork must be rolled up") require.Equal(t, int16(1), cursor.UsageMins) - require.Equal(t, int16(1), cursor.VscodeMins) - require.Zero(t, cursor.SshMins) + require.Equal(t, map[string]int64{"vscode": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", cursor.StartTime, cursor.UserID, cursorTemplate)) + require.Equal(t, map[string]int64{"cursor": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", cursor.StartTime, cursor.UserID, cursorTemplate)) zed, ok := byTemplate[zedTemplate] require.True(t, ok, "an SSH-speaking editor must be rolled up") require.Equal(t, int16(1), zed.UsageMins) - require.Equal(t, int16(1), zed.SshMins) - require.Zero(t, zed.VscodeMins) + require.Equal(t, map[string]int64{"ssh": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", zed.StartTime, zed.UserID, zedTemplate)) + require.Equal(t, map[string]int64{"zed": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", zed.StartTime, zed.UserID, zedTemplate)) // An app with no family is still activity, so it produces usage minutes - // without any family minutes. + // attributed to the unknown family. unknown, ok := byTemplate[unknownTemplate] require.True(t, ok, "a session with no family must still appear as usage") require.Equal(t, int16(1), unknown.UsageMins) - require.Zero(t, unknown.VscodeMins) - require.Zero(t, unknown.SshMins) - require.Zero(t, unknown.JetbrainsMins) - require.Zero(t, unknown.ReconnectingPtyMins) + require.Equal(t, map[string]int64{"unknown": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", unknown.StartTime, unknown.UserID, unknownTemplate)) + require.Equal(t, map[string]int64{"some_new_ide": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", unknown.StartTime, unknown.UserID, unknownTemplate)) } func sessionFamilyCounts(t *testing.T, data json.RawMessage) map[codersdk.AppFamilyName]int64 { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index fd710b3194a..f1757920680 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -16645,57 +16645,135 @@ func (q *sqlQuerier) GetTemplateAppInsightsByTemplate(ctx context.Context, arg G const getTemplateInsights = `-- name: GetTemplateInsights :one WITH - insights AS ( + base AS MATERIALIZED ( + -- One pass over the main table answers three questions: how many + -- templates each user touched in a half hour, that user's capped + -- minutes, and the list of templates in the window. GROUPING marks + -- which of the two sets a row belongs to. SELECT + GROUPING(template_id) AS is_user_row, + start_time, user_id, - -- See motivation in GetTemplateInsights for LEAST(SUM(n), 30). - LEAST(SUM(usage_mins), 30) AS usage_mins, - LEAST(SUM(ssh_mins), 30) AS ssh_mins, - LEAST(SUM(sftp_mins), 30) AS sftp_mins, - LEAST(SUM(reconnecting_pty_mins), 30) AS reconnecting_pty_mins, - LEAST(SUM(vscode_mins), 30) AS vscode_mins, - LEAST(SUM(jetbrains_mins), 30) AS jetbrains_mins + template_id, + COUNT(*) AS templates, + LEAST(SUM(usage_mins), 30) AS usage_mins FROM template_usage_stats WHERE start_time >= $1::timestamptz AND end_time <= $2::timestamptz AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN template_id = ANY($3::uuid[]) ELSE TRUE END + GROUP BY GROUPING SETS ((start_time, user_id), (template_id)) + ), + users AS ( + SELECT + start_time, + user_id, + templates, + usage_mins + FROM + base + WHERE + is_user_row = 1 + ), + multi AS MATERIALIZED ( + -- Only a user who used more than one template in the same half hour + -- can exceed the 30 minute cap, and there are usually none. + SELECT + start_time, + user_id + FROM + users + WHERE + templates > 1 + ), + single_family_usage AS ( + -- Everything but the multi-template buckets, which cannot exceed the + -- cap, so they need no per-user grouping. This also collects the + -- template list per family, for which the cap is irrelevant. + SELECT + sessions.family, + sessions.template_id, + SUM(LEAST(sessions.usage_mins, 30)) FILTER ( + WHERE (sessions.start_time, sessions.user_id) NOT IN (SELECT start_time, user_id FROM multi) + ) AS usage_mins + FROM + template_usage_stats_session_families AS sessions + WHERE + sessions.start_time >= $1::timestamptz + AND sessions.start_time + '30 minutes'::interval <= $2::timestamptz + AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY($3::uuid[]) ELSE TRUE END + AND sessions.usage_mins > 0 GROUP BY - start_time, user_id + sessions.family, sessions.template_id ), - templates AS ( + multi_family_usage AS ( + -- See motivation in GetTemplateInsights for LEAST(SUM(n), 30). SELECT - array_agg(DISTINCT template_id) AS template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE ssh_mins > 0) AS ssh_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE sftp_mins > 0) AS sftp_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE reconnecting_pty_mins > 0) AS reconnecting_pty_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE vscode_mins > 0) AS vscode_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE jetbrains_mins > 0) AS jetbrains_template_ids + sessions.family, + LEAST(SUM(sessions.usage_mins), 30) AS usage_mins FROM - template_usage_stats + template_usage_stats_session_families AS sessions WHERE - start_time >= $1::timestamptz - AND end_time <= $2::timestamptz - AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN template_id = ANY($3::uuid[]) ELSE TRUE END + sessions.start_time >= $1::timestamptz + AND sessions.start_time + '30 minutes'::interval <= $2::timestamptz + AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY($3::uuid[]) ELSE TRUE END + AND sessions.usage_mins > 0 + AND EXISTS ( + SELECT 1 + FROM multi + WHERE multi.start_time = sessions.start_time + AND multi.user_id = sessions.user_id + ) + GROUP BY + sessions.start_time, sessions.user_id, sessions.family + ), + family_usage AS ( + SELECT + family, + (SUM(usage_mins) * 60)::bigint AS usage_seconds + FROM ( + SELECT + family, + COALESCE(SUM(usage_mins), 0) AS usage_mins + FROM + single_family_usage + GROUP BY + family + + UNION ALL + + SELECT + family, + SUM(usage_mins) AS usage_mins + FROM + multi_family_usage + GROUP BY + family + ) AS parts + GROUP BY + family + ), + family_templates AS ( + SELECT + family, + array_agg(DISTINCT template_id) AS template_ids + FROM + single_family_usage + GROUP BY + family ) SELECT - COALESCE((SELECT template_ids FROM templates), '{}')::uuid[] AS template_ids, -- Includes app usage. - COALESCE((SELECT ssh_template_ids FROM templates), '{}')::uuid[] AS ssh_template_ids, - COALESCE((SELECT sftp_template_ids FROM templates), '{}')::uuid[] AS sftp_template_ids, - COALESCE((SELECT reconnecting_pty_template_ids FROM templates), '{}')::uuid[] AS reconnecting_pty_template_ids, - COALESCE((SELECT vscode_template_ids FROM templates), '{}')::uuid[] AS vscode_template_ids, - COALESCE((SELECT jetbrains_template_ids FROM templates), '{}')::uuid[] AS jetbrains_template_ids, + COALESCE((SELECT array_agg(DISTINCT template_id) FROM base WHERE is_user_row = 0), '{}')::uuid[] AS template_ids, -- Includes app usage. COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage. COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage. - COALESCE(SUM(ssh_mins) * 60, 0)::bigint AS usage_ssh_seconds, - COALESCE(SUM(sftp_mins) * 60, 0)::bigint AS usage_sftp_seconds, - COALESCE(SUM(reconnecting_pty_mins) * 60, 0)::bigint AS usage_reconnecting_pty_seconds, - COALESCE(SUM(vscode_mins) * 60, 0)::bigint AS usage_vscode_seconds, - COALESCE(SUM(jetbrains_mins) * 60, 0)::bigint AS usage_jetbrains_seconds + -- Family name to usage seconds, and family name to the templates that saw + -- the family. + COALESCE((SELECT jsonb_object_agg(family, usage_seconds) FROM family_usage), '{}'::jsonb)::jsonb AS session_family_usage_seconds, + COALESCE((SELECT jsonb_object_agg(family, template_ids) FROM family_templates), '{}'::jsonb)::jsonb AS session_family_template_ids FROM - insights + users ` type GetTemplateInsightsParams struct { @@ -16705,25 +16783,21 @@ type GetTemplateInsightsParams struct { } type GetTemplateInsightsRow struct { - TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` - SshTemplateIds []uuid.UUID `db:"ssh_template_ids" json:"ssh_template_ids"` - SftpTemplateIds []uuid.UUID `db:"sftp_template_ids" json:"sftp_template_ids"` - ReconnectingPtyTemplateIds []uuid.UUID `db:"reconnecting_pty_template_ids" json:"reconnecting_pty_template_ids"` - VscodeTemplateIds []uuid.UUID `db:"vscode_template_ids" json:"vscode_template_ids"` - JetbrainsTemplateIds []uuid.UUID `db:"jetbrains_template_ids" json:"jetbrains_template_ids"` - ActiveUsers int64 `db:"active_users" json:"active_users"` - UsageTotalSeconds int64 `db:"usage_total_seconds" json:"usage_total_seconds"` - UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"` - UsageSftpSeconds int64 `db:"usage_sftp_seconds" json:"usage_sftp_seconds"` - UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"` - UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"` - UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"` + TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"` + ActiveUsers int64 `db:"active_users" json:"active_users"` + UsageTotalSeconds int64 `db:"usage_total_seconds" json:"usage_total_seconds"` + SessionFamilyUsageSeconds json.RawMessage `db:"session_family_usage_seconds" json:"session_family_usage_seconds"` + SessionFamilyTemplateIds json.RawMessage `db:"session_family_template_ids" json:"session_family_template_ids"` } // GetTemplateInsights returns the aggregate user-produced usage of all // workspaces in a given timeframe. The template IDs, active users, and // usage_seconds all reflect any usage in the template, including apps. // +// Session usage comes from the family child table exactly as the rollup +// recorded it, so a family the rollup learns about later is reported without a +// change here. +// // When combining data from multiple templates, we must make a guess at // how the user behaved for the 30 minute interval. In this case we make // the assumption that if the user used two workspaces for 15 minutes, @@ -16734,18 +16808,10 @@ func (q *sqlQuerier) GetTemplateInsights(ctx context.Context, arg GetTemplateIns var i GetTemplateInsightsRow err := row.Scan( pq.Array(&i.TemplateIDs), - pq.Array(&i.SshTemplateIds), - pq.Array(&i.SftpTemplateIds), - pq.Array(&i.ReconnectingPtyTemplateIds), - pq.Array(&i.VscodeTemplateIds), - pq.Array(&i.JetbrainsTemplateIds), &i.ActiveUsers, &i.UsageTotalSeconds, - &i.UsageSshSeconds, - &i.UsageSftpSeconds, - &i.UsageReconnectingPtySeconds, - &i.UsageVscodeSeconds, - &i.UsageJetbrainsSeconds, + &i.SessionFamilyUsageSeconds, + &i.SessionFamilyTemplateIds, ) return i, err } @@ -16839,72 +16905,107 @@ func (q *sqlQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg GetT const getTemplateInsightsByTemplate = `-- name: GetTemplateInsightsByTemplate :many WITH - -- app_families maps each attributed family to its app names, so the - -- probes below stay one expression per family: adding a family needs a - -- new list in fams plus one probe here, because sqlc output columns are - -- static. fams turns the jsonb parameter into arrays once for the whole - -- query. These probes only ask whether any app of a family is present, - -- so the jsonb key-existence operator beats decomposing session_counts - -- per row (measured ~2.4x faster on a 1M row scan). - fams AS ( + expanded AS ( + -- Each row's app names are expanded once and mapped to their family. + -- app_families maps app name to family name; an app the registry does + -- not know is attributed to 'unknown' rather than dropped. SELECT - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'ssh')) AS ssh, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'reconnecting_pty')) AS reconnecting_pty, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'vscode')) AS vscode, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'jetbrains')) AS jetbrains + was.template_id, + was.user_id, + date_trunc('minute', was.created_at) AS minute, + COALESCE($1::jsonb ->> app_name, 'unknown') AS family + FROM + workspace_agent_stats AS was, jsonb_object_keys(was.session_counts) AS app_name + WHERE + was.created_at >= $2::timestamptz + AND was.created_at < $3::timestamptz + AND was.session_counts <> '{}'::jsonb ), - -- Deduplicate activity by template, user, and minute. - minute_activity AS ( + minute_family AS ( + -- Deduplicate activity by template, user, minute, and family, so a + -- minute with two apps of one family counts once for that family. SELECT template_id, user_id, - date_trunc('minute', created_at) AS minute, - BOOL_OR(session_counts ?| fams.ssh) AS ssh, - BOOL_OR(session_counts ?| fams.reconnecting_pty) AS reconnecting_pty, - BOOL_OR(session_counts ?| fams.vscode) AS vscode, - BOOL_OR(session_counts ?| fams.jetbrains) AS jetbrains, - BOOL_OR(connection_count > 0) AS has_connection + minute, + family FROM - workspace_agent_stats, fams + expanded + GROUP BY + template_id, user_id, minute, family + ), + connected AS ( + -- NOTE(mafredri): The agent stats are currently very unreliable, and + -- sometimes the connections are missing, even during active sessions. + -- Since we can't fully rely on this, we check for "any connection + -- within this bucket". A better solution here would be preferable. + SELECT + template_id, + user_id + FROM + workspace_agent_stats WHERE created_at >= $2::timestamptz AND created_at < $3::timestamptz AND session_counts <> '{}'::jsonb GROUP BY - template_id, user_id, minute + template_id, user_id + HAVING + BOOL_OR(connection_count > 0) ), insights AS ( + SELECT + mf.template_id, + mf.user_id, + mf.family, + COUNT(*) AS usage_mins + FROM + minute_family AS mf + JOIN + connected AS c + ON + c.template_id = mf.template_id + AND c.user_id = mf.user_id + GROUP BY + mf.template_id, mf.user_id, mf.family + ), + family_usage AS ( SELECT template_id, - user_id, - COUNT(*) FILTER (WHERE ssh) AS ssh_mins, - COUNT(*) FILTER (WHERE reconnecting_pty) AS reconnecting_pty_mins, - COUNT(*) FILTER (WHERE vscode) AS vscode_mins, - COUNT(*) FILTER (WHERE jetbrains) AS jetbrains_mins, - -- NOTE(mafredri): The agent stats are currently very unreliable, and - -- sometimes the connections are missing, even during active sessions. - -- Since we can't fully rely on this, we check for "any connection - -- within this bucket". A better solution here would be preferable. - BOOL_OR(has_connection) AS has_connection + jsonb_object_agg(family, usage_seconds) AS session_family_usage_seconds + FROM ( + SELECT + template_id, + family, + (SUM(usage_mins) * 60)::bigint AS usage_seconds + FROM + insights + GROUP BY + template_id, family + ) AS family_seconds + GROUP BY + template_id + ), + active_users AS ( + SELECT + template_id, + COUNT(DISTINCT user_id)::bigint AS active_users FROM - minute_activity + insights GROUP BY - template_id, user_id + template_id ) SELECT - template_id, - COUNT(DISTINCT user_id)::bigint AS active_users, - (SUM(vscode_mins) * 60)::bigint AS usage_vscode_seconds, - (SUM(jetbrains_mins) * 60)::bigint AS usage_jetbrains_seconds, - (SUM(reconnecting_pty_mins) * 60)::bigint AS usage_reconnecting_pty_seconds, - (SUM(ssh_mins) * 60)::bigint AS usage_ssh_seconds + au.template_id, + au.active_users, + COALESCE(fu.session_family_usage_seconds, '{}'::jsonb)::jsonb AS session_family_usage_seconds FROM - insights -WHERE - has_connection -GROUP BY - template_id + active_users AS au +LEFT JOIN + family_usage AS fu +ON + fu.template_id = au.template_id ` type GetTemplateInsightsByTemplateParams struct { @@ -16914,12 +17015,9 @@ type GetTemplateInsightsByTemplateParams struct { } type GetTemplateInsightsByTemplateRow struct { - TemplateID uuid.UUID `db:"template_id" json:"template_id"` - ActiveUsers int64 `db:"active_users" json:"active_users"` - UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"` - UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"` - UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"` - UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + ActiveUsers int64 `db:"active_users" json:"active_users"` + SessionFamilyUsageSeconds json.RawMessage `db:"session_family_usage_seconds" json:"session_family_usage_seconds"` } // GetTemplateInsightsByTemplate is used for Prometheus metrics. Keep @@ -16933,14 +17031,7 @@ func (q *sqlQuerier) GetTemplateInsightsByTemplate(ctx context.Context, arg GetT var items []GetTemplateInsightsByTemplateRow for rows.Next() { var i GetTemplateInsightsByTemplateRow - if err := rows.Scan( - &i.TemplateID, - &i.ActiveUsers, - &i.UsageVscodeSeconds, - &i.UsageJetbrainsSeconds, - &i.UsageReconnectingPtySeconds, - &i.UsageSshSeconds, - ); err != nil { + if err := rows.Scan(&i.TemplateID, &i.ActiveUsers, &i.SessionFamilyUsageSeconds); err != nil { return nil, err } items = append(items, i) @@ -17062,7 +17153,7 @@ func (q *sqlQuerier) GetTemplateParameterInsights(ctx context.Context, arg GetTe const getTemplateUsageStats = `-- name: GetTemplateUsageStats :many SELECT - start_time, end_time, template_id, user_id, median_latency_ms, usage_mins, ssh_mins, sftp_mins, reconnecting_pty_mins, vscode_mins, jetbrains_mins, app_usage_mins + start_time, end_time, template_id, user_id, median_latency_ms, usage_mins, app_usage_mins FROM template_usage_stats WHERE @@ -17093,11 +17184,6 @@ func (q *sqlQuerier) GetTemplateUsageStats(ctx context.Context, arg GetTemplateU &i.UserID, &i.MedianLatencyMs, &i.UsageMins, - &i.SshMins, - &i.SftpMins, - &i.ReconnectingPtyMins, - &i.VscodeMins, - &i.JetbrainsMins, &i.AppUsageMins, ); err != nil { return nil, err @@ -17421,20 +17507,6 @@ func (q *sqlQuerier) GetUserStatusCounts(ctx context.Context, arg GetUserStatusC const upsertTemplateUsageStats = `-- name: UpsertTemplateUsageStats :exec WITH - -- app_families maps each attributed family to its app names, so the - -- probes below stay one expression per family: adding a family needs a - -- new list in fams plus one probe here, because sqlc output columns are - -- static. fams turns the jsonb parameter into arrays once for the whole - -- query. These probes only ask whether any app of a family is present, - -- so the jsonb key-existence operator beats decomposing session_counts - -- per row (measured ~2.4x faster on a 1M row scan). - fams AS ( - SELECT - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'ssh')) AS ssh, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'reconnecting_pty')) AS reconnecting_pty, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'vscode')) AS vscode, - ARRAY(SELECT jsonb_array_elements_text($1::jsonb -> 'jetbrains')) AS jetbrains - ), latest_start AS ( SELECT -- Truncate to hour so that we always look at even ranges of data. @@ -17504,35 +17576,118 @@ WITH GROUP BY time_bucket, w.template_id, fas.user_id, fas.access_method, fas.slug_or_port ), - agent_stats_buckets AS ( + agent_stats_rows AS ( + -- One filtered pass over workspace_agent_stats feeds both the bucket + -- grouping and the per-app mask grouping below, instead of scanning the + -- table once for each. The minute bit is computed here so the mask + -- grouping never touches created_at again. SELECT - -- Truncate the minute to the nearest half hour, this is the bucket size - -- for the data. date_trunc('hour', created_at) + trunc(date_part('minute', created_at) / 30) * 30 * '1 minute'::interval AS time_bucket, template_id, user_id, - -- Store each unique minute bucket for later merge between datasets. - array_agg(DISTINCT date_trunc('minute', created_at)) AS minute_buckets, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.ssh THEN date_trunc('minute', created_at) ELSE NULL END) AS ssh_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.reconnecting_pty THEN date_trunc('minute', created_at) ELSE NULL END) AS reconnecting_pty_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.vscode THEN date_trunc('minute', created_at) ELSE NULL END) AS vscode_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.jetbrains THEN date_trunc('minute', created_at) ELSE NULL END) AS jetbrains_mins, - -- NOTE(mafredri): The agent stats are currently very unreliable, and - -- sometimes the connections are missing, even during active sessions. - -- Since we can't fully rely on this, we check for "any connection - -- during this half-hour". A better solution here would be preferable. - MAX(connection_count) > 0 AS has_connection + date_trunc('minute', created_at) AS minute_bucket, + (1 << (date_part('minute', created_at)::int % 30)) AS minute_bit, + connection_count, + session_counts FROM - workspace_agent_stats, fams + workspace_agent_stats WHERE -- created_at >= @start_time::timestamptz -- AND created_at < @end_time::timestamptz created_at >= (SELECT t FROM latest_start) AND created_at < NOW() AND session_counts <> '{}'::jsonb + ), + agent_stats_buckets AS ( + SELECT + time_bucket, + template_id, + user_id, + -- Store each unique minute bucket for later merge between datasets. + array_agg(DISTINCT minute_bucket) AS minute_buckets, + -- NOTE(mafredri): The agent stats are currently very unreliable, and + -- sometimes the connections are missing, even during active sessions. + -- Since we can't fully rely on this, we check for "any connection + -- during this half-hour". A better solution here would be preferable. + MAX(connection_count) > 0 AS has_connection + FROM + agent_stats_rows GROUP BY time_bucket, template_id, user_id ), + app_family_registry AS ( + -- The session count attribution registry, app name to family name. + SELECT + app, + family + FROM + jsonb_each_text($1::jsonb) AS registry(app, family) + ), + agent_stats_app_minutes AS ( + -- One bit per minute of the half-hour bucket, per app name, instead of + -- a count: masks can be OR'd into a family below without counting a + -- minute twice when two apps of the same family were active in it. + -- Transient, only the minute counts derived from them are stored. + SELECT + time_bucket, + template_id, + user_id, + app_name, + bit_or(minute_bit) AS minute_mask + FROM + agent_stats_rows, jsonb_object_keys(session_counts) AS app_name + GROUP BY + time_bucket, template_id, user_id, app_name + ), + agent_stats_session_masks AS ( + -- One pass over the per-app masks emits both groupings: the app rows + -- keep each mask as it is, the family rows OR the masks of every app + -- in the family. An app name the registry does not know is attributed + -- to 'unknown' rather than dropped, so a newly reported app still + -- lands somewhere. GROUPING() marks which set a row came from, because + -- the app column is null in the family rows. + SELECT + time_bucket, + template_id, + user_id, + agent_stats_app_minutes.app_name, + COALESCE(app_family_registry.family, 'unknown') AS family, + GROUPING(agent_stats_app_minutes.app_name) AS family_group, + bit_or(minute_mask) AS minute_mask + FROM + agent_stats_app_minutes + LEFT JOIN + app_family_registry + ON + app_family_registry.app = agent_stats_app_minutes.app_name + GROUP BY GROUPING SETS ( + (time_bucket, template_id, user_id, agent_stats_app_minutes.app_name), + (time_bucket, template_id, user_id, COALESCE(app_family_registry.family, 'unknown')) + ) + ), + agent_stats_session_minutes AS ( + -- The minutes each name was active, counted from the bits set in its + -- mask. Postgres 13 has no bit_count, so the set bits are counted by + -- stripping the zeros out of the mask's text form. + SELECT + masks.time_bucket, + masks.template_id, + masks.user_id, + masks.family_group, + CASE WHEN masks.family_group = 0 THEN masks.app_name ELSE masks.family END AS name, + length(replace(masks.minute_mask::bit(30)::text, '0', ''))::smallint AS usage_mins + FROM + agent_stats_session_masks AS masks + JOIN + agent_stats_buckets AS buckets + ON + buckets.time_bucket = masks.time_bucket + AND buckets.template_id = masks.template_id + AND buckets.user_id = masks.user_id + -- The same gate the union below applies to agent stats, so a + -- bucket that only has app stats records no session usage. + AND buckets.has_connection + ), stats AS ( SELECT stats.time_bucket AS start_time, @@ -17542,11 +17697,6 @@ WITH -- Sum/distinct to handle zero/duplicate values due union and to unnest. COUNT(DISTINCT minute_bucket) AS usage_mins, array_agg(DISTINCT minute_bucket) AS minute_buckets, - SUM(DISTINCT stats.ssh_mins) AS ssh_mins, - SUM(DISTINCT stats.sftp_mins) AS sftp_mins, - SUM(DISTINCT stats.reconnecting_pty_mins) AS reconnecting_pty_mins, - SUM(DISTINCT stats.vscode_mins) AS vscode_mins, - SUM(DISTINCT stats.jetbrains_mins) AS jetbrains_mins, -- This is what we unnested, re-nest as json. jsonb_object_agg(stats.app_name, stats.app_minutes) FILTER (WHERE stats.app_name IS NOT NULL) AS app_usage_mins FROM ( @@ -17554,11 +17704,6 @@ WITH time_bucket, template_id, user_id, - 0 AS ssh_mins, - 0 AS sftp_mins, - 0 AS reconnecting_pty_mins, - 0 AS vscode_mins, - 0 AS jetbrains_mins, app_name, app_minutes, minute_buckets @@ -17571,12 +17716,6 @@ WITH time_bucket, template_id, user_id, - ssh_mins, - -- TODO(mafredri): Enable when we have the column. - 0 AS sftp_mins, - reconnecting_pty_mins, - vscode_mins, - jetbrains_mins, NULL AS app_name, NULL AS app_minutes, minute_buckets @@ -17623,65 +17762,156 @@ WITH AND was.connection_median_latency_ms > 0 GROUP BY mb.start_time, mb.template_id, mb.user_id + ), + upsert_stats AS ( + INSERT INTO template_usage_stats AS tus ( + start_time, + end_time, + template_id, + user_id, + usage_mins, + median_latency_ms, + app_usage_mins + ) ( + SELECT + stats.start_time, + stats.end_time, + stats.template_id, + stats.user_id, + stats.usage_mins, + latencies.median_latency_ms, + stats.app_usage_mins + FROM + stats + LEFT JOIN + latencies + ON + -- The latencies group-by ensures there at most one row. + latencies.start_time = stats.start_time + AND latencies.template_id = stats.template_id + AND latencies.user_id = stats.user_id + ) + ON CONFLICT + (start_time, template_id, user_id) + DO UPDATE + SET + usage_mins = EXCLUDED.usage_mins, + median_latency_ms = EXCLUDED.median_latency_ms, + app_usage_mins = EXCLUDED.app_usage_mins + WHERE + (tus.*) IS DISTINCT FROM (EXCLUDED.*) + RETURNING + tus.start_time + ), + -- The child writes below run in this same statement, so the foreign key + -- triggers fire once it completes and see the main rows the upsert above + -- added. The delete and the insert of each pair never touch the same row: + -- the delete matches names the recomputed bucket no longer has, the insert + -- only the names it does have. + -- + -- The deletes test membership with NOT IN rather than NOT EXISTS on purpose. + -- The planner has no statistics for the recomputed CTE and estimates it at + -- a few rows, which turns NOT EXISTS into a nested loop that rescans the + -- CTE per candidate row, measured at 20 seconds per rollup. NOT IN is + -- planned as a hashed subplan built once, whatever the estimate. Every + -- column in the subquery is non-null, so the two forms delete the same + -- rows; the IS NOT NULL guard keeps that true if the CTE ever changes. + delete_families AS ( + DELETE FROM + template_usage_stats_session_families AS families + USING + stats + WHERE + families.start_time = stats.start_time + AND families.template_id = stats.template_id + AND families.user_id = stats.user_id + AND (families.start_time, families.template_id, families.user_id, families.family) NOT IN ( + SELECT time_bucket, template_id, user_id, name + FROM agent_stats_session_minutes + WHERE family_group = 1 AND name IS NOT NULL + ) + ), + upsert_families AS ( + INSERT INTO template_usage_stats_session_families AS families ( + start_time, + template_id, + user_id, + family, + usage_mins + ) ( + SELECT + time_bucket, + template_id, + user_id, + name, + usage_mins + FROM + agent_stats_session_minutes + WHERE + family_group = 1 + ) + ON CONFLICT + (start_time, template_id, user_id, family) + DO UPDATE + SET + usage_mins = EXCLUDED.usage_mins + WHERE + families.usage_mins IS DISTINCT FROM EXCLUDED.usage_mins + ), + delete_apps AS ( + DELETE FROM + template_usage_stats_session_apps AS apps + USING + stats + WHERE + apps.start_time = stats.start_time + AND apps.template_id = stats.template_id + AND apps.user_id = stats.user_id + AND (apps.start_time, apps.template_id, apps.user_id, apps.app_name) NOT IN ( + SELECT time_bucket, template_id, user_id, name + FROM agent_stats_session_minutes + WHERE family_group = 0 AND name IS NOT NULL + ) ) -INSERT INTO template_usage_stats AS tus ( +INSERT INTO template_usage_stats_session_apps AS apps ( start_time, - end_time, template_id, user_id, - usage_mins, - median_latency_ms, - ssh_mins, - sftp_mins, - reconnecting_pty_mins, - vscode_mins, - jetbrains_mins, - app_usage_mins + app_name, + usage_mins ) ( SELECT - stats.start_time, - stats.end_time, - stats.template_id, - stats.user_id, - stats.usage_mins, - latencies.median_latency_ms, - stats.ssh_mins, - stats.sftp_mins, - stats.reconnecting_pty_mins, - stats.vscode_mins, - stats.jetbrains_mins, - stats.app_usage_mins + time_bucket, + template_id, + user_id, + name, + usage_mins FROM - stats - LEFT JOIN - latencies - ON - -- The latencies group-by ensures there at most one row. - latencies.start_time = stats.start_time - AND latencies.template_id = stats.template_id - AND latencies.user_id = stats.user_id + agent_stats_session_minutes + WHERE + family_group = 0 ) ON CONFLICT - (start_time, template_id, user_id) + (start_time, template_id, user_id, app_name) DO UPDATE SET - usage_mins = EXCLUDED.usage_mins, - median_latency_ms = EXCLUDED.median_latency_ms, - ssh_mins = EXCLUDED.ssh_mins, - sftp_mins = EXCLUDED.sftp_mins, - reconnecting_pty_mins = EXCLUDED.reconnecting_pty_mins, - vscode_mins = EXCLUDED.vscode_mins, - jetbrains_mins = EXCLUDED.jetbrains_mins, - app_usage_mins = EXCLUDED.app_usage_mins + usage_mins = EXCLUDED.usage_mins WHERE - (tus.*) IS DISTINCT FROM (EXCLUDED.*) + apps.usage_mins IS DISTINCT FROM EXCLUDED.usage_mins ` // This query aggregates the workspace_agent_stats and workspace_app_stats data // into a single table for efficient storage and querying. Half-hour buckets are // used to store the data, and the minutes are summed for each user and template // combination. The result is stored in the template_usage_stats table. +// +// Session usage is stored per app name and per app family in the child tables, +// so the main row carries no session columns at all. Every recomputed bucket +// rewrites its own child rows: names that disappeared are deleted, the rest +// are upserted. The keys come from the computed set rather than from the main +// upsert, because the no-op guard below suppresses main rows whose columns did +// not change while their session usage still has to be corrected. func (q *sqlQuerier) UpsertTemplateUsageStats(ctx context.Context, appFamilies json.RawMessage) error { _, err := q.db.ExecContext(ctx, upsertTemplateUsageStats, appFamilies) return err diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 92c140bdf23..8c81a8c6fda 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -86,134 +86,251 @@ ORDER BY -- workspaces in a given timeframe. The template IDs, active users, and -- usage_seconds all reflect any usage in the template, including apps. -- +-- Session usage comes from the family child table exactly as the rollup +-- recorded it, so a family the rollup learns about later is reported without a +-- change here. +-- -- When combining data from multiple templates, we must make a guess at -- how the user behaved for the 30 minute interval. In this case we make -- the assumption that if the user used two workspaces for 15 minutes, -- they did so sequentially, thus we sum the usage up to a maximum of -- 30 minutes with LEAST(SUM(n), 30). WITH - insights AS ( + base AS MATERIALIZED ( + -- One pass over the main table answers three questions: how many + -- templates each user touched in a half hour, that user's capped + -- minutes, and the list of templates in the window. GROUPING marks + -- which of the two sets a row belongs to. SELECT + GROUPING(template_id) AS is_user_row, + start_time, user_id, - -- See motivation in GetTemplateInsights for LEAST(SUM(n), 30). - LEAST(SUM(usage_mins), 30) AS usage_mins, - LEAST(SUM(ssh_mins), 30) AS ssh_mins, - LEAST(SUM(sftp_mins), 30) AS sftp_mins, - LEAST(SUM(reconnecting_pty_mins), 30) AS reconnecting_pty_mins, - LEAST(SUM(vscode_mins), 30) AS vscode_mins, - LEAST(SUM(jetbrains_mins), 30) AS jetbrains_mins + template_id, + COUNT(*) AS templates, + LEAST(SUM(usage_mins), 30) AS usage_mins FROM template_usage_stats WHERE start_time >= @start_time::timestamptz AND end_time <= @end_time::timestamptz AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END + GROUP BY GROUPING SETS ((start_time, user_id), (template_id)) + ), + users AS ( + SELECT + start_time, + user_id, + templates, + usage_mins + FROM + base + WHERE + is_user_row = 1 + ), + multi AS MATERIALIZED ( + -- Only a user who used more than one template in the same half hour + -- can exceed the 30 minute cap, and there are usually none. + SELECT + start_time, + user_id + FROM + users + WHERE + templates > 1 + ), + single_family_usage AS ( + -- Everything but the multi-template buckets, which cannot exceed the + -- cap, so they need no per-user grouping. This also collects the + -- template list per family, for which the cap is irrelevant. + SELECT + sessions.family, + sessions.template_id, + SUM(LEAST(sessions.usage_mins, 30)) FILTER ( + WHERE (sessions.start_time, sessions.user_id) NOT IN (SELECT start_time, user_id FROM multi) + ) AS usage_mins + FROM + template_usage_stats_session_families AS sessions + WHERE + sessions.start_time >= @start_time::timestamptz + AND sessions.start_time + '30 minutes'::interval <= @end_time::timestamptz + AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END + AND sessions.usage_mins > 0 GROUP BY - start_time, user_id + sessions.family, sessions.template_id ), - templates AS ( + multi_family_usage AS ( + -- See motivation in GetTemplateInsights for LEAST(SUM(n), 30). SELECT - array_agg(DISTINCT template_id) AS template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE ssh_mins > 0) AS ssh_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE sftp_mins > 0) AS sftp_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE reconnecting_pty_mins > 0) AS reconnecting_pty_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE vscode_mins > 0) AS vscode_template_ids, - array_agg(DISTINCT template_id) FILTER (WHERE jetbrains_mins > 0) AS jetbrains_template_ids + sessions.family, + LEAST(SUM(sessions.usage_mins), 30) AS usage_mins FROM - template_usage_stats + template_usage_stats_session_families AS sessions WHERE - start_time >= @start_time::timestamptz - AND end_time <= @end_time::timestamptz - AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END + sessions.start_time >= @start_time::timestamptz + AND sessions.start_time + '30 minutes'::interval <= @end_time::timestamptz + AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END + AND sessions.usage_mins > 0 + AND EXISTS ( + SELECT 1 + FROM multi + WHERE multi.start_time = sessions.start_time + AND multi.user_id = sessions.user_id + ) + GROUP BY + sessions.start_time, sessions.user_id, sessions.family + ), + family_usage AS ( + SELECT + family, + (SUM(usage_mins) * 60)::bigint AS usage_seconds + FROM ( + SELECT + family, + COALESCE(SUM(usage_mins), 0) AS usage_mins + FROM + single_family_usage + GROUP BY + family + + UNION ALL + + SELECT + family, + SUM(usage_mins) AS usage_mins + FROM + multi_family_usage + GROUP BY + family + ) AS parts + GROUP BY + family + ), + family_templates AS ( + SELECT + family, + array_agg(DISTINCT template_id) AS template_ids + FROM + single_family_usage + GROUP BY + family ) SELECT - COALESCE((SELECT template_ids FROM templates), '{}')::uuid[] AS template_ids, -- Includes app usage. - COALESCE((SELECT ssh_template_ids FROM templates), '{}')::uuid[] AS ssh_template_ids, - COALESCE((SELECT sftp_template_ids FROM templates), '{}')::uuid[] AS sftp_template_ids, - COALESCE((SELECT reconnecting_pty_template_ids FROM templates), '{}')::uuid[] AS reconnecting_pty_template_ids, - COALESCE((SELECT vscode_template_ids FROM templates), '{}')::uuid[] AS vscode_template_ids, - COALESCE((SELECT jetbrains_template_ids FROM templates), '{}')::uuid[] AS jetbrains_template_ids, + COALESCE((SELECT array_agg(DISTINCT template_id) FROM base WHERE is_user_row = 0), '{}')::uuid[] AS template_ids, -- Includes app usage. COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage. COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage. - COALESCE(SUM(ssh_mins) * 60, 0)::bigint AS usage_ssh_seconds, - COALESCE(SUM(sftp_mins) * 60, 0)::bigint AS usage_sftp_seconds, - COALESCE(SUM(reconnecting_pty_mins) * 60, 0)::bigint AS usage_reconnecting_pty_seconds, - COALESCE(SUM(vscode_mins) * 60, 0)::bigint AS usage_vscode_seconds, - COALESCE(SUM(jetbrains_mins) * 60, 0)::bigint AS usage_jetbrains_seconds + -- Family name to usage seconds, and family name to the templates that saw + -- the family. + COALESCE((SELECT jsonb_object_agg(family, usage_seconds) FROM family_usage), '{}'::jsonb)::jsonb AS session_family_usage_seconds, + COALESCE((SELECT jsonb_object_agg(family, template_ids) FROM family_templates), '{}'::jsonb)::jsonb AS session_family_template_ids FROM - insights; + users; -- name: GetTemplateInsightsByTemplate :many -- GetTemplateInsightsByTemplate is used for Prometheus metrics. Keep -- in sync with GetTemplateInsights and UpsertTemplateUsageStats. WITH - -- app_families maps each attributed family to its app names, so the - -- probes below stay one expression per family: adding a family needs a - -- new list in fams plus one probe here, because sqlc output columns are - -- static. fams turns the jsonb parameter into arrays once for the whole - -- query. These probes only ask whether any app of a family is present, - -- so the jsonb key-existence operator beats decomposing session_counts - -- per row (measured ~2.4x faster on a 1M row scan). - fams AS ( - SELECT - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'ssh')) AS ssh, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'reconnecting_pty')) AS reconnecting_pty, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'vscode')) AS vscode, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'jetbrains')) AS jetbrains + expanded AS ( + -- Each row's app names are expanded once and mapped to their family. + -- app_families maps app name to family name; an app the registry does + -- not know is attributed to 'unknown' rather than dropped. + SELECT + was.template_id, + was.user_id, + date_trunc('minute', was.created_at) AS minute, + COALESCE(@app_families::jsonb ->> app_name, 'unknown') AS family + FROM + workspace_agent_stats AS was, jsonb_object_keys(was.session_counts) AS app_name + WHERE + was.created_at >= @start_time::timestamptz + AND was.created_at < @end_time::timestamptz + AND was.session_counts <> '{}'::jsonb ), - -- Deduplicate activity by template, user, and minute. - minute_activity AS ( + minute_family AS ( + -- Deduplicate activity by template, user, minute, and family, so a + -- minute with two apps of one family counts once for that family. SELECT template_id, user_id, - date_trunc('minute', created_at) AS minute, - BOOL_OR(session_counts ?| fams.ssh) AS ssh, - BOOL_OR(session_counts ?| fams.reconnecting_pty) AS reconnecting_pty, - BOOL_OR(session_counts ?| fams.vscode) AS vscode, - BOOL_OR(session_counts ?| fams.jetbrains) AS jetbrains, - BOOL_OR(connection_count > 0) AS has_connection - FROM - workspace_agent_stats, fams + minute, + family + FROM + expanded + GROUP BY + template_id, user_id, minute, family + ), + connected AS ( + -- NOTE(mafredri): The agent stats are currently very unreliable, and + -- sometimes the connections are missing, even during active sessions. + -- Since we can't fully rely on this, we check for "any connection + -- within this bucket". A better solution here would be preferable. + SELECT + template_id, + user_id + FROM + workspace_agent_stats WHERE created_at >= @start_time::timestamptz AND created_at < @end_time::timestamptz AND session_counts <> '{}'::jsonb GROUP BY - template_id, user_id, minute + template_id, user_id + HAVING + BOOL_OR(connection_count > 0) ), insights AS ( + SELECT + mf.template_id, + mf.user_id, + mf.family, + COUNT(*) AS usage_mins + FROM + minute_family AS mf + JOIN + connected AS c + ON + c.template_id = mf.template_id + AND c.user_id = mf.user_id + GROUP BY + mf.template_id, mf.user_id, mf.family + ), + family_usage AS ( SELECT template_id, - user_id, - COUNT(*) FILTER (WHERE ssh) AS ssh_mins, - COUNT(*) FILTER (WHERE reconnecting_pty) AS reconnecting_pty_mins, - COUNT(*) FILTER (WHERE vscode) AS vscode_mins, - COUNT(*) FILTER (WHERE jetbrains) AS jetbrains_mins, - -- NOTE(mafredri): The agent stats are currently very unreliable, and - -- sometimes the connections are missing, even during active sessions. - -- Since we can't fully rely on this, we check for "any connection - -- within this bucket". A better solution here would be preferable. - BOOL_OR(has_connection) AS has_connection + jsonb_object_agg(family, usage_seconds) AS session_family_usage_seconds + FROM ( + SELECT + template_id, + family, + (SUM(usage_mins) * 60)::bigint AS usage_seconds + FROM + insights + GROUP BY + template_id, family + ) AS family_seconds + GROUP BY + template_id + ), + active_users AS ( + SELECT + template_id, + COUNT(DISTINCT user_id)::bigint AS active_users FROM - minute_activity + insights GROUP BY - template_id, user_id + template_id ) SELECT - template_id, - COUNT(DISTINCT user_id)::bigint AS active_users, - (SUM(vscode_mins) * 60)::bigint AS usage_vscode_seconds, - (SUM(jetbrains_mins) * 60)::bigint AS usage_jetbrains_seconds, - (SUM(reconnecting_pty_mins) * 60)::bigint AS usage_reconnecting_pty_seconds, - (SUM(ssh_mins) * 60)::bigint AS usage_ssh_seconds + au.template_id, + au.active_users, + COALESCE(fu.session_family_usage_seconds, '{}'::jsonb)::jsonb AS session_family_usage_seconds FROM - insights -WHERE - has_connection -GROUP BY - template_id; + active_users AS au +LEFT JOIN + family_usage AS fu +ON + fu.template_id = au.template_id; -- name: GetTemplateAppInsights :many -- GetTemplateAppInsights returns the aggregate usage of each app in a given @@ -497,21 +614,14 @@ WHERE -- into a single table for efficient storage and querying. Half-hour buckets are -- used to store the data, and the minutes are summed for each user and template -- combination. The result is stored in the template_usage_stats table. +-- +-- Session usage is stored per app name and per app family in the child tables, +-- so the main row carries no session columns at all. Every recomputed bucket +-- rewrites its own child rows: names that disappeared are deleted, the rest +-- are upserted. The keys come from the computed set rather than from the main +-- upsert, because the no-op guard below suppresses main rows whose columns did +-- not change while their session usage still has to be corrected. WITH - -- app_families maps each attributed family to its app names, so the - -- probes below stay one expression per family: adding a family needs a - -- new list in fams plus one probe here, because sqlc output columns are - -- static. fams turns the jsonb parameter into arrays once for the whole - -- query. These probes only ask whether any app of a family is present, - -- so the jsonb key-existence operator beats decomposing session_counts - -- per row (measured ~2.4x faster on a 1M row scan). - fams AS ( - SELECT - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'ssh')) AS ssh, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'reconnecting_pty')) AS reconnecting_pty, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'vscode')) AS vscode, - ARRAY(SELECT jsonb_array_elements_text(@app_families::jsonb -> 'jetbrains')) AS jetbrains - ), latest_start AS ( SELECT -- Truncate to hour so that we always look at even ranges of data. @@ -581,35 +691,118 @@ WITH GROUP BY time_bucket, w.template_id, fas.user_id, fas.access_method, fas.slug_or_port ), - agent_stats_buckets AS ( + agent_stats_rows AS ( + -- One filtered pass over workspace_agent_stats feeds both the bucket + -- grouping and the per-app mask grouping below, instead of scanning the + -- table once for each. The minute bit is computed here so the mask + -- grouping never touches created_at again. SELECT - -- Truncate the minute to the nearest half hour, this is the bucket size - -- for the data. date_trunc('hour', created_at) + trunc(date_part('minute', created_at) / 30) * 30 * '1 minute'::interval AS time_bucket, template_id, user_id, - -- Store each unique minute bucket for later merge between datasets. - array_agg(DISTINCT date_trunc('minute', created_at)) AS minute_buckets, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.ssh THEN date_trunc('minute', created_at) ELSE NULL END) AS ssh_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.reconnecting_pty THEN date_trunc('minute', created_at) ELSE NULL END) AS reconnecting_pty_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.vscode THEN date_trunc('minute', created_at) ELSE NULL END) AS vscode_mins, - COUNT(DISTINCT CASE WHEN session_counts ?| fams.jetbrains THEN date_trunc('minute', created_at) ELSE NULL END) AS jetbrains_mins, - -- NOTE(mafredri): The agent stats are currently very unreliable, and - -- sometimes the connections are missing, even during active sessions. - -- Since we can't fully rely on this, we check for "any connection - -- during this half-hour". A better solution here would be preferable. - MAX(connection_count) > 0 AS has_connection + date_trunc('minute', created_at) AS minute_bucket, + (1 << (date_part('minute', created_at)::int % 30)) AS minute_bit, + connection_count, + session_counts FROM - workspace_agent_stats, fams + workspace_agent_stats WHERE -- created_at >= @start_time::timestamptz -- AND created_at < @end_time::timestamptz created_at >= (SELECT t FROM latest_start) AND created_at < NOW() AND session_counts <> '{}'::jsonb + ), + agent_stats_buckets AS ( + SELECT + time_bucket, + template_id, + user_id, + -- Store each unique minute bucket for later merge between datasets. + array_agg(DISTINCT minute_bucket) AS minute_buckets, + -- NOTE(mafredri): The agent stats are currently very unreliable, and + -- sometimes the connections are missing, even during active sessions. + -- Since we can't fully rely on this, we check for "any connection + -- during this half-hour". A better solution here would be preferable. + MAX(connection_count) > 0 AS has_connection + FROM + agent_stats_rows GROUP BY time_bucket, template_id, user_id ), + app_family_registry AS ( + -- The session count attribution registry, app name to family name. + SELECT + app, + family + FROM + jsonb_each_text(@app_families::jsonb) AS registry(app, family) + ), + agent_stats_app_minutes AS ( + -- One bit per minute of the half-hour bucket, per app name, instead of + -- a count: masks can be OR'd into a family below without counting a + -- minute twice when two apps of the same family were active in it. + -- Transient, only the minute counts derived from them are stored. + SELECT + time_bucket, + template_id, + user_id, + app_name, + bit_or(minute_bit) AS minute_mask + FROM + agent_stats_rows, jsonb_object_keys(session_counts) AS app_name + GROUP BY + time_bucket, template_id, user_id, app_name + ), + agent_stats_session_masks AS ( + -- One pass over the per-app masks emits both groupings: the app rows + -- keep each mask as it is, the family rows OR the masks of every app + -- in the family. An app name the registry does not know is attributed + -- to 'unknown' rather than dropped, so a newly reported app still + -- lands somewhere. GROUPING() marks which set a row came from, because + -- the app column is null in the family rows. + SELECT + time_bucket, + template_id, + user_id, + agent_stats_app_minutes.app_name, + COALESCE(app_family_registry.family, 'unknown') AS family, + GROUPING(agent_stats_app_minutes.app_name) AS family_group, + bit_or(minute_mask) AS minute_mask + FROM + agent_stats_app_minutes + LEFT JOIN + app_family_registry + ON + app_family_registry.app = agent_stats_app_minutes.app_name + GROUP BY GROUPING SETS ( + (time_bucket, template_id, user_id, agent_stats_app_minutes.app_name), + (time_bucket, template_id, user_id, COALESCE(app_family_registry.family, 'unknown')) + ) + ), + agent_stats_session_minutes AS ( + -- The minutes each name was active, counted from the bits set in its + -- mask. Postgres 13 has no bit_count, so the set bits are counted by + -- stripping the zeros out of the mask's text form. + SELECT + masks.time_bucket, + masks.template_id, + masks.user_id, + masks.family_group, + CASE WHEN masks.family_group = 0 THEN masks.app_name ELSE masks.family END AS name, + length(replace(masks.minute_mask::bit(30)::text, '0', ''))::smallint AS usage_mins + FROM + agent_stats_session_masks AS masks + JOIN + agent_stats_buckets AS buckets + ON + buckets.time_bucket = masks.time_bucket + AND buckets.template_id = masks.template_id + AND buckets.user_id = masks.user_id + -- The same gate the union below applies to agent stats, so a + -- bucket that only has app stats records no session usage. + AND buckets.has_connection + ), stats AS ( SELECT stats.time_bucket AS start_time, @@ -619,11 +812,6 @@ WITH -- Sum/distinct to handle zero/duplicate values due union and to unnest. COUNT(DISTINCT minute_bucket) AS usage_mins, array_agg(DISTINCT minute_bucket) AS minute_buckets, - SUM(DISTINCT stats.ssh_mins) AS ssh_mins, - SUM(DISTINCT stats.sftp_mins) AS sftp_mins, - SUM(DISTINCT stats.reconnecting_pty_mins) AS reconnecting_pty_mins, - SUM(DISTINCT stats.vscode_mins) AS vscode_mins, - SUM(DISTINCT stats.jetbrains_mins) AS jetbrains_mins, -- This is what we unnested, re-nest as json. jsonb_object_agg(stats.app_name, stats.app_minutes) FILTER (WHERE stats.app_name IS NOT NULL) AS app_usage_mins FROM ( @@ -631,11 +819,6 @@ WITH time_bucket, template_id, user_id, - 0 AS ssh_mins, - 0 AS sftp_mins, - 0 AS reconnecting_pty_mins, - 0 AS vscode_mins, - 0 AS jetbrains_mins, app_name, app_minutes, minute_buckets @@ -648,12 +831,6 @@ WITH time_bucket, template_id, user_id, - ssh_mins, - -- TODO(mafredri): Enable when we have the column. - 0 AS sftp_mins, - reconnecting_pty_mins, - vscode_mins, - jetbrains_mins, NULL AS app_name, NULL AS app_minutes, minute_buckets @@ -700,59 +877,143 @@ WITH AND was.connection_median_latency_ms > 0 GROUP BY mb.start_time, mb.template_id, mb.user_id + ), + upsert_stats AS ( + INSERT INTO template_usage_stats AS tus ( + start_time, + end_time, + template_id, + user_id, + usage_mins, + median_latency_ms, + app_usage_mins + ) ( + SELECT + stats.start_time, + stats.end_time, + stats.template_id, + stats.user_id, + stats.usage_mins, + latencies.median_latency_ms, + stats.app_usage_mins + FROM + stats + LEFT JOIN + latencies + ON + -- The latencies group-by ensures there at most one row. + latencies.start_time = stats.start_time + AND latencies.template_id = stats.template_id + AND latencies.user_id = stats.user_id + ) + ON CONFLICT + (start_time, template_id, user_id) + DO UPDATE + SET + usage_mins = EXCLUDED.usage_mins, + median_latency_ms = EXCLUDED.median_latency_ms, + app_usage_mins = EXCLUDED.app_usage_mins + WHERE + (tus.*) IS DISTINCT FROM (EXCLUDED.*) + RETURNING + tus.start_time + ), + -- The child writes below run in this same statement, so the foreign key + -- triggers fire once it completes and see the main rows the upsert above + -- added. The delete and the insert of each pair never touch the same row: + -- the delete matches names the recomputed bucket no longer has, the insert + -- only the names it does have. + -- + -- The deletes test membership with NOT IN rather than NOT EXISTS on purpose. + -- The planner has no statistics for the recomputed CTE and estimates it at + -- a few rows, which turns NOT EXISTS into a nested loop that rescans the + -- CTE per candidate row, measured at 20 seconds per rollup. NOT IN is + -- planned as a hashed subplan built once, whatever the estimate. Every + -- column in the subquery is non-null, so the two forms delete the same + -- rows; the IS NOT NULL guard keeps that true if the CTE ever changes. + delete_families AS ( + DELETE FROM + template_usage_stats_session_families AS families + USING + stats + WHERE + families.start_time = stats.start_time + AND families.template_id = stats.template_id + AND families.user_id = stats.user_id + AND (families.start_time, families.template_id, families.user_id, families.family) NOT IN ( + SELECT time_bucket, template_id, user_id, name + FROM agent_stats_session_minutes + WHERE family_group = 1 AND name IS NOT NULL + ) + ), + upsert_families AS ( + INSERT INTO template_usage_stats_session_families AS families ( + start_time, + template_id, + user_id, + family, + usage_mins + ) ( + SELECT + time_bucket, + template_id, + user_id, + name, + usage_mins + FROM + agent_stats_session_minutes + WHERE + family_group = 1 + ) + ON CONFLICT + (start_time, template_id, user_id, family) + DO UPDATE + SET + usage_mins = EXCLUDED.usage_mins + WHERE + families.usage_mins IS DISTINCT FROM EXCLUDED.usage_mins + ), + delete_apps AS ( + DELETE FROM + template_usage_stats_session_apps AS apps + USING + stats + WHERE + apps.start_time = stats.start_time + AND apps.template_id = stats.template_id + AND apps.user_id = stats.user_id + AND (apps.start_time, apps.template_id, apps.user_id, apps.app_name) NOT IN ( + SELECT time_bucket, template_id, user_id, name + FROM agent_stats_session_minutes + WHERE family_group = 0 AND name IS NOT NULL + ) ) -INSERT INTO template_usage_stats AS tus ( +INSERT INTO template_usage_stats_session_apps AS apps ( start_time, - end_time, template_id, user_id, - usage_mins, - median_latency_ms, - ssh_mins, - sftp_mins, - reconnecting_pty_mins, - vscode_mins, - jetbrains_mins, - app_usage_mins + app_name, + usage_mins ) ( SELECT - stats.start_time, - stats.end_time, - stats.template_id, - stats.user_id, - stats.usage_mins, - latencies.median_latency_ms, - stats.ssh_mins, - stats.sftp_mins, - stats.reconnecting_pty_mins, - stats.vscode_mins, - stats.jetbrains_mins, - stats.app_usage_mins + time_bucket, + template_id, + user_id, + name, + usage_mins FROM - stats - LEFT JOIN - latencies - ON - -- The latencies group-by ensures there at most one row. - latencies.start_time = stats.start_time - AND latencies.template_id = stats.template_id - AND latencies.user_id = stats.user_id + agent_stats_session_minutes + WHERE + family_group = 0 ) ON CONFLICT - (start_time, template_id, user_id) + (start_time, template_id, user_id, app_name) DO UPDATE SET - usage_mins = EXCLUDED.usage_mins, - median_latency_ms = EXCLUDED.median_latency_ms, - ssh_mins = EXCLUDED.ssh_mins, - sftp_mins = EXCLUDED.sftp_mins, - reconnecting_pty_mins = EXCLUDED.reconnecting_pty_mins, - vscode_mins = EXCLUDED.vscode_mins, - jetbrains_mins = EXCLUDED.jetbrains_mins, - app_usage_mins = EXCLUDED.app_usage_mins + usage_mins = EXCLUDED.usage_mins WHERE - (tus.*) IS DISTINCT FROM (EXCLUDED.*); + apps.usage_mins IS DISTINCT FROM EXCLUDED.usage_mins; -- name: GetTemplateParameterInsights :many -- GetTemplateParameterInsights does for each template in a given timeframe, diff --git a/coderd/database/session_usage_history_test.go b/coderd/database/session_usage_history_test.go new file mode 100644 index 00000000000..4354c6a81b2 --- /dev/null +++ b/coderd/database/session_usage_history_test.go @@ -0,0 +1,76 @@ +package database_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/database/migrations" +) + +func TestSessionUsageHistoryCapsAndNamespaces(t *testing.T) { + t.Parallel() + sqlDB := testSQLDB(t) + require.NoError(t, migrations.Up(sqlDB)) + db := database.New(sqlDB) + ctx := context.Background() + start := dbtime.Now().Add(-2 * time.Hour).Truncate(30 * time.Minute) + user, template1, template2, appTemplate := uuid.New(), uuid.New(), uuid.New(), uuid.New() + // The same user in two templates in one half hour, so the family minutes + // have to be capped at 30 across templates. There is no sqlc query for the + // child tables, so the history is seeded directly. + for _, template := range []uuid.UUID{template1, template2} { + _, err := sqlDB.ExecContext(ctx, `INSERT INTO template_usage_stats(start_time,end_time,user_id,template_id,usage_mins) VALUES($1,$2,$3,$4,20)`, + start, start.Add(30*time.Minute), user, template) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, `INSERT INTO template_usage_stats_session_families(start_time,template_id,user_id,family,usage_mins) VALUES($1,$2,$3,'new_family',20),($1,$2,$3,'ssh',20),($1,$2,$3,'sftp',2)`, + start, template, user) + require.NoError(t, err) + } + // A bucket produced by app stats alone: it has no session usage, so it has + // no child rows. + _, err := sqlDB.ExecContext(ctx, `INSERT INTO template_usage_stats(start_time,end_time,user_id,template_id,usage_mins,app_usage_mins) VALUES($1,$2,$3,$4,5,'{"ssh":5}')`, + start, start.Add(30*time.Minute), uuid.New(), appTemplate) + require.NoError(t, err) + usage, err := db.GetTemplateInsights(ctx, database.GetTemplateInsightsParams{StartTime: start, EndTime: start.Add(30 * time.Minute)}) + require.NoError(t, err) + require.EqualValues(t, 2, usage.ActiveUsers) + require.EqualValues(t, 35*60, usage.UsageTotalSeconds) + require.ElementsMatch(t, []uuid.UUID{template1, template2, appTemplate}, usage.TemplateIDs) + require.JSONEq(t, `{"new_family":1800,"ssh":1800,"sftp":240}`, string(usage.SessionFamilyUsageSeconds)) + var ids map[string][]uuid.UUID + require.NoError(t, json.Unmarshal(usage.SessionFamilyTemplateIds, &ids)) + require.ElementsMatch(t, []uuid.UUID{template1, template2}, ids["new_family"]) + require.ElementsMatch(t, []uuid.UUID{template1, template2}, ids["ssh"]) + onlyApp, err := db.GetTemplateInsights(ctx, database.GetTemplateInsightsParams{StartTime: start, EndTime: start.Add(30 * time.Minute), TemplateIDs: []uuid.UUID{appTemplate}}) + require.NoError(t, err) + require.EqualValues(t, 1, onlyApp.ActiveUsers) + require.EqualValues(t, 300, onlyApp.UsageTotalSeconds) + require.JSONEq(t, `{}`, string(onlyApp.SessionFamilyUsageSeconds)) + require.JSONEq(t, `{}`, string(onlyApp.SessionFamilyTemplateIds)) + // Filtering to one of the two templates leaves a single-template user, so + // the capped path is not taken and the minutes are that template's alone. + onlyOne, err := db.GetTemplateInsights(ctx, database.GetTemplateInsightsParams{StartTime: start, EndTime: start.Add(30 * time.Minute), TemplateIDs: []uuid.UUID{template1}}) + require.NoError(t, err) + require.JSONEq(t, `{"new_family":1200,"ssh":1200,"sftp":120}`, string(onlyOne.SessionFamilyUsageSeconds)) +} + +func TestSessionUsageEmptyHistory(t *testing.T) { + t.Parallel() + db, _ := dbtestutil.NewDB(t) + start := dbtime.Now().Truncate(30 * time.Minute) + row, err := db.GetTemplateInsights(context.Background(), database.GetTemplateInsightsParams{StartTime: start, EndTime: start.Add(time.Hour)}) + require.NoError(t, err) + require.Empty(t, row.TemplateIDs) + require.Zero(t, row.ActiveUsers) + require.Zero(t, row.UsageTotalSeconds) + require.JSONEq(t, `{}`, string(row.SessionFamilyUsageSeconds)) + require.JSONEq(t, `{}`, string(row.SessionFamilyTemplateIds)) +} diff --git a/coderd/database/session_usage_test.go b/coderd/database/session_usage_test.go new file mode 100644 index 00000000000..486c64fbfae --- /dev/null +++ b/coderd/database/session_usage_test.go @@ -0,0 +1,141 @@ +package database_test + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/codersdk" +) + +// sessionUsageMins reads a session usage child table for one bucket. There is +// no sqlc query for the child tables, so the tests read them directly. +func sessionUsageMins(ctx context.Context, t *testing.T, sqlDB *sql.DB, table, nameColumn string, startTime time.Time, userID, templateID uuid.UUID) map[string]int64 { + t.Helper() + + //nolint:gosec // Table and column names are constants in this test. + rows, err := sqlDB.QueryContext(ctx, "SELECT "+nameColumn+", usage_mins FROM "+table+ + " WHERE start_time = $1 AND user_id = $2 AND template_id = $3", startTime, userID, templateID) + require.NoError(t, err) + defer rows.Close() + + got := map[string]int64{} + for rows.Next() { + var name string + var usageMins int64 + require.NoError(t, rows.Scan(&name, &usageMins)) + got[name] = usageMins + } + require.NoError(t, rows.Err()) + return got +} + +func TestSessionUsageMapNull(t *testing.T) { + t.Parallel() + m := database.StringMapOfInt{"cursor": 1} + require.NoError(t, m.Scan(nil)) + require.Nil(t, m) + require.NoError(t, m.Scan([]byte(`{}`))) + require.NotNil(t, m) + require.Empty(t, m) +} + +func TestSessionUsageRollupOverlapAndRegistry(t *testing.T) { + t.Parallel() + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := context.Background() + start := dbtime.Now().Add(-3 * time.Hour).Truncate(30 * time.Minute) + user, template := uuid.New(), uuid.New() + insert := func(offset time.Duration, userID, templateID uuid.UUID, connected int64, counts map[string]int64) { + dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ + CreatedAt: start.Add(offset), UserID: userID, TemplateID: templateID, AgentID: uuid.New(), + ConnectionCount: connected, SessionCounts: dbgen.SessionCounts(t, counts), + }) + } + insert(0, user, template, 1, map[string]int64{"cursor": 2, "vscode": 1, "new_app": 1}) + insert(30*time.Second, user, template, 0, map[string]int64{"cursor": 1, "new_app": 1}) + insert(time.Minute, user, template, 0, map[string]int64{"cursor": 1}) + insert(29*time.Minute, user, template, 0, map[string]int64{"vscode": 1}) + insert(30*time.Minute, user, template, 1, map[string]int64{"cursor": 1}) + insert(0, uuid.New(), template, 1, map[string]int64{"vscode": 1}) + insert(0, user, uuid.New(), 1, map[string]int64{"cursor": 1}) + disconnected := uuid.New() + insert(0, user, disconnected, 0, map[string]int64{"cursor": 1}) + empty := uuid.New() + insert(0, user, empty, 1, map[string]int64{}) + + var registry map[string]string + require.NoError(t, json.Unmarshal(codersdk.SessionCountAppFamiliesJSON(), ®istry)) + registry["new_app"] = "new_family" + mapping, err := json.Marshal(registry) + require.NoError(t, err) + require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) + params := database.GetTemplateUsageStatsParams{StartTime: start, EndTime: start.Add(time.Hour)} + rows, err := db.GetTemplateUsageStats(ctx, params) + require.NoError(t, err) + require.Len(t, rows, 4) + found := false + for _, row := range rows { + require.NotEqual(t, disconnected, row.TemplateID) + require.NotEqual(t, empty, row.TemplateID) + if row.UserID == user && row.TemplateID == template && row.StartTime.Equal(start) { + found = true + require.EqualValues(t, 3, row.UsageMins) + } + } + require.True(t, found, "the overlapping app bucket must be present") + // Overlapping apps of one family share their minutes in the family table + // but keep their own minutes in the app table. + require.Equal(t, map[string]int64{"cursor": 2, "vscode": 2, "new_app": 1}, + sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template)) + require.Equal(t, map[string]int64{"vscode": 3, "new_family": 1}, + sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template)) + + // The existing watermark deliberately recomputes recent buckets. + require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) + repeated, err := db.GetTemplateUsageStats(ctx, params) + require.NoError(t, err) + require.ElementsMatch(t, rows, repeated) + require.Equal(t, map[string]int64{"vscode": 3, "new_family": 1}, + sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template)) + + // Renaming a family in the registry moves the minutes: the recomputed + // bucket drops the family row it no longer has and keeps its app rows, + // even though the main row itself does not change. + registry["new_app"] = "renamed_family" + mapping, err = json.Marshal(registry) + require.NoError(t, err) + require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) + repeated, err = db.GetTemplateUsageStats(ctx, params) + require.NoError(t, err) + require.ElementsMatch(t, rows, repeated) + families := sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template) + require.EqualValues(t, 1, families["renamed_family"]) + require.NotContains(t, families, "new_family") + apps := sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template) + require.EqualValues(t, 1, apps["new_app"]) + + // A bucket that only has app stats records no session usage at all, and a + // deleted bucket takes its session usage with it. + require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, disconnected)) + _, err = sqlDB.ExecContext(ctx, `DELETE FROM template_usage_stats WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, start, user, template) + require.NoError(t, err) + require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template)) + require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template)) + + // Live insights deduplicate the overlapping apps without half-hour caps. + live, err := db.GetTemplateInsightsByTemplate(ctx, database.GetTemplateInsightsByTemplateParams{ + StartTime: start.Add(30 * time.Second), EndTime: start.Add(30 * time.Minute), AppFamilies: mapping, + }) + require.NoError(t, err) + require.Len(t, live, 0, "no connected report occurs in this partial request window") +} diff --git a/coderd/database/types.go b/coderd/database/types.go index 85c84d4cf10..c0a99a4895b 100644 --- a/coderd/database/types.go +++ b/coderd/database/types.go @@ -267,6 +267,7 @@ type StringMapOfInt map[string]int64 func (m *StringMapOfInt) Scan(src interface{}) error { if src == nil { + *m = nil return nil } switch src := src.(type) { diff --git a/coderd/database/unique_constraint.go b/coderd/database/unique_constraint.go index 180ba145508..f1af2c8c64d 100644 --- a/coderd/database/unique_constraint.go +++ b/coderd/database/unique_constraint.go @@ -92,6 +92,8 @@ const ( UniqueTelemetryItemsPkey UniqueConstraint = "telemetry_items_pkey" // ALTER TABLE ONLY telemetry_items ADD CONSTRAINT telemetry_items_pkey PRIMARY KEY (key); UniqueTelemetryLocksPkey UniqueConstraint = "telemetry_locks_pkey" // ALTER TABLE ONLY telemetry_locks ADD CONSTRAINT telemetry_locks_pkey PRIMARY KEY (event_type, period_ending_at); UniqueTemplateUsageStatsPkey UniqueConstraint = "template_usage_stats_pkey" // ALTER TABLE ONLY template_usage_stats ADD CONSTRAINT template_usage_stats_pkey PRIMARY KEY (start_time, template_id, user_id); + UniqueTemplateUsageStatsSessionAppsPkey UniqueConstraint = "template_usage_stats_session_apps_pkey" // ALTER TABLE ONLY template_usage_stats_session_apps ADD CONSTRAINT template_usage_stats_session_apps_pkey PRIMARY KEY (start_time, user_id, template_id, app_name); + UniqueTemplateUsageStatsSessionFamiliesPkey UniqueConstraint = "template_usage_stats_session_families_pkey" // ALTER TABLE ONLY template_usage_stats_session_families ADD CONSTRAINT template_usage_stats_session_families_pkey PRIMARY KEY (start_time, user_id, template_id, family); UniqueTemplateVersionParametersTemplateVersionIDNameKey UniqueConstraint = "template_version_parameters_template_version_id_name_key" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name); UniqueTemplateVersionPresetParametersPkey UniqueConstraint = "template_version_preset_parameters_pkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id); UniqueTemplateVersionPresetPrebuildSchedulesPkey UniqueConstraint = "template_version_preset_prebuild_schedules_pkey" // ALTER TABLE ONLY template_version_preset_prebuild_schedules ADD CONSTRAINT template_version_preset_prebuild_schedules_pkey PRIMARY KEY (id); diff --git a/coderd/insights.go b/coderd/insights.go index 4cdb8e81f97..ad24f007664 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -3,6 +3,7 @@ package coderd import ( "context" "database/sql" + "encoding/json" "fmt" "net/http" "slices" @@ -540,12 +541,21 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { } if slices.Contains(sections, codersdk.TemplateInsightsSectionReport) { + appsUsage, err := convertTemplateInsightsApps(usage, appUsage) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error converting template app insights.", + Detail: err.Error(), + }) + return + } + resp.Report = &codersdk.TemplateInsightsReport{ StartTime: startTime, EndTime: endTime, TemplateIDs: usage.TemplateIDs, ActiveUsers: usage.ActiveUsers, - AppsUsage: convertTemplateInsightsApps(usage, appUsage), + AppsUsage: appsUsage, ParametersUsage: parametersUsage, } } @@ -564,27 +574,96 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, resp) } +// sessionFamilySFTP is the session family the insights queries report SFTP +// usage under. No app name maps to it, so nothing in the app family registry +// produces it, but template_usage_stats carries historical sftp minutes under +// this key and the API has always exposed them as a builtin app. Families the +// query does not report simply have no usage. +const sessionFamilySFTP codersdk.AppFamilyName = "sftp" + +// templateInsightsSessionFamilies is the decoded form of the per-session-family +// JSONB columns on database.GetTemplateInsightsRow. +type templateInsightsSessionFamilies struct { + usageSecondsByFamily map[codersdk.AppFamilyName]int64 + templateIDsByFamily map[codersdk.AppFamilyName][]uuid.UUID +} + +// usageSeconds returns the usage seconds reported for a session family. A +// family the query did not report had no usage. +func (f templateInsightsSessionFamilies) usageSeconds(family codersdk.AppFamilyName) int64 { + return f.usageSecondsByFamily[family] +} + +// templateIDs returns the templates that reported usage for a session family. +// The result is never nil so that the API keeps serializing an empty list +// instead of null. +func (f templateInsightsSessionFamilies) templateIDs(family codersdk.AppFamilyName) []uuid.UUID { + if ids := f.templateIDsByFamily[family]; ids != nil { + return ids + } + return []uuid.UUID{} +} + +// decodeTemplateInsightsSessionFamilies decodes the JSONB session family +// columns of a template insights row. +func decodeTemplateInsightsSessionFamilies(usage database.GetTemplateInsightsRow) (templateInsightsSessionFamilies, error) { + usageSeconds, err := decodeSessionFamilyMap[int64](usage.SessionFamilyUsageSeconds) + if err != nil { + return templateInsightsSessionFamilies{}, xerrors.Errorf("decode session family usage seconds: %w", err) + } + templateIDs, err := decodeSessionFamilyMap[[]uuid.UUID](usage.SessionFamilyTemplateIds) + if err != nil { + return templateInsightsSessionFamilies{}, xerrors.Errorf("decode session family template ids: %w", err) + } + return templateInsightsSessionFamilies{ + usageSecondsByFamily: usageSeconds, + templateIDsByFamily: templateIDs, + }, nil +} + +// decodeSessionFamilyMap decodes a JSONB payload keyed by session family. An +// absent payload decodes to an empty map, but a malformed one is an error so +// that callers report the failure instead of reporting zero usage. +func decodeSessionFamilyMap[V any](raw json.RawMessage) (map[codersdk.AppFamilyName]V, error) { + if len(raw) == 0 { + return map[codersdk.AppFamilyName]V{}, nil + } + var decoded map[codersdk.AppFamilyName]V + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, xerrors.Errorf("unmarshal session family map: %w", err) + } + if decoded == nil { + return map[codersdk.AppFamilyName]V{}, nil + } + return decoded, nil +} + // convertTemplateInsightsApps builds the list of builtin apps and template apps // from the provided database rows, builtin apps are implicitly a part of all // templates. -func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) []codersdk.TemplateAppUsage { +func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) ([]codersdk.TemplateAppUsage, error) { + families, err := decodeTemplateInsightsSessionFamilies(usage) + if err != nil { + return nil, xerrors.Errorf("convert template insights apps: %w", err) + } + // Builtin apps. apps := []codersdk.TemplateAppUsage{ { - TemplateIDs: usage.VscodeTemplateIds, + TemplateIDs: families.templateIDs(codersdk.AppFamilyVSCode), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameVSCode, Slug: "vscode", Icon: "/icon/code.svg", - Seconds: usage.UsageVscodeSeconds, + Seconds: families.usageSeconds(codersdk.AppFamilyVSCode), }, { - TemplateIDs: usage.JetbrainsTemplateIds, + TemplateIDs: families.templateIDs(codersdk.AppFamilyJetBrains), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameJetBrains, Slug: "jetbrains", Icon: "/icon/intellij.svg", - Seconds: usage.UsageJetbrainsSeconds, + Seconds: families.usageSeconds(codersdk.AppFamilyJetBrains), }, // TODO(mafredri): We could take Web Terminal usage from appUsage since // that should be more accurate. The difference is that this reflects @@ -593,28 +672,28 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage // condition finding the corresponding app entry in appUsage is: // !app.IsApp && app.AccessMethod == "terminal" && app.SlugOrPort == "" { - TemplateIDs: usage.ReconnectingPtyTemplateIds, + TemplateIDs: families.templateIDs(codersdk.AppFamilyReconnectingPTY), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameWebTerminal, Slug: "reconnecting-pty", Icon: "/icon/terminal.svg", - Seconds: usage.UsageReconnectingPtySeconds, + Seconds: families.usageSeconds(codersdk.AppFamilyReconnectingPTY), }, { - TemplateIDs: usage.SshTemplateIds, + TemplateIDs: families.templateIDs(codersdk.AppFamilySSH), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameSSH, Slug: "ssh", Icon: "/icon/terminal.svg", - Seconds: usage.UsageSshSeconds, + Seconds: families.usageSeconds(codersdk.AppFamilySSH), }, { - TemplateIDs: usage.SftpTemplateIds, + TemplateIDs: families.templateIDs(sessionFamilySFTP), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameSFTP, Slug: "sftp", Icon: "/icon/terminal.svg", - Seconds: usage.UsageSftpSeconds, + Seconds: families.usageSeconds(sessionFamilySFTP), }, } @@ -646,7 +725,7 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage }) } - return apps + return apps, nil } // parseInsightsStartAndEndTime parses the start and end time query parameters diff --git a/coderd/insights_session_family_internal_test.go b/coderd/insights_session_family_internal_test.go new file mode 100644 index 00000000000..d50e80a9d09 --- /dev/null +++ b/coderd/insights_session_family_internal_test.go @@ -0,0 +1,79 @@ +package coderd + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/codersdk" +) + +func TestDecodeSessionFamilyMap(t *testing.T) { + t.Parallel() + + for name, raw := range map[string]json.RawMessage{ + "NotJSON": json.RawMessage(`{`), + "NotAnObject": json.RawMessage(`[1, 2]`), + "WrongValue": json.RawMessage(`{"vscode": "sixty"}`), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + // A malformed payload must not decode to zero usage, or an + // encoding bug would look like an idle deployment. + _, err := decodeSessionFamilyMap[int64](raw) + require.Error(t, err) + }) + } +} + +func TestConvertTemplateInsightsApps(t *testing.T) { + t.Parallel() + + t.Run("SFTPLegacy", func(t *testing.T) { + t.Parallel() + + sftpTemplateID := uuid.New() + apps, err := convertTemplateInsightsApps(database.GetTemplateInsightsRow{ + SessionFamilyUsageSeconds: json.RawMessage(`{"sftp": 300}`), + SessionFamilyTemplateIds: json.RawMessage(`{"sftp": ["` + sftpTemplateID.String() + `"]}`), + }, nil) + require.NoError(t, err) + require.Contains(t, apps, codersdk.TemplateAppUsage{ + // The rollup no longer produces SFTP usage, but rows migrated + // from the old sftp_mins column still report it. + TemplateIDs: []uuid.UUID{sftpTemplateID}, + Type: codersdk.TemplateAppsTypeBuiltin, + DisplayName: codersdk.TemplateBuiltinAppDisplayNameSFTP, + Slug: "sftp", + Icon: "/icon/terminal.svg", + Seconds: 300, + }) + }) + + t.Run("Malformed", func(t *testing.T) { + t.Parallel() + + for name, usage := range map[string]database.GetTemplateInsightsRow{ + "UsageSeconds": { + SessionFamilyUsageSeconds: json.RawMessage(`{"vscode": {}}`), + SessionFamilyTemplateIds: json.RawMessage(`{}`), + }, + "TemplateIDs": { + SessionFamilyUsageSeconds: json.RawMessage(`{}`), + SessionFamilyTemplateIds: json.RawMessage(`{"vscode": "not-a-uuid"}`), + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + apps, err := convertTemplateInsightsApps(usage, nil) + require.Error(t, err) + require.Nil(t, apps) + }) + } + }) +} diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go index 8a2674ce899..924848e7f88 100644 --- a/coderd/prometheusmetrics/insights/metricscollector.go +++ b/coderd/prometheusmetrics/insights/metricscollector.go @@ -2,6 +2,7 @@ package insights import ( "context" + "encoding/json" "slices" "sync/atomic" "time" @@ -34,7 +35,7 @@ type MetricsCollector struct { } type insightsData struct { - templates []database.GetTemplateInsightsByTemplateRow + templates []templateInsightsRow apps []database.GetTemplateAppInsightsByTemplateRow params []parameterRow @@ -42,6 +43,21 @@ type insightsData struct { organizationNames map[uuid.UUID]string // template ID → org name } +// templateInsightsRow is the decoded form of +// database.GetTemplateInsightsByTemplateRow, whose per-session-family usage +// arrives as a JSONB payload. +type templateInsightsRow struct { + templateID uuid.UUID + activeUsers int64 + usageSecondsByFamily map[codersdk.AppFamilyName]int64 +} + +// usageSeconds returns the usage seconds reported for a session family. A +// family the query did not report had no usage. +func (r templateInsightsRow) usageSeconds(family codersdk.AppFamilyName) int64 { + return r.usageSecondsByFamily[family] +} + type parameterRow struct { templateID uuid.UUID name string @@ -91,21 +107,26 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { eg, egCtx := errgroup.WithContext(ctx) eg.SetLimit(3) - var templateInsights []database.GetTemplateInsightsByTemplateRow + var templateInsights []templateInsightsRow var appInsights []database.GetTemplateAppInsightsByTemplateRow var paramInsights []parameterRow eg.Go(func() error { - var err error - templateInsights, err = mc.database.GetTemplateInsightsByTemplate(egCtx, database.GetTemplateInsightsByTemplateParams{ + rows, err := mc.database.GetTemplateInsightsByTemplate(egCtx, database.GetTemplateInsightsByTemplateParams{ StartTime: startTime, EndTime: endTime, AppFamilies: codersdk.SessionCountAppFamiliesJSON(), }) if err != nil { mc.logger.Error(ctx, "unable to fetch template insights from database", slog.Error(err)) + return err } - return err + templateInsights, err = convertTemplateInsights(rows) + if err != nil { + mc.logger.Error(ctx, "unable to convert template insights", slog.Error(err)) + return err + } + return nil }) eg.Go(func() error { var err error @@ -228,36 +249,36 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { // Built-in apps for _, templateRow := range data.templates { - orgName := data.organizationNames[templateRow.TemplateID] + orgName := data.organizationNames[templateRow.templateID] metricsCh <- prometheus.MustNewConstMetric(applicationsUsageSecondsDesc, prometheus.GaugeValue, - float64(templateRow.UsageVscodeSeconds), - data.templateNames[templateRow.TemplateID], + float64(templateRow.usageSeconds(codersdk.AppFamilyVSCode)), + data.templateNames[templateRow.templateID], codersdk.TemplateBuiltinAppDisplayNameVSCode, "", orgName) metricsCh <- prometheus.MustNewConstMetric(applicationsUsageSecondsDesc, prometheus.GaugeValue, - float64(templateRow.UsageJetbrainsSeconds), - data.templateNames[templateRow.TemplateID], + float64(templateRow.usageSeconds(codersdk.AppFamilyJetBrains)), + data.templateNames[templateRow.templateID], codersdk.TemplateBuiltinAppDisplayNameJetBrains, "", orgName) metricsCh <- prometheus.MustNewConstMetric(applicationsUsageSecondsDesc, prometheus.GaugeValue, - float64(templateRow.UsageReconnectingPtySeconds), - data.templateNames[templateRow.TemplateID], + float64(templateRow.usageSeconds(codersdk.AppFamilyReconnectingPTY)), + data.templateNames[templateRow.templateID], codersdk.TemplateBuiltinAppDisplayNameWebTerminal, "", orgName) metricsCh <- prometheus.MustNewConstMetric(applicationsUsageSecondsDesc, prometheus.GaugeValue, - float64(templateRow.UsageSshSeconds), - data.templateNames[templateRow.TemplateID], + float64(templateRow.usageSeconds(codersdk.AppFamilySSH)), + data.templateNames[templateRow.templateID], codersdk.TemplateBuiltinAppDisplayNameSSH, "", orgName) } // Templates for _, templateRow := range data.templates { - metricsCh <- prometheus.MustNewConstMetric(templatesActiveUsersDesc, prometheus.GaugeValue, float64(templateRow.ActiveUsers), data.templateNames[templateRow.TemplateID], data.organizationNames[templateRow.TemplateID]) + metricsCh <- prometheus.MustNewConstMetric(templatesActiveUsersDesc, prometheus.GaugeValue, float64(templateRow.activeUsers), data.templateNames[templateRow.templateID], data.organizationNames[templateRow.templateID]) } // Parameters @@ -268,10 +289,10 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) { // Helper functions below. -func uniqueTemplateIDs(templateInsights []database.GetTemplateInsightsByTemplateRow, appInsights []database.GetTemplateAppInsightsByTemplateRow, paramInsights []parameterRow) []uuid.UUID { +func uniqueTemplateIDs(templateInsights []templateInsightsRow, appInsights []database.GetTemplateAppInsightsByTemplateRow, paramInsights []parameterRow) []uuid.UUID { tids := map[uuid.UUID]bool{} for _, t := range templateInsights { - tids[t.TemplateID] = true + tids[t.templateID] = true } for _, t := range appInsights { tids[t.TemplateID] = true @@ -297,6 +318,43 @@ func onlyTemplateNames(templates []database.Template) map[uuid.UUID]string { return m } +// convertTemplateInsights decodes the JSONB session family usage of each +// template insights row. A malformed payload is an error rather than zero +// usage, so the collector keeps serving the previous snapshot instead of +// reporting idle templates. +func convertTemplateInsights(rows []database.GetTemplateInsightsByTemplateRow) ([]templateInsightsRow, error) { + converted := make([]templateInsightsRow, 0, len(rows)) + for _, row := range rows { + usageSeconds, err := decodeSessionFamilyUsageSeconds(row.SessionFamilyUsageSeconds) + if err != nil { + return nil, xerrors.Errorf("template %s: %w", row.TemplateID, err) + } + converted = append(converted, templateInsightsRow{ + templateID: row.TemplateID, + activeUsers: row.ActiveUsers, + usageSecondsByFamily: usageSeconds, + }) + } + return converted, nil +} + +// decodeSessionFamilyUsageSeconds decodes a usage seconds JSONB payload keyed +// by session family. An absent payload decodes to an empty map, but a +// malformed one is an error. +func decodeSessionFamilyUsageSeconds(raw json.RawMessage) (map[codersdk.AppFamilyName]int64, error) { + if len(raw) == 0 { + return map[codersdk.AppFamilyName]int64{}, nil + } + var decoded map[codersdk.AppFamilyName]int64 + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, xerrors.Errorf("unmarshal session family usage seconds: %w", err) + } + if decoded == nil { + return map[codersdk.AppFamilyName]int64{}, nil + } + return decoded, nil +} + func convertParameterInsights(rows []database.GetTemplateParameterInsightsRow) []parameterRow { type uniqueKey struct { templateID uuid.UUID diff --git a/coderd/prometheusmetrics/insights/metricscollector_internal_test.go b/coderd/prometheusmetrics/insights/metricscollector_internal_test.go new file mode 100644 index 00000000000..4a22858b8b0 --- /dev/null +++ b/coderd/prometheusmetrics/insights/metricscollector_internal_test.go @@ -0,0 +1,50 @@ +package insights + +import ( + "encoding/json" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/codersdk" +) + +func TestConvertTemplateInsights(t *testing.T) { + t.Parallel() + + t.Run("Malformed", func(t *testing.T) { + t.Parallel() + + for name, raw := range map[string]json.RawMessage{ + "NotJSON": json.RawMessage(`{`), + "NotAnObject": json.RawMessage(`[1, 2]`), + "WrongValue": json.RawMessage(`{"ssh": "sixty"}`), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + // Reporting zero usage would look like an idle template, so + // the collector must fail the tick instead. + rows, err := convertTemplateInsights([]database.GetTemplateInsightsByTemplateRow{ + {TemplateID: uuid.New(), SessionFamilyUsageSeconds: raw}, + }) + require.Error(t, err) + require.Nil(t, rows) + }) + } + }) + + t.Run("AbsentPayload", func(t *testing.T) { + t.Parallel() + + rows, err := convertTemplateInsights([]database.GetTemplateInsightsByTemplateRow{ + {TemplateID: uuid.New()}, + }) + require.NoError(t, err) + require.Len(t, rows, 1) + require.NotNil(t, rows[0].usageSecondsByFamily) + require.EqualValues(t, 0, rows[0].usageSeconds(codersdk.AppFamilySSH)) + }) +} diff --git a/codersdk/appname.go b/codersdk/appname.go index 4aa436def71..206643d6ba6 100644 --- a/codersdk/appname.go +++ b/codersdk/appname.go @@ -2,7 +2,6 @@ package codersdk import ( "encoding/json" - "slices" "strings" "golang.org/x/xerrors" @@ -28,8 +27,11 @@ const ( AppFamilyUnknown AppFamilyName = "unknown" ) -// appNameFamilies never gates storage, so a missing alias only costs an -// AppFamilyUnknown label. Keys are the IDs Coder's registry modules use. +// appNameFamilies is the only place an app name is attributed to a family. +// Storage keeps the raw app name, so a missing alias only costs an +// AppFamilyUnknown attribution rather than a dropped session. Keys are the +// IDs Coder's registry modules use, normalized as NormalizeAppName leaves +// them. var appNameFamilies = map[string]AppFamilyName{ "vscode": AppFamilyVSCode, "vscode_insiders": AppFamilyVSCode, @@ -53,66 +55,24 @@ var appNameFamilies = map[string]AppFamilyName{ "reconnecting_pty": AppFamilyReconnectingPTY, } -// attributedAppFamilies are the families usage reporting has somewhere to -// put, and the single definition of the families the session count read -// queries know about. Every value in appNameFamilies must appear here or its -// sessions go uncounted, which TestEveryFamilyIsAttributed enforces. -// -// Adding a family here is not enough on its own: see AttributedAppFamilies. -var attributedAppFamilies = []AppFamilyName{ - AppFamilyVSCode, - AppFamilyJetBrains, - AppFamilySSH, - AppFamilyReconnectingPTY, -} - -// AttributedAppFamilies returns the families session count reporting can -// attribute to, in registry order. It is the source of truth that dbauthz -// validates the query parameter against, so a family that exists here but not -// in the queries (or the reverse) fails loudly instead of silently reporting -// zero. -func AttributedAppFamilies() []AppFamilyName { - return slices.Clone(attributedAppFamilies) -} - -// AppNamesInFamily returns the app names belonging to a family, sorted so -// query parameters stay stable across calls. -func AppNamesInFamily(family AppFamilyName) []string { - var names []string - for appName, appFamily := range appNameFamilies { - if appFamily == family { - names = append(names, appName) - } - } - slices.Sort(names) - return names -} - -// SessionCountAppFamilies returns the session count attribution registry: -// every attributed family mapped to its sorted app names. The session count -// read queries take it as one parameter, so a new app name reaches every -// caller by being added to appNameFamilies alone. -// -// A new family costs more. sqlc output columns are static, so each family -// needs, in addition to its attributedAppFamilies entry, one probe expression -// in every query that reports per-family session counts (see -// coderd/database/queries/workspaceagentstats.sql and insights.sql) plus the -// matching output column. dbauthz validation rejects a registry whose -// families do not match AttributedAppFamilies, and -// TestAttributedAppFamiliesMatchQueries pins that list to what the queries -// probe, so neither half can drift unnoticed. -func SessionCountAppFamilies() map[AppFamilyName][]string { - families := make(map[AppFamilyName][]string, len(attributedAppFamilies)) - for _, family := range attributedAppFamilies { - families[family] = AppNamesInFamily(family) +// SessionCountAppFamilies returns the app-to-family attribution registry: one +// entry per known app name, mapped to the family it reports under. Callers +// that need a fixed family value derive it from this map, so registering a +// new app or family means editing appNameFamilies alone. +func SessionCountAppFamilies() map[string]AppFamilyName { + families := make(map[string]AppFamilyName, len(appNameFamilies)) + for appName, family := range appNameFamilies { + families[appName] = family } return families } -// SessionCountAppFamiliesJSON is SessionCountAppFamilies marshaled for the -// jsonb parameter the session count read queries accept. +// SessionCountAppFamiliesJSON is SessionCountAppFamilies marshaled as the +// jsonb object of app name to family name that the minute aggregation queries +// decompose with jsonb_each_text. Queries join it by app name, so no query +// names a family and no family needs its own column or probe. func SessionCountAppFamiliesJSON() json.RawMessage { - // Marshaling a map with a string-keyed type cannot fail. + // Marshaling a map with string-kinded keys and values cannot fail. data, err := json.Marshal(SessionCountAppFamilies()) if err != nil { panic("developer error: marshal session count app families: " + err.Error()) diff --git a/codersdk/appname_internal_test.go b/codersdk/appname_internal_test.go index 8358ad1e1b2..2a894a1c517 100644 --- a/codersdk/appname_internal_test.go +++ b/codersdk/appname_internal_test.go @@ -11,16 +11,21 @@ import ( func TestAppNameFamilyKeysAreNormalized(t *testing.T) { t.Parallel() for name := range appNameFamilies { + require.NotEmpty(t, name) require.Equal(t, NormalizeAppName(name), name) } } -// A family with no destination in attributedAppFamilies silently drops its -// sessions from usage reporting, so adding one must fail here first. -func TestEveryFamilyIsAttributed(t *testing.T) { +// Families reach the queries as jsonb values and metric labels, so a blank +// entry would attribute sessions to an unusable name. AppFamilyUnknown is the +// fold destination for unregistered apps, not something to register. +func TestAppNameFamilyValuesAreUsable(t *testing.T) { t.Parallel() for appName, family := range appNameFamilies { - require.Contains(t, attributedAppFamilies, family, - "app %q maps to family %q, which usage reporting cannot report", appName, family) + require.NotEmpty(t, family, "app %q has no family", appName) + require.NotEqual(t, AppFamilyUnknown, family, + "app %q must not register the unknown family", appName) + require.Equal(t, NormalizeAppName(string(family)), string(family), + "family %q must be normalized", family) } } diff --git a/codersdk/appname_test.go b/codersdk/appname_test.go index 2e53df813b7..2ac573c8c86 100644 --- a/codersdk/appname_test.go +++ b/codersdk/appname_test.go @@ -2,7 +2,6 @@ package codersdk_test import ( "encoding/json" - "maps" "slices" "strings" "testing" @@ -87,63 +86,48 @@ func TestAppNameFamily(t *testing.T) { } } -func TestAppNamesInFamily(t *testing.T) { +// Family sets are derived from the one registry, so callers that need the app +// names in a family do not need a second list. +func TestRegistryFamilySets(t *testing.T) { t.Parallel() - // Forks share the VS Code family, and the list is sorted. - vscode := codersdk.AppNamesInFamily(codersdk.AppFamilyVSCode) + registry := codersdk.SessionCountAppFamilies() + inFamily := func(want codersdk.AppFamilyName) []string { + var names []string + for appName, family := range registry { + if family == want { + names = append(names, appName) + } + } + slices.Sort(names) + return names + } + + // Forks share the VS Code family. + vscode := inFamily(codersdk.AppFamilyVSCode) require.Contains(t, vscode, "cursor") require.Contains(t, vscode, "vscode") - require.True(t, slices.IsSorted(vscode)) // Zed speaks SSH, so it reports under the SSH family. - require.Equal(t, []string{"ssh", "zed"}, - codersdk.AppNamesInFamily(codersdk.AppFamilySSH)) + require.Equal(t, []string{"ssh", "zed"}, inFamily(codersdk.AppFamilySSH)) - require.Empty(t, codersdk.AppNamesInFamily("no_such_family")) -} - -// The SQL queries that report per-family session counts hardcode one probe -// expression and one output column per family, because sqlc output columns -// are static. This pins the Go registry to the families those queries know -// about. -// -// When adding a family, update, in this order: -// 1. attributedAppFamilies in appname.go. -// 2. Every query that reports per-family session counts: the fams CTE list, -// the probe expression, and the output column, in -// coderd/database/queries/workspaceagentstats.sql and insights.sql. -// 3. The Go readers of those columns, then this list. -// -// dbauthz validation rejects a registry that does not match -// AttributedAppFamilies, so a family added to Go but not to SQL fails at -// runtime too. -func TestAttributedAppFamiliesMatchQueries(t *testing.T) { - t.Parallel() - - require.Equal(t, []codersdk.AppFamilyName{ - codersdk.AppFamilyVSCode, - codersdk.AppFamilyJetBrains, - codersdk.AppFamilySSH, - codersdk.AppFamilyReconnectingPTY, - }, codersdk.AttributedAppFamilies()) - - // The registry keys are what the queries index by name. - require.ElementsMatch(t, codersdk.AttributedAppFamilies(), - slices.Collect(maps.Keys(codersdk.SessionCountAppFamilies())), - "registry keys must match the attributed families") + require.Empty(t, inFamily("no_such_family")) } func TestSessionCountAppFamilies(t *testing.T) { t.Parallel() - families := codersdk.SessionCountAppFamilies() - require.Len(t, families, 4, "every attributed family must be present") - require.Contains(t, families, codersdk.AppFamilyVSCode) - require.Contains(t, families, codersdk.AppFamilyJetBrains) - require.Contains(t, families, codersdk.AppFamilySSH) - require.Contains(t, families, codersdk.AppFamilyReconnectingPTY) - require.Equal(t, codersdk.AppNamesInFamily(codersdk.AppFamilyVSCode), families[codersdk.AppFamilyVSCode]) + registry := codersdk.SessionCountAppFamilies() + require.NotEmpty(t, registry) + for appName, family := range registry { + require.Equal(t, family, codersdk.AppNameFamily(appName), + "registry entry %q must agree with lookup", appName) + } + + // The registry is a copy, so a caller cannot corrupt attribution. + registry["cursor"] = codersdk.AppFamilySSH + require.Equal(t, codersdk.AppFamilyVSCode, codersdk.AppNameFamily("cursor")) + require.Equal(t, codersdk.AppFamilyVSCode, codersdk.SessionCountAppFamilies()["cursor"]) } func TestSessionCountAppFamiliesJSON(t *testing.T) { @@ -152,7 +136,9 @@ func TestSessionCountAppFamiliesJSON(t *testing.T) { raw := codersdk.SessionCountAppFamiliesJSON() require.NotEmpty(t, raw) - var decoded map[codersdk.AppFamilyName][]string + // The minute aggregation queries decompose this with jsonb_each_text, so + // it must be a flat object of app name to family name. + var decoded map[string]codersdk.AppFamilyName require.NoError(t, json.Unmarshal(raw, &decoded), "registry must marshal to a valid jsonb object") require.Equal(t, codersdk.SessionCountAppFamilies(), decoded) } @@ -219,18 +205,16 @@ func TestSessionCountsByFamily(t *testing.T) { } } -// Every app name the registry attributes must fold back into the family it is -// registered under, so no attributed family can go uncounted. -func TestSessionCountsByFamilyCoversEveryAttributedFamily(t *testing.T) { +// A family is registered by adding app names alone, with no SQL, column, or +// second list to update. +func TestSessionCountsByFamilyCoversEveryRegisteredFamily(t *testing.T) { t.Parallel() appCounts := map[string]int64{} want := map[codersdk.AppFamilyName]int64{} - for family, appNames := range codersdk.SessionCountAppFamilies() { - for _, appName := range appNames { - appCounts[appName] = 1 - want[family]++ - } + for appName, family := range codersdk.SessionCountAppFamilies() { + appCounts[appName] = 1 + want[family]++ } require.Equal(t, want, codersdk.SessionCountsByFamily(appCounts)) } From 7c44d2eaed2f8cc4678a6b9d671c05e078cedb16 Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Wed, 9 Sep 2026 12:15:35 +0000 Subject: [PATCH 2/7] perf(coderd/database): skip child usage writes for unchanged buckets Record a digest of each bucket's session usage rows on template_usage_stats and drive the child table deletes and upserts from the main upsert's RETURNING set, so a rollup only touches child rows for buckets whose session usage changed. Steady-state rollups measured 10% to 17% faster. --- coderd/database/dbrollup/dbrollup_test.go | 6 + coderd/database/dump.sql | 5 +- ...emplate_usage_stats_session_usage.down.sql | 3 + ..._template_usage_stats_session_usage.up.sql | 5 + coderd/database/models.go | 2 + coderd/database/queries.sql.go | 109 ++++++++++++++---- coderd/database/queries/insights.sql | 106 +++++++++++++---- coderd/database/session_usage_test.go | 71 +++++++++++- 8 files changed, 258 insertions(+), 49 deletions(-) diff --git a/coderd/database/dbrollup/dbrollup_test.go b/coderd/database/dbrollup/dbrollup_test.go index cb81abac366..e749ab4e9d8 100644 --- a/coderd/database/dbrollup/dbrollup_test.go +++ b/coderd/database/dbrollup/dbrollup_test.go @@ -244,6 +244,12 @@ func TestRollupTemplateUsageStats(t *testing.T) { stats[0].EndTime = stats[0].EndTime.UTC() stats[0].StartTime = stats[0].StartTime.UTC() + // The digest is a hash of the child rows; its value is not worth pinning, + // only that the rollup recorded one. + require.True(t, stats[0].SessionUsageDigest.Valid, "the rollup must record a session usage digest") + require.NotZero(t, stats[0].SessionUsageDigest.Int64) + stats[0].SessionUsageDigest = sql.NullInt64{} + require.Equal(t, database.TemplateUsageStat{ TemplateID: tpl.ID, UserID: user.ID, diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 3858e958e6a..af5782264aa 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -3056,7 +3056,8 @@ CREATE TABLE template_usage_stats ( user_id uuid NOT NULL, median_latency_ms real, usage_mins smallint NOT NULL, - app_usage_mins jsonb + app_usage_mins jsonb, + session_usage_digest bigint ); COMMENT ON TABLE template_usage_stats IS 'Records aggregated usage statistics for templates/users. All usage is rounded up to the nearest minute.'; @@ -3075,6 +3076,8 @@ COMMENT ON COLUMN template_usage_stats.usage_mins IS 'Total minutes the user has COMMENT ON COLUMN template_usage_stats.app_usage_mins IS 'Object with app names as keys and total minutes used as values. Null means no app usage was recorded.'; +COMMENT ON COLUMN template_usage_stats.session_usage_digest IS 'Hash of the bucket''s session usage rows in both child tables, so a rollup that recomputes an unchanged bucket rewrites no child rows. Null for buckets rolled up before the column existed, which reads as changed.'; + CREATE TABLE template_usage_stats_session_apps ( start_time timestamp with time zone NOT NULL, template_id uuid NOT NULL, diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql b/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql index c74189aca4a..a1ab50c0c44 100644 --- a/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql +++ b/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql @@ -59,3 +59,6 @@ WHERE DROP TABLE template_usage_stats_session_apps; DROP TABLE template_usage_stats_session_families; + +ALTER TABLE template_usage_stats + DROP COLUMN session_usage_digest; diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql index 1f3201fbf53..28244076e64 100644 --- a/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql +++ b/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql @@ -40,6 +40,11 @@ COMMENT ON COLUMN template_usage_stats_session_apps.app_name IS 'App name as the COMMENT ON COLUMN template_usage_stats_session_apps.usage_mins IS 'Total minutes the user has been using the app.'; +ALTER TABLE template_usage_stats + ADD COLUMN session_usage_digest bigint; + +COMMENT ON COLUMN template_usage_stats.session_usage_digest IS 'Hash of the bucket''s session usage rows in both child tables, so a rollup that recomputes an unchanged bucket rewrites no child rows. Null for buckets rolled up before the column existed, which reads as changed.'; + -- Carry every family the fixed columns recorded, sftp included: the rollup has -- never written it, but a row that has a value must not lose it. Zero minutes -- are skipped so a bucket has rows only for the families it saw, which is what diff --git a/coderd/database/models.go b/coderd/database/models.go index 64296545280..b9769352e36 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -6050,6 +6050,8 @@ type TemplateUsageStat struct { UsageMins int16 `db:"usage_mins" json:"usage_mins"` // Object with app names as keys and total minutes used as values. Null means no app usage was recorded. AppUsageMins StringMapOfInt `db:"app_usage_mins" json:"app_usage_mins"` + // Hash of the bucket's session usage rows in both child tables, so a rollup that recomputes an unchanged bucket rewrites no child rows. Null for buckets rolled up before the column existed, which reads as changed. + SessionUsageDigest sql.NullInt64 `db:"session_usage_digest" json:"session_usage_digest"` } // Session usage of each template_usage_stats bucket, split by app name. A bucket with family rows but no rows here predates per-app recording, so its per-app usage is unknown rather than zero. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index f1757920680..52c600719c2 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -17153,7 +17153,7 @@ func (q *sqlQuerier) GetTemplateParameterInsights(ctx context.Context, arg GetTe const getTemplateUsageStats = `-- name: GetTemplateUsageStats :many SELECT - start_time, end_time, template_id, user_id, median_latency_ms, usage_mins, app_usage_mins + start_time, end_time, template_id, user_id, median_latency_ms, usage_mins, app_usage_mins, session_usage_digest FROM template_usage_stats WHERE @@ -17185,6 +17185,7 @@ func (q *sqlQuerier) GetTemplateUsageStats(ctx context.Context, arg GetTemplateU &i.MedianLatencyMs, &i.UsageMins, &i.AppUsageMins, + &i.SessionUsageDigest, ); err != nil { return nil, err } @@ -17688,6 +17689,31 @@ WITH -- bucket that only has app stats records no session usage. AND buckets.has_connection ), + session_digests AS ( + -- A stable hash of the bucket's session usage: the ordered set of + -- (kind, name, minutes). It is carried on the main row so the upsert's + -- IS DISTINCT FROM guard fires when session usage changes, which is + -- what lets the child writes below skip unchanged buckets. + -- + -- INVARIANT: the digest must cover every column the child tables store. + -- A column added to either table but left out of the digest would leave + -- a bucket looking unchanged whenever only that column changed, and the + -- bucket would keep stale child rows. + SELECT + time_bucket AS start_time, + template_id, + user_id, + hashtextextended(string_agg( + family_group || ':' || name || ':' || usage_mins, + '|' ORDER BY family_group, name + ), 0) AS digest + FROM + agent_stats_session_minutes + WHERE + name IS NOT NULL + GROUP BY + time_bucket, template_id, user_id + ), stats AS ( SELECT stats.time_bucket AS start_time, @@ -17771,7 +17797,8 @@ WITH user_id, usage_mins, median_latency_ms, - app_usage_mins + app_usage_mins, + session_usage_digest ) ( SELECT stats.start_time, @@ -17780,9 +17807,19 @@ WITH stats.user_id, stats.usage_mins, latencies.median_latency_ms, - stats.app_usage_mins + stats.app_usage_mins, + -- A bucket with no session usage still gets a digest, so a null + -- left by a rollup that predates the column reads as changed + -- once and then settles. + COALESCE(session_digests.digest, hashtextextended('', 0)) FROM stats + LEFT JOIN + session_digests + ON + session_digests.start_time = stats.start_time + AND session_digests.template_id = stats.template_id + AND session_digests.user_id = stats.user_id LEFT JOIN latencies ON @@ -17797,11 +17834,25 @@ WITH SET usage_mins = EXCLUDED.usage_mins, median_latency_ms = EXCLUDED.median_latency_ms, - app_usage_mins = EXCLUDED.app_usage_mins + app_usage_mins = EXCLUDED.app_usage_mins, + session_usage_digest = EXCLUDED.session_usage_digest WHERE (tus.*) IS DISTINCT FROM (EXCLUDED.*) RETURNING - tus.start_time + tus.start_time, + tus.template_id, + tus.user_id + ), + changed_buckets AS ( + -- New or changed buckets only. A bucket whose main row, digest + -- included, was already correct returns nothing from the upsert, so the + -- four child writes below never touch it. + SELECT + start_time, + template_id, + user_id + FROM + upsert_stats ), -- The child writes below run in this same statement, so the foreign key -- triggers fire once it completes and see the main rows the upsert above @@ -17820,11 +17871,11 @@ WITH DELETE FROM template_usage_stats_session_families AS families USING - stats + changed_buckets AS changed WHERE - families.start_time = stats.start_time - AND families.template_id = stats.template_id - AND families.user_id = stats.user_id + families.start_time = changed.start_time + AND families.template_id = changed.template_id + AND families.user_id = changed.user_id AND (families.start_time, families.template_id, families.user_id, families.family) NOT IN ( SELECT time_bucket, template_id, user_id, name FROM agent_stats_session_minutes @@ -17840,13 +17891,19 @@ WITH usage_mins ) ( SELECT - time_bucket, - template_id, - user_id, - name, - usage_mins + agent_stats_session_minutes.time_bucket, + agent_stats_session_minutes.template_id, + agent_stats_session_minutes.user_id, + agent_stats_session_minutes.name, + agent_stats_session_minutes.usage_mins FROM agent_stats_session_minutes + JOIN + changed_buckets AS changed + ON + changed.start_time = agent_stats_session_minutes.time_bucket + AND changed.template_id = agent_stats_session_minutes.template_id + AND changed.user_id = agent_stats_session_minutes.user_id WHERE family_group = 1 ) @@ -17862,11 +17919,11 @@ WITH DELETE FROM template_usage_stats_session_apps AS apps USING - stats + changed_buckets AS changed WHERE - apps.start_time = stats.start_time - AND apps.template_id = stats.template_id - AND apps.user_id = stats.user_id + apps.start_time = changed.start_time + AND apps.template_id = changed.template_id + AND apps.user_id = changed.user_id AND (apps.start_time, apps.template_id, apps.user_id, apps.app_name) NOT IN ( SELECT time_bucket, template_id, user_id, name FROM agent_stats_session_minutes @@ -17882,13 +17939,19 @@ INSERT INTO template_usage_stats_session_apps AS apps ( usage_mins ) ( SELECT - time_bucket, - template_id, - user_id, - name, - usage_mins + agent_stats_session_minutes.time_bucket, + agent_stats_session_minutes.template_id, + agent_stats_session_minutes.user_id, + agent_stats_session_minutes.name, + agent_stats_session_minutes.usage_mins FROM agent_stats_session_minutes + JOIN + changed_buckets AS changed + ON + changed.start_time = agent_stats_session_minutes.time_bucket + AND changed.template_id = agent_stats_session_minutes.template_id + AND changed.user_id = agent_stats_session_minutes.user_id WHERE family_group = 0 ) diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 8c81a8c6fda..6a5b2a73e67 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -803,6 +803,31 @@ WITH -- bucket that only has app stats records no session usage. AND buckets.has_connection ), + session_digests AS ( + -- A stable hash of the bucket's session usage: the ordered set of + -- (kind, name, minutes). It is carried on the main row so the upsert's + -- IS DISTINCT FROM guard fires when session usage changes, which is + -- what lets the child writes below skip unchanged buckets. + -- + -- INVARIANT: the digest must cover every column the child tables store. + -- A column added to either table but left out of the digest would leave + -- a bucket looking unchanged whenever only that column changed, and the + -- bucket would keep stale child rows. + SELECT + time_bucket AS start_time, + template_id, + user_id, + hashtextextended(string_agg( + family_group || ':' || name || ':' || usage_mins, + '|' ORDER BY family_group, name + ), 0) AS digest + FROM + agent_stats_session_minutes + WHERE + name IS NOT NULL + GROUP BY + time_bucket, template_id, user_id + ), stats AS ( SELECT stats.time_bucket AS start_time, @@ -886,7 +911,8 @@ WITH user_id, usage_mins, median_latency_ms, - app_usage_mins + app_usage_mins, + session_usage_digest ) ( SELECT stats.start_time, @@ -895,9 +921,19 @@ WITH stats.user_id, stats.usage_mins, latencies.median_latency_ms, - stats.app_usage_mins + stats.app_usage_mins, + -- A bucket with no session usage still gets a digest, so a null + -- left by a rollup that predates the column reads as changed + -- once and then settles. + COALESCE(session_digests.digest, hashtextextended('', 0)) FROM stats + LEFT JOIN + session_digests + ON + session_digests.start_time = stats.start_time + AND session_digests.template_id = stats.template_id + AND session_digests.user_id = stats.user_id LEFT JOIN latencies ON @@ -912,11 +948,25 @@ WITH SET usage_mins = EXCLUDED.usage_mins, median_latency_ms = EXCLUDED.median_latency_ms, - app_usage_mins = EXCLUDED.app_usage_mins + app_usage_mins = EXCLUDED.app_usage_mins, + session_usage_digest = EXCLUDED.session_usage_digest WHERE (tus.*) IS DISTINCT FROM (EXCLUDED.*) RETURNING - tus.start_time + tus.start_time, + tus.template_id, + tus.user_id + ), + changed_buckets AS ( + -- New or changed buckets only. A bucket whose main row, digest + -- included, was already correct returns nothing from the upsert, so the + -- four child writes below never touch it. + SELECT + start_time, + template_id, + user_id + FROM + upsert_stats ), -- The child writes below run in this same statement, so the foreign key -- triggers fire once it completes and see the main rows the upsert above @@ -935,11 +985,11 @@ WITH DELETE FROM template_usage_stats_session_families AS families USING - stats + changed_buckets AS changed WHERE - families.start_time = stats.start_time - AND families.template_id = stats.template_id - AND families.user_id = stats.user_id + families.start_time = changed.start_time + AND families.template_id = changed.template_id + AND families.user_id = changed.user_id AND (families.start_time, families.template_id, families.user_id, families.family) NOT IN ( SELECT time_bucket, template_id, user_id, name FROM agent_stats_session_minutes @@ -955,13 +1005,19 @@ WITH usage_mins ) ( SELECT - time_bucket, - template_id, - user_id, - name, - usage_mins + agent_stats_session_minutes.time_bucket, + agent_stats_session_minutes.template_id, + agent_stats_session_minutes.user_id, + agent_stats_session_minutes.name, + agent_stats_session_minutes.usage_mins FROM agent_stats_session_minutes + JOIN + changed_buckets AS changed + ON + changed.start_time = agent_stats_session_minutes.time_bucket + AND changed.template_id = agent_stats_session_minutes.template_id + AND changed.user_id = agent_stats_session_minutes.user_id WHERE family_group = 1 ) @@ -977,11 +1033,11 @@ WITH DELETE FROM template_usage_stats_session_apps AS apps USING - stats + changed_buckets AS changed WHERE - apps.start_time = stats.start_time - AND apps.template_id = stats.template_id - AND apps.user_id = stats.user_id + apps.start_time = changed.start_time + AND apps.template_id = changed.template_id + AND apps.user_id = changed.user_id AND (apps.start_time, apps.template_id, apps.user_id, apps.app_name) NOT IN ( SELECT time_bucket, template_id, user_id, name FROM agent_stats_session_minutes @@ -997,13 +1053,19 @@ INSERT INTO template_usage_stats_session_apps AS apps ( usage_mins ) ( SELECT - time_bucket, - template_id, - user_id, - name, - usage_mins + agent_stats_session_minutes.time_bucket, + agent_stats_session_minutes.template_id, + agent_stats_session_minutes.user_id, + agent_stats_session_minutes.name, + agent_stats_session_minutes.usage_mins FROM agent_stats_session_minutes + JOIN + changed_buckets AS changed + ON + changed.start_time = agent_stats_session_minutes.time_bucket + AND changed.template_id = agent_stats_session_minutes.template_id + AND changed.user_id = agent_stats_session_minutes.user_id WHERE family_group = 0 ) diff --git a/coderd/database/session_usage_test.go b/coderd/database/session_usage_test.go index 486c64fbfae..ce0244b4cc9 100644 --- a/coderd/database/session_usage_test.go +++ b/coderd/database/session_usage_test.go @@ -39,6 +39,37 @@ func sessionUsageMins(ctx context.Context, t *testing.T, sqlDB *sql.DB, table, n return got } +// sessionUsageDigest reads the digest the rollup stored for one bucket, and +// emptySessionUsageDigest is the digest of a bucket with no session usage. +func sessionUsageDigest(ctx context.Context, t *testing.T, sqlDB *sql.DB, startTime time.Time, userID, templateID uuid.UUID) sql.NullInt64 { + t.Helper() + + var digest sql.NullInt64 + require.NoError(t, sqlDB.QueryRowContext(ctx, + `SELECT session_usage_digest FROM template_usage_stats WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, + startTime, userID, templateID).Scan(&digest)) + return digest +} + +func emptySessionUsageDigest(ctx context.Context, t *testing.T, sqlDB *sql.DB) sql.NullInt64 { + t.Helper() + + var digest sql.NullInt64 + require.NoError(t, sqlDB.QueryRowContext(ctx, `SELECT hashtextextended('', 0)`).Scan(&digest)) + return digest +} + +// withoutDigest clears the session usage digest, which is a hash of the child +// rows rather than a value worth comparing. +func withoutDigest(stats []database.TemplateUsageStat) []database.TemplateUsageStat { + cleared := make([]database.TemplateUsageStat, 0, len(stats)) + for _, stat := range stats { + stat.SessionUsageDigest = sql.NullInt64{} + cleared = append(cleared, stat) + } + return cleared +} + func TestSessionUsageMapNull(t *testing.T) { t.Parallel() m := database.StringMapOfInt{"cursor": 1} @@ -99,6 +130,9 @@ func TestSessionUsageRollupOverlapAndRegistry(t *testing.T) { sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template)) require.Equal(t, map[string]int64{"vscode": 3, "new_family": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template)) + digest := sessionUsageDigest(ctx, t, sqlDB, start, user, template) + require.True(t, digest.Valid, "a rolled up bucket must carry a session usage digest") + require.NotEqual(t, emptySessionUsageDigest(ctx, t, sqlDB), digest, "this bucket has session usage") // The existing watermark deliberately recomputes recent buckets. require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) @@ -107,6 +141,10 @@ func TestSessionUsageRollupOverlapAndRegistry(t *testing.T) { require.ElementsMatch(t, rows, repeated) require.Equal(t, map[string]int64{"vscode": 3, "new_family": 1}, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template)) + // Recomputing identical session usage leaves the digest alone, which is + // what keeps the rollup from rewriting the child rows of every bucket in + // its window. + require.Equal(t, digest, sessionUsageDigest(ctx, t, sqlDB, start, user, template)) // Renaming a family in the registry moves the minutes: the recomputed // bucket drops the family row it no longer has and keeps its app rows, @@ -117,15 +155,42 @@ func TestSessionUsageRollupOverlapAndRegistry(t *testing.T) { require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) repeated, err = db.GetTemplateUsageStats(ctx, params) require.NoError(t, err) - require.ElementsMatch(t, rows, repeated) + // Only the digest may differ on the main row: the rename changes what the + // child rows hold and nothing else. + require.ElementsMatch(t, withoutDigest(rows), withoutDigest(repeated)) families := sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template) require.EqualValues(t, 1, families["renamed_family"]) require.NotContains(t, families, "new_family") apps := sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template) require.EqualValues(t, 1, apps["new_app"]) + // The rename shows up in no other column of the main row, so the digest is + // what marks the bucket changed and lets the child writes reach it. + require.NotEqual(t, digest, sessionUsageDigest(ctx, t, sqlDB, start, user, template)) + + // A bucket that only has app stats records no session usage at all, so it + // carries the empty digest rather than a null, and a deleted bucket takes + // its session usage with it. + org := dbgen.Organization(t, db, database.Organization{}) + owner := dbgen.User(t, db, database.User{Name: "app-stats-only"}) + appTemplate := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, CreatedBy: owner.ID}) + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{OrganizationID: org.ID, TemplateID: appTemplate.ID, OwnerID: owner.ID}) + job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{OrganizationID: org.ID}) + resource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{JobID: job.ID}) + agent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{ResourceID: resource.ID}) + dbgen.WorkspaceAppStat(t, db, database.WorkspaceAppStat{ + UserID: owner.ID, + WorkspaceID: workspace.ID, + AgentID: agent.ID, + AccessMethod: "path", + SlugOrPort: "code-server", + SessionStartedAt: start.Add(time.Minute), + SessionEndedAt: start.Add(3 * time.Minute), + }) + require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) + require.Equal(t, emptySessionUsageDigest(ctx, t, sqlDB), sessionUsageDigest(ctx, t, sqlDB, start, owner.ID, appTemplate.ID)) + require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, owner.ID, appTemplate.ID)) + require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, owner.ID, appTemplate.ID)) - // A bucket that only has app stats records no session usage at all, and a - // deleted bucket takes its session usage with it. require.Empty(t, sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, disconnected)) _, err = sqlDB.ExecContext(ctx, `DELETE FROM template_usage_stats WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, start, user, template) require.NoError(t, err) From 89fad5735228366b1d2c81f72e841d6b0d8c410d Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Thu, 10 Sep 2026 10:39:19 +0000 Subject: [PATCH 3/7] fix(coderd/database): disambiguate session usage digests Length-prefix names in the digest input so registry changes cannot alias different child row sets. Pin child replacement and unchanged-row write elision with a regression. Renumber the unmerged session usage migration to 000592 because main now uses 000591. Generated by Coder Agents for @EhabY. --- ...mplate_usage_stats_session_usage.down.sql} | 0 ...template_usage_stats_session_usage.up.sql} | 0 ...000591_test.go => migration000592_test.go} | 30 ++++---- ...template_usage_stats_session_usage.up.sql} | 0 coderd/database/queries.sql.go | 6 +- coderd/database/queries/insights.sql | 6 +- coderd/database/session_usage_digest_test.go | 68 +++++++++++++++++++ 7 files changed, 91 insertions(+), 19 deletions(-) rename coderd/database/migrations/{000591_template_usage_stats_session_usage.down.sql => 000592_template_usage_stats_session_usage.down.sql} (100%) rename coderd/database/migrations/{000591_template_usage_stats_session_usage.up.sql => 000592_template_usage_stats_session_usage.up.sql} (100%) rename coderd/database/migrations/{migration000591_test.go => migration000592_test.go} (93%) rename coderd/database/migrations/testdata/fixtures/{000591_template_usage_stats_session_usage.up.sql => 000592_template_usage_stats_session_usage.up.sql} (100%) create mode 100644 coderd/database/session_usage_digest_test.go diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql b/coderd/database/migrations/000592_template_usage_stats_session_usage.down.sql similarity index 100% rename from coderd/database/migrations/000591_template_usage_stats_session_usage.down.sql rename to coderd/database/migrations/000592_template_usage_stats_session_usage.down.sql diff --git a/coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql similarity index 100% rename from coderd/database/migrations/000591_template_usage_stats_session_usage.up.sql rename to coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql diff --git a/coderd/database/migrations/migration000591_test.go b/coderd/database/migrations/migration000592_test.go similarity index 93% rename from coderd/database/migrations/migration000591_test.go rename to coderd/database/migrations/migration000592_test.go index e6ab1cbdf03..5d02bc444e2 100644 --- a/coderd/database/migrations/migration000591_test.go +++ b/coderd/database/migrations/migration000592_test.go @@ -55,20 +55,20 @@ func sessionRows(t *testing.T, tx *sql.Tx, table, nameColumn string) []familyRow return got } -// TestMigration000591TemplateUsageStatsSessionUsage covers the conversion of +// TestMigration000592TemplateUsageStatsSessionUsage covers the conversion of // the fixed per-family minute columns into the family child table, which the // testdata/fixtures run does not reach: its template_usage_stats rows record // no session minutes, so the backfill matches zero rows in CI. // //nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. -func TestMigration000591TemplateUsageStatsSessionUsage(t *testing.T) { +func TestMigration000592TemplateUsageStatsSessionUsage(t *testing.T) { t.Parallel() sqlDB := testSQLDB(t) - stepTo(t, sqlDB, 590) + stepTo(t, sqlDB, 591) ctx := testutil.Context(t, testutil.WaitSuperLong) - migrationSQL, err := os.ReadFile("000591_template_usage_stats_session_usage.up.sql") + migrationSQL, err := os.ReadFile("000592_template_usage_stats_session_usage.up.sql") require.NoError(t, err) // insertUsageStats writes one row per minute set, keyed by // (ssh, sftp, reconnecting_pty, vscode, jetbrains). @@ -209,14 +209,14 @@ func TestMigration000591TemplateUsageStatsSessionUsage(t *testing.T) { }) } -// TestMigration000591ChainFrom589 walks the whole window this change spans, -// 589 up to 591 and back down to 589, with data present at every step. The -// isolated 591 tests start at 590, so they never see 590 converting raw +// TestMigration000592ChainFrom589 walks the whole window this change spans, +// 589 up to 592 and back down to 589, with data present at every step. The +// isolated 592 tests start at 591, so they never see 590 converting raw // session counts the rollup has not consumed, which is the state an upgrade // actually finds. // //nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. -func TestMigration000591ChainFrom589(t *testing.T) { +func TestMigration000592ChainFrom589(t *testing.T) { t.Parallel() sqlDB := testSQLDB(t) @@ -227,9 +227,9 @@ func TestMigration000591ChainFrom589(t *testing.T) { require.NoError(t, err) down590, err := os.ReadFile("000590_workspace_agent_session_counts.down.sql") require.NoError(t, err) - up591, err := os.ReadFile("000591_template_usage_stats_session_usage.up.sql") + up592, err := os.ReadFile("000592_template_usage_stats_session_usage.up.sql") require.NoError(t, err) - down591, err := os.ReadFile("000591_template_usage_stats_session_usage.down.sql") + down592, err := os.ReadFile("000592_template_usage_stats_session_usage.down.sql") require.NoError(t, err) // backlogHours spans more than a day, and two backlogged rows sit inside @@ -248,7 +248,7 @@ func TestMigration000591ChainFrom589(t *testing.T) { t.Cleanup(func() { _ = tx.Rollback() }) // One rolled-up half hour, so 590 has a watermark to measure the backlog - // against, and so 591 has a row whose fixed family minutes must convert. + // against, and so 592 has a row whose fixed family minutes must convert. _, err = tx.ExecContext(ctx, ` INSERT INTO template_usage_stats ( start_time, end_time, template_id, user_id, median_latency_ms, @@ -289,7 +289,7 @@ func TestMigration000591ChainFrom589(t *testing.T) { `, backlogHours) require.NoError(t, err) - for _, step := range []chainStep{{"590 up", up590}, {"591 up", up591}} { + for _, step := range []chainStep{{"590 up", up590}, {"592 up", up592}} { _, err = tx.ExecContext(ctx, string(step.sql)) require.NoError(t, err, "%s", step.name) } @@ -313,14 +313,14 @@ func TestMigration000591ChainFrom589(t *testing.T) { require.JSONEq(t, `{"vscode": 4, "ssh": 2}`, gotCounts[1], "backlogged, over a day old") require.JSONEq(t, `{"vscode": 2, "ssh": 1}`, gotCounts[2], "backlogged, recent") - // 591 converted the fixed family minutes and recorded no per-app usage. + // 592 converted the fixed family minutes and recorded no per-app usage. require.Equal(t, []familyRow{{"sftp", 2}, {"ssh", 3}, {"vscode", 4}}, sessionRows(t, tx, "template_usage_stats_session_families", "family")) require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_apps", "app_name")) - // Back down: 591 restores the fixed columns, then 590 restores the fixed + // Back down: 592 restores the fixed columns, then 590 restores the fixed // session counts, landing on the 589 schema. - for _, step := range []chainStep{{"591 down", down591}, {"590 down", down590}} { + for _, step := range []chainStep{{"592 down", down592}, {"590 down", down590}} { _, err = tx.ExecContext(ctx, string(step.sql)) require.NoError(t, err, "%s", step.name) } diff --git a/coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/testdata/fixtures/000592_template_usage_stats_session_usage.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000591_template_usage_stats_session_usage.up.sql rename to coderd/database/migrations/testdata/fixtures/000592_template_usage_stats_session_usage.up.sql diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 52c600719c2..59c4be8ec74 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -17691,7 +17691,9 @@ WITH ), session_digests AS ( -- A stable hash of the bucket's session usage: the ordered set of - -- (kind, name, minutes). It is carried on the main row so the upsert's + -- (kind, name, minutes). Names are length-prefixed so embedded + -- delimiters cannot make different row sets encode identically. + -- It is carried on the main row so the upsert's -- IS DISTINCT FROM guard fires when session usage changes, which is -- what lets the child writes below skip unchanged buckets. -- @@ -17704,7 +17706,7 @@ WITH template_id, user_id, hashtextextended(string_agg( - family_group || ':' || name || ':' || usage_mins, + family_group || ':' || length(name) || ':' || name || ':' || usage_mins, '|' ORDER BY family_group, name ), 0) AS digest FROM diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index 6a5b2a73e67..d0553aab829 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -805,7 +805,9 @@ WITH ), session_digests AS ( -- A stable hash of the bucket's session usage: the ordered set of - -- (kind, name, minutes). It is carried on the main row so the upsert's + -- (kind, name, minutes). Names are length-prefixed so embedded + -- delimiters cannot make different row sets encode identically. + -- It is carried on the main row so the upsert's -- IS DISTINCT FROM guard fires when session usage changes, which is -- what lets the child writes below skip unchanged buckets. -- @@ -818,7 +820,7 @@ WITH template_id, user_id, hashtextextended(string_agg( - family_group || ':' || name || ':' || usage_mins, + family_group || ':' || length(name) || ':' || name || ':' || usage_mins, '|' ORDER BY family_group, name ), 0) AS digest FROM diff --git a/coderd/database/session_usage_digest_test.go b/coderd/database/session_usage_digest_test.go new file mode 100644 index 00000000000..eb312448a13 --- /dev/null +++ b/coderd/database/session_usage_digest_test.go @@ -0,0 +1,68 @@ +package database_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" +) + +func TestSessionUsageDigestDistinguishesDelimitedNames(t *testing.T) { + t.Parallel() + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := context.Background() + start := dbtime.Now().Add(-time.Hour).Truncate(30 * time.Minute) + user, template := uuid.New(), uuid.New() + dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ + CreatedAt: start, UserID: user, TemplateID: template, + AgentID: uuid.New(), ConnectionCount: 1, + SessionCounts: dbgen.SessionCounts(t, map[string]int64{"app_a": 1, "app_b": 1}), + }) + + rollup := func(registry map[string]string) { + t.Helper() + mapping, err := json.Marshal(registry) + require.NoError(t, err) + require.NoError(t, db.UpsertTemplateUsageStats(ctx, mapping)) + } + families := func() map[string]int64 { + return sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_families", "family", start, user, template) + } + + rollup(map[string]string{"app_a": "a", "app_b": "b"}) + require.Equal(t, map[string]int64{"a": 1, "b": 1}, families()) + before := sessionUsageDigest(ctx, t, sqlDB, start, user, template) + + // A registry change can merge two families without changing app usage or + // the main bucket totals. Delimiters in a family name must not make its + // digest identical to the two separate family rows it replaces. + registry := map[string]string{"app_a": "a:1|1:b", "app_b": "a:1|1:b"} + rollup(registry) + require.Equal(t, map[string]int64{"a:1|1:b": 1}, families()) + after := sessionUsageDigest(ctx, t, sqlDB, start, user, template) + require.NotEqual(t, before, after) + require.Equal(t, map[string]int64{"app_a": 1, "app_b": 1}, + sessionUsageMins(ctx, t, sqlDB, "template_usage_stats_session_apps", "app_name", start, user, template)) + + // Repeating the same attribution must not rewrite any child row. + var rowVersion string + require.NoError(t, sqlDB.QueryRowContext(ctx, + `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, + start, user, template).Scan(&rowVersion)) + rollup(registry) + var repeatedVersion string + require.NoError(t, sqlDB.QueryRowContext(ctx, + `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, + start, user, template).Scan(&repeatedVersion)) + require.Equal(t, rowVersion, repeatedVersion) + require.Equal(t, after, sessionUsageDigest(ctx, t, sqlDB, start, user, template)) +} From ada859638e81b01e33b784b23dba8e4fd78dd5ca Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Thu, 10 Sep 2026 16:20:17 +0000 Subject: [PATCH 4/7] refactor: consolidate session family decoders and tighten validation Generated by Coder Agents for @EhabY. --- coderd/database/dbauthz/dbauthz_test.go | 4 + coderd/database/dbauthz/sessioncountparams.go | 21 +++-- coderd/database/session_usage_digest_test.go | 8 +- coderd/insights.go | 93 +++++-------------- .../insights_session_family_internal_test.go | 26 ++---- .../insights/metricscollector.go | 20 +--- .../metricscollector_internal_test.go | 34 ++----- codersdk/appname.go | 21 ++++- codersdk/appname_test.go | 39 ++++++++ 9 files changed, 111 insertions(+), 155 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 352f008460b..6bab77a8a45 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -7880,6 +7880,10 @@ func TestSessionCountAppFamiliesShape(t *testing.T) { {"NotAnObject", json.RawMessage(`["vscode"]`), "must be a JSON object"}, {"FamilyToAppNames", json.RawMessage(`{"vscode":["cursor"]}`), "must be a JSON object"}, {"UnnormalizedAppName", json.RawMessage(`{"VSCode-Insiders":"vscode"}`), "is not normalized"}, + {"UnnormalizedFamily", json.RawMessage(`{"vscode":"VS Code"}`), "is not normalized"}, + {"HyphenatedFamily", json.RawMessage(`{"vscode":"vs-code"}`), "is not normalized"}, + {"PaddedFamily", json.RawMessage(`{"vscode":" vscode "}`), "is not normalized"}, + {"UnknownFamily", json.RawMessage(`{"vscode":"unknown"}`), `unknown family for app "vscode"`}, {"EmptyAppName", json.RawMessage(`{"":"vscode"}`), "empty app name"}, {"EmptyFamily", json.RawMessage(`{"vscode":""}`), `no family for app "vscode"`}, {"WhitespaceFamily", json.RawMessage(`{"vscode":" "}`), `no family for app "vscode"`}, diff --git a/coderd/database/dbauthz/sessioncountparams.go b/coderd/database/dbauthz/sessioncountparams.go index 71aa8ed34e7..1b2a191b4c0 100644 --- a/coderd/database/dbauthz/sessioncountparams.go +++ b/coderd/database/dbauthz/sessioncountparams.go @@ -28,17 +28,14 @@ import ( // defined outside that file. // validateSessionCountAppFamilies checks that the registry is a non-empty -// jsonb object of normalized app name to family name. It deliberately does -// not check which families appear: no query names a family, so a registry -// entry for a new family is valid without any SQL change. +// jsonb object of normalized app names to normalized, non-unknown family +// names. New families are valid without any SQL change. func validateSessionCountAppFamilies(appFamilies json.RawMessage) error { - if len(appFamilies) == 0 { - return xerrors.New("developer error: session count app families must not be empty, populate them with codersdk.SessionCountAppFamiliesJSON()") - } - var families map[string]codersdk.AppFamilyName - if err := json.Unmarshal(appFamilies, &families); err != nil { - return xerrors.Errorf("developer error: session count app families must be a JSON object of app name to family, populate them with codersdk.SessionCountAppFamiliesJSON(): %w", err) + if len(appFamilies) > 0 { + if err := json.Unmarshal(appFamilies, &families); err != nil { + return xerrors.Errorf("developer error: session count app families must be a JSON object of app name to family, populate them with codersdk.SessionCountAppFamiliesJSON(): %w", err) + } } if len(families) == 0 { return xerrors.New("developer error: session count app families must not be empty, populate them with codersdk.SessionCountAppFamiliesJSON()") @@ -58,6 +55,12 @@ func validateSessionCountAppFamilies(appFamilies json.RawMessage) error { if strings.TrimSpace(string(family)) == "" { return xerrors.Errorf("developer error: session count app families has no family for app %q, so its sessions would be misattributed", appName) } + if normalized := codersdk.NormalizeAppName(string(family)); normalized != string(family) { + return xerrors.Errorf("developer error: session count app families family %q for app %q is not normalized, expected %q", family, appName, normalized) + } + if family == codersdk.AppFamilyUnknown { + return xerrors.Errorf("developer error: session count app families has unknown family for app %q, so its sessions would be misattributed", appName) + } } return nil } diff --git a/coderd/database/session_usage_digest_test.go b/coderd/database/session_usage_digest_test.go index eb312448a13..aa5c8be43b1 100644 --- a/coderd/database/session_usage_digest_test.go +++ b/coderd/database/session_usage_digest_test.go @@ -56,13 +56,13 @@ func TestSessionUsageDigestDistinguishesDelimitedNames(t *testing.T) { // Repeating the same attribution must not rewrite any child row. var rowVersion string require.NoError(t, sqlDB.QueryRowContext(ctx, - `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, - start, user, template).Scan(&rowVersion)) + `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3 AND family = $4`, + start, user, template, registry["app_a"]).Scan(&rowVersion)) rollup(registry) var repeatedVersion string require.NoError(t, sqlDB.QueryRowContext(ctx, - `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3`, - start, user, template).Scan(&repeatedVersion)) + `SELECT xmin::text FROM template_usage_stats_session_families WHERE start_time = $1 AND user_id = $2 AND template_id = $3 AND family = $4`, + start, user, template, registry["app_a"]).Scan(&repeatedVersion)) require.Equal(t, rowVersion, repeatedVersion) require.Equal(t, after, sessionUsageDigest(ctx, t, sqlDB, start, user, template)) } diff --git a/coderd/insights.go b/coderd/insights.go index ad24f007664..a24ba697238 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -3,7 +3,6 @@ package coderd import ( "context" "database/sql" - "encoding/json" "fmt" "net/http" "slices" @@ -581,89 +580,43 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { // query does not report simply have no usage. const sessionFamilySFTP codersdk.AppFamilyName = "sftp" -// templateInsightsSessionFamilies is the decoded form of the per-session-family -// JSONB columns on database.GetTemplateInsightsRow. -type templateInsightsSessionFamilies struct { - usageSecondsByFamily map[codersdk.AppFamilyName]int64 - templateIDsByFamily map[codersdk.AppFamilyName][]uuid.UUID -} - -// usageSeconds returns the usage seconds reported for a session family. A -// family the query did not report had no usage. -func (f templateInsightsSessionFamilies) usageSeconds(family codersdk.AppFamilyName) int64 { - return f.usageSecondsByFamily[family] -} - -// templateIDs returns the templates that reported usage for a session family. -// The result is never nil so that the API keeps serializing an empty list -// instead of null. -func (f templateInsightsSessionFamilies) templateIDs(family codersdk.AppFamilyName) []uuid.UUID { - if ids := f.templateIDsByFamily[family]; ids != nil { - return ids - } - return []uuid.UUID{} -} - -// decodeTemplateInsightsSessionFamilies decodes the JSONB session family -// columns of a template insights row. -func decodeTemplateInsightsSessionFamilies(usage database.GetTemplateInsightsRow) (templateInsightsSessionFamilies, error) { - usageSeconds, err := decodeSessionFamilyMap[int64](usage.SessionFamilyUsageSeconds) - if err != nil { - return templateInsightsSessionFamilies{}, xerrors.Errorf("decode session family usage seconds: %w", err) - } - templateIDs, err := decodeSessionFamilyMap[[]uuid.UUID](usage.SessionFamilyTemplateIds) - if err != nil { - return templateInsightsSessionFamilies{}, xerrors.Errorf("decode session family template ids: %w", err) - } - return templateInsightsSessionFamilies{ - usageSecondsByFamily: usageSeconds, - templateIDsByFamily: templateIDs, - }, nil -} - -// decodeSessionFamilyMap decodes a JSONB payload keyed by session family. An -// absent payload decodes to an empty map, but a malformed one is an error so -// that callers report the failure instead of reporting zero usage. -func decodeSessionFamilyMap[V any](raw json.RawMessage) (map[codersdk.AppFamilyName]V, error) { - if len(raw) == 0 { - return map[codersdk.AppFamilyName]V{}, nil - } - var decoded map[codersdk.AppFamilyName]V - if err := json.Unmarshal(raw, &decoded); err != nil { - return nil, xerrors.Errorf("unmarshal session family map: %w", err) - } - if decoded == nil { - return map[codersdk.AppFamilyName]V{}, nil - } - return decoded, nil -} - // convertTemplateInsightsApps builds the list of builtin apps and template apps // from the provided database rows, builtin apps are implicitly a part of all // templates. func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) ([]codersdk.TemplateAppUsage, error) { - families, err := decodeTemplateInsightsSessionFamilies(usage) + usageSeconds, err := codersdk.DecodeAppFamilyMap[int64](usage.SessionFamilyUsageSeconds) + if err != nil { + return nil, xerrors.Errorf("convert template insights apps: decode session family usage seconds: %w", err) + } + templateIDsByFamily, err := codersdk.DecodeAppFamilyMap[[]uuid.UUID](usage.SessionFamilyTemplateIds) if err != nil { - return nil, xerrors.Errorf("convert template insights apps: %w", err) + return nil, xerrors.Errorf("convert template insights apps: decode session family template ids: %w", err) + } + // Keep serializing empty template lists as [] instead of null. + templateIDs := func(family codersdk.AppFamilyName) []uuid.UUID { + if ids := templateIDsByFamily[family]; ids != nil { + return ids + } + return []uuid.UUID{} } // Builtin apps. apps := []codersdk.TemplateAppUsage{ { - TemplateIDs: families.templateIDs(codersdk.AppFamilyVSCode), + TemplateIDs: templateIDs(codersdk.AppFamilyVSCode), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameVSCode, Slug: "vscode", Icon: "/icon/code.svg", - Seconds: families.usageSeconds(codersdk.AppFamilyVSCode), + Seconds: usageSeconds[codersdk.AppFamilyVSCode], }, { - TemplateIDs: families.templateIDs(codersdk.AppFamilyJetBrains), + TemplateIDs: templateIDs(codersdk.AppFamilyJetBrains), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameJetBrains, Slug: "jetbrains", Icon: "/icon/intellij.svg", - Seconds: families.usageSeconds(codersdk.AppFamilyJetBrains), + Seconds: usageSeconds[codersdk.AppFamilyJetBrains], }, // TODO(mafredri): We could take Web Terminal usage from appUsage since // that should be more accurate. The difference is that this reflects @@ -672,28 +625,28 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage // condition finding the corresponding app entry in appUsage is: // !app.IsApp && app.AccessMethod == "terminal" && app.SlugOrPort == "" { - TemplateIDs: families.templateIDs(codersdk.AppFamilyReconnectingPTY), + TemplateIDs: templateIDs(codersdk.AppFamilyReconnectingPTY), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameWebTerminal, Slug: "reconnecting-pty", Icon: "/icon/terminal.svg", - Seconds: families.usageSeconds(codersdk.AppFamilyReconnectingPTY), + Seconds: usageSeconds[codersdk.AppFamilyReconnectingPTY], }, { - TemplateIDs: families.templateIDs(codersdk.AppFamilySSH), + TemplateIDs: templateIDs(codersdk.AppFamilySSH), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameSSH, Slug: "ssh", Icon: "/icon/terminal.svg", - Seconds: families.usageSeconds(codersdk.AppFamilySSH), + Seconds: usageSeconds[codersdk.AppFamilySSH], }, { - TemplateIDs: families.templateIDs(sessionFamilySFTP), + TemplateIDs: templateIDs(sessionFamilySFTP), Type: codersdk.TemplateAppsTypeBuiltin, DisplayName: codersdk.TemplateBuiltinAppDisplayNameSFTP, Slug: "sftp", Icon: "/icon/terminal.svg", - Seconds: families.usageSeconds(sessionFamilySFTP), + Seconds: usageSeconds[sessionFamilySFTP], }, } diff --git a/coderd/insights_session_family_internal_test.go b/coderd/insights_session_family_internal_test.go index d50e80a9d09..50392c2214a 100644 --- a/coderd/insights_session_family_internal_test.go +++ b/coderd/insights_session_family_internal_test.go @@ -11,25 +11,6 @@ import ( "github.com/coder/coder/v2/codersdk" ) -func TestDecodeSessionFamilyMap(t *testing.T) { - t.Parallel() - - for name, raw := range map[string]json.RawMessage{ - "NotJSON": json.RawMessage(`{`), - "NotAnObject": json.RawMessage(`[1, 2]`), - "WrongValue": json.RawMessage(`{"vscode": "sixty"}`), - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - // A malformed payload must not decode to zero usage, or an - // encoding bug would look like an idle deployment. - _, err := decodeSessionFamilyMap[int64](raw) - require.Error(t, err) - }) - } -} - func TestConvertTemplateInsightsApps(t *testing.T) { t.Parallel() @@ -42,6 +23,11 @@ func TestConvertTemplateInsightsApps(t *testing.T) { SessionFamilyTemplateIds: json.RawMessage(`{"sftp": ["` + sftpTemplateID.String() + `"]}`), }, nil) require.NoError(t, err) + for _, app := range apps { + if app.Slug != "sftp" { + require.Equal(t, []uuid.UUID{}, app.TemplateIDs) + } + } require.Contains(t, apps, codersdk.TemplateAppUsage{ // The rollup no longer produces SFTP usage, but rows migrated // from the old sftp_mins column still report it. @@ -71,7 +57,7 @@ func TestConvertTemplateInsightsApps(t *testing.T) { t.Parallel() apps, err := convertTemplateInsightsApps(usage, nil) - require.Error(t, err) + require.ErrorContains(t, err, "convert template insights apps: decode session family") require.Nil(t, apps) }) } diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go index 924848e7f88..d97b90abb38 100644 --- a/coderd/prometheusmetrics/insights/metricscollector.go +++ b/coderd/prometheusmetrics/insights/metricscollector.go @@ -2,7 +2,6 @@ package insights import ( "context" - "encoding/json" "slices" "sync/atomic" "time" @@ -325,7 +324,7 @@ func onlyTemplateNames(templates []database.Template) map[uuid.UUID]string { func convertTemplateInsights(rows []database.GetTemplateInsightsByTemplateRow) ([]templateInsightsRow, error) { converted := make([]templateInsightsRow, 0, len(rows)) for _, row := range rows { - usageSeconds, err := decodeSessionFamilyUsageSeconds(row.SessionFamilyUsageSeconds) + usageSeconds, err := codersdk.DecodeAppFamilyMap[int64](row.SessionFamilyUsageSeconds) if err != nil { return nil, xerrors.Errorf("template %s: %w", row.TemplateID, err) } @@ -338,23 +337,6 @@ func convertTemplateInsights(rows []database.GetTemplateInsightsByTemplateRow) ( return converted, nil } -// decodeSessionFamilyUsageSeconds decodes a usage seconds JSONB payload keyed -// by session family. An absent payload decodes to an empty map, but a -// malformed one is an error. -func decodeSessionFamilyUsageSeconds(raw json.RawMessage) (map[codersdk.AppFamilyName]int64, error) { - if len(raw) == 0 { - return map[codersdk.AppFamilyName]int64{}, nil - } - var decoded map[codersdk.AppFamilyName]int64 - if err := json.Unmarshal(raw, &decoded); err != nil { - return nil, xerrors.Errorf("unmarshal session family usage seconds: %w", err) - } - if decoded == nil { - return map[codersdk.AppFamilyName]int64{}, nil - } - return decoded, nil -} - func convertParameterInsights(rows []database.GetTemplateParameterInsightsRow) []parameterRow { type uniqueKey struct { templateID uuid.UUID diff --git a/coderd/prometheusmetrics/insights/metricscollector_internal_test.go b/coderd/prometheusmetrics/insights/metricscollector_internal_test.go index 4a22858b8b0..a462f86f029 100644 --- a/coderd/prometheusmetrics/insights/metricscollector_internal_test.go +++ b/coderd/prometheusmetrics/insights/metricscollector_internal_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/codersdk" ) func TestConvertTemplateInsights(t *testing.T) { @@ -17,34 +16,13 @@ func TestConvertTemplateInsights(t *testing.T) { t.Run("Malformed", func(t *testing.T) { t.Parallel() - for name, raw := range map[string]json.RawMessage{ - "NotJSON": json.RawMessage(`{`), - "NotAnObject": json.RawMessage(`[1, 2]`), - "WrongValue": json.RawMessage(`{"ssh": "sixty"}`), - } { - t.Run(name, func(t *testing.T) { - t.Parallel() - - // Reporting zero usage would look like an idle template, so - // the collector must fail the tick instead. - rows, err := convertTemplateInsights([]database.GetTemplateInsightsByTemplateRow{ - {TemplateID: uuid.New(), SessionFamilyUsageSeconds: raw}, - }) - require.Error(t, err) - require.Nil(t, rows) - }) - } - }) - - t.Run("AbsentPayload", func(t *testing.T) { - t.Parallel() - + // Reporting zero usage would look like an idle template, so the + // collector must fail the tick instead. + templateID := uuid.New() rows, err := convertTemplateInsights([]database.GetTemplateInsightsByTemplateRow{ - {TemplateID: uuid.New()}, + {TemplateID: templateID, SessionFamilyUsageSeconds: json.RawMessage(`{"ssh": "sixty"}`)}, }) - require.NoError(t, err) - require.Len(t, rows, 1) - require.NotNil(t, rows[0].usageSecondsByFamily) - require.EqualValues(t, 0, rows[0].usageSeconds(codersdk.AppFamilySSH)) + require.ErrorContains(t, err, "template "+templateID.String()) + require.Nil(t, rows) }) } diff --git a/codersdk/appname.go b/codersdk/appname.go index 206643d6ba6..d98f3a3a7e2 100644 --- a/codersdk/appname.go +++ b/codersdk/appname.go @@ -2,6 +2,7 @@ package codersdk import ( "encoding/json" + "maps" "strings" "golang.org/x/xerrors" @@ -60,11 +61,7 @@ var appNameFamilies = map[string]AppFamilyName{ // that need a fixed family value derive it from this map, so registering a // new app or family means editing appNameFamilies alone. func SessionCountAppFamilies() map[string]AppFamilyName { - families := make(map[string]AppFamilyName, len(appNameFamilies)) - for appName, family := range appNameFamilies { - families[appName] = family - } - return families + return maps.Clone(appNameFamilies) } // SessionCountAppFamiliesJSON is SessionCountAppFamilies marshaled as the @@ -131,3 +128,17 @@ func NormalizeAppName(appName string) string { } return strings.ReplaceAll(strings.ToLower(appName), "-", "_") } + +// DecodeAppFamilyMap decodes a JSONB payload keyed by app family. An absent +// payload decodes to an empty map, but a malformed one is an error so that +// callers report the failure instead of reporting zero usage. +func DecodeAppFamilyMap[V any](raw json.RawMessage) (map[AppFamilyName]V, error) { + if len(raw) == 0 { + return map[AppFamilyName]V{}, nil + } + var decoded map[AppFamilyName]V + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, xerrors.Errorf("unmarshal session family map: %w", err) + } + return decoded, nil +} diff --git a/codersdk/appname_test.go b/codersdk/appname_test.go index 2ac573c8c86..6cbdb26249a 100644 --- a/codersdk/appname_test.go +++ b/codersdk/appname_test.go @@ -268,3 +268,42 @@ func TestSessionCountsByFamilyJSONMalformed(t *testing.T) { }) } } + +func TestDecodeAppFamilyMap(t *testing.T) { + t.Parallel() + + for name, raw := range map[string]json.RawMessage{ + "NotJSON": json.RawMessage(`{`), + "NotAnObject": json.RawMessage(`[1, 2]`), + "WrongValue": json.RawMessage(`{"vscode": "sixty"}`), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + // A malformed payload must not decode to zero usage, or an + // encoding bug would look like an idle deployment. + got, err := codersdk.DecodeAppFamilyMap[int64](raw) + require.Error(t, err) + require.Nil(t, got) + }) + } + + t.Run("UsageSeconds", func(t *testing.T) { + t.Parallel() + + got, err := codersdk.DecodeAppFamilyMap[int64](json.RawMessage(`{"vscode": 60}`)) + require.NoError(t, err) + require.Equal(t, map[codersdk.AppFamilyName]int64{codersdk.AppFamilyVSCode: 60}, got) + }) + + for name, raw := range map[string]json.RawMessage{"Absent": nil, "EmptyObject": json.RawMessage(`{}`)} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got, err := codersdk.DecodeAppFamilyMap[int64](raw) + require.NoError(t, err) + require.Equal(t, map[codersdk.AppFamilyName]int64{}, got) + require.Zero(t, got[codersdk.AppFamilySSH]) + }) + } +} From c5511e50ec1e084c5d47846784dfd9bc30c3d61e Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Thu, 10 Sep 2026 16:40:13 +0000 Subject: [PATCH 5/7] docs: tighten comments --- coderd/database/dbauthz/sessioncountparams.go | 20 ++++------- coderd/database/dbrollup/dbrollup_test.go | 6 ++-- ..._template_usage_stats_session_usage.up.sql | 19 +++++----- coderd/database/queries.sql.go | 35 +++++++++---------- coderd/database/queries/insights.sql | 35 +++++++++---------- coderd/insights.go | 8 ++--- 6 files changed, 55 insertions(+), 68 deletions(-) diff --git a/coderd/database/dbauthz/sessioncountparams.go b/coderd/database/dbauthz/sessioncountparams.go index 1b2a191b4c0..435b14cb82d 100644 --- a/coderd/database/dbauthz/sessioncountparams.go +++ b/coderd/database/dbauthz/sessioncountparams.go @@ -13,19 +13,13 @@ import ( "github.com/coder/coder/v2/codersdk" ) -// The minute aggregation queries take the app-to-family attribution registry -// as a single jsonb parameter and join it by app name, falling back to the -// unknown family for any app the registry does not cover. A registry that is -// empty, malformed, or keyed by something other than normalized app names -// therefore still runs and still totals every session: it misattributes known -// activity to the unknown family instead. The raw per-app data survives, so -// nothing is lost, but the fixed per-family compatibility fields reported to -// insights, Prometheus, and telemetry undercount for as long as it goes -// unnoticed. dbauthz wraps every production store, including the transaction -// stores used by the rollup, so validating here makes a wrong registry fail -// the call loudly instead of quietly skewing the attribution. These methods -// override the generated ones in dbauthz.go; scripts/dbgen preserves methods -// defined outside that file. +// The minute aggregation queries take the app-to-family registry as jsonb and +// fall back to the unknown family for unregistered apps, so a bad registry +// still succeeds while silently misattributing usage. dbauthz wraps every +// production store, including the rollup's transaction stores, so validating +// here fails the call loudly instead. These methods override the generated +// ones in dbauthz.go; scripts/dbgen preserves methods defined outside that +// file. // validateSessionCountAppFamilies checks that the registry is a non-empty // jsonb object of normalized app names to normalized, non-unknown family diff --git a/coderd/database/dbrollup/dbrollup_test.go b/coderd/database/dbrollup/dbrollup_test.go index e749ab4e9d8..ebcbbd13618 100644 --- a/coderd/database/dbrollup/dbrollup_test.go +++ b/coderd/database/dbrollup/dbrollup_test.go @@ -244,8 +244,7 @@ func TestRollupTemplateUsageStats(t *testing.T) { stats[0].EndTime = stats[0].EndTime.UTC() stats[0].StartTime = stats[0].StartTime.UTC() - // The digest is a hash of the child rows; its value is not worth pinning, - // only that the rollup recorded one. + // The digest value isn't pinned, just that the rollup recorded one. require.True(t, stats[0].SessionUsageDigest.Valid, "the rollup must record a session usage digest") require.NotZero(t, stats[0].SessionUsageDigest.Int64) stats[0].SessionUsageDigest = sql.NullInt64{} @@ -262,8 +261,7 @@ func TestRollupTemplateUsageStats(t *testing.T) { }, }, stats[0]) - // Session minutes live in the child tables, keyed by the reported app - // name and by its family. + // Session minutes live in the child tables, keyed by app name and family. for _, tc := range []struct { table, nameColumn, name string }{ diff --git a/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql index 28244076e64..620821ab1d7 100644 --- a/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql +++ b/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql @@ -1,7 +1,7 @@ --- The primary keys put user_id before template_id: the insights read caps a --- user's minutes per half hour across templates, and looks up one user's rows --- in a half hour through this prefix. The upsert's conflict target names the --- same columns in the parent's order, which the unique index satisfies. +-- The primary keys put user_id before template_id so the insights read, which +-- caps a user's minutes per half hour across templates, can look up one user's +-- rows through that prefix. The upsert's conflict target names the same +-- columns in the parent's order, satisfied by the unique index. CREATE TABLE template_usage_stats_session_families ( start_time timestamptz NOT NULL, template_id uuid NOT NULL, @@ -45,12 +45,11 @@ ALTER TABLE template_usage_stats COMMENT ON COLUMN template_usage_stats.session_usage_digest IS 'Hash of the bucket''s session usage rows in both child tables, so a rollup that recomputes an unchanged bucket rewrites no child rows. Null for buckets rolled up before the column existed, which reads as changed.'; --- Carry every family the fixed columns recorded, sftp included: the rollup has --- never written it, but a row that has a value must not lose it. Zero minutes --- are skipped so a bucket has rows only for the families it saw, which is what --- the rollup writes from now on. No app rows are written: the fixed columns --- only ever recorded the family, so per-app usage stays unknown for these --- buckets rather than being invented from family totals. +-- Carry every family the fixed columns recorded, sftp included: the rollup +-- never writes it, but an existing value must not be lost. Zero minutes are +-- skipped, matching what the rollup writes from now on. The fixed columns only +-- recorded families, so per-app usage stays unknown for these buckets rather +-- than being invented from family totals. INSERT INTO template_usage_stats_session_families (start_time, template_id, user_id, family, usage_mins) SELECT tus.start_time, diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 59c4be8ec74..392e742592f 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -16646,10 +16646,9 @@ func (q *sqlQuerier) GetTemplateAppInsightsByTemplate(ctx context.Context, arg G const getTemplateInsights = `-- name: GetTemplateInsights :one WITH base AS MATERIALIZED ( - -- One pass over the main table answers three questions: how many - -- templates each user touched in a half hour, that user's capped - -- minutes, and the list of templates in the window. GROUPING marks - -- which of the two sets a row belongs to. + -- One pass computes each user's template count and capped minutes + -- per half hour, plus the templates in the window. GROUPING + -- distinguishes user rows from template rows. SELECT GROUPING(template_id) AS is_user_row, start_time, @@ -16688,9 +16687,9 @@ WITH templates > 1 ), single_family_usage AS ( - -- Everything but the multi-template buckets, which cannot exceed the - -- cap, so they need no per-user grouping. This also collects the - -- template list per family, for which the cap is irrelevant. + -- Everything but the multi-template buckets. These cannot exceed the + -- cap, so they need no per-user grouping. The template list per + -- family comes from here too, where the cap is irrelevant. SELECT sessions.family, sessions.template_id, @@ -17692,10 +17691,10 @@ WITH session_digests AS ( -- A stable hash of the bucket's session usage: the ordered set of -- (kind, name, minutes). Names are length-prefixed so embedded - -- delimiters cannot make different row sets encode identically. - -- It is carried on the main row so the upsert's - -- IS DISTINCT FROM guard fires when session usage changes, which is - -- what lets the child writes below skip unchanged buckets. + -- delimiters cannot make different row sets encode identically. The + -- main row stores it, so the upsert's IS DISTINCT FROM guard fires on + -- session usage changes and the child writes below skip unchanged + -- buckets. -- -- INVARIANT: the digest must cover every column the child tables store. -- A column added to either table but left out of the digest would leave @@ -17862,13 +17861,13 @@ WITH -- the delete matches names the recomputed bucket no longer has, the insert -- only the names it does have. -- - -- The deletes test membership with NOT IN rather than NOT EXISTS on purpose. - -- The planner has no statistics for the recomputed CTE and estimates it at - -- a few rows, which turns NOT EXISTS into a nested loop that rescans the - -- CTE per candidate row, measured at 20 seconds per rollup. NOT IN is - -- planned as a hashed subplan built once, whatever the estimate. Every - -- column in the subquery is non-null, so the two forms delete the same - -- rows; the IS NOT NULL guard keeps that true if the CTE ever changes. + -- The deletes use NOT IN rather than NOT EXISTS on purpose. The planner + -- has no statistics for the recomputed CTE and estimates a few rows, + -- making NOT EXISTS a nested loop that rescans the CTE per candidate row: + -- 20 seconds per rollup. NOT IN is planned as a hashed subplan built + -- once, whatever the estimate. Every column in the subquery is non-null, + -- so both forms delete the same rows; the IS NOT NULL guard keeps that + -- true if the CTE changes. delete_families AS ( DELETE FROM template_usage_stats_session_families AS families diff --git a/coderd/database/queries/insights.sql b/coderd/database/queries/insights.sql index d0553aab829..8b8c038025c 100644 --- a/coderd/database/queries/insights.sql +++ b/coderd/database/queries/insights.sql @@ -97,10 +97,9 @@ ORDER BY -- 30 minutes with LEAST(SUM(n), 30). WITH base AS MATERIALIZED ( - -- One pass over the main table answers three questions: how many - -- templates each user touched in a half hour, that user's capped - -- minutes, and the list of templates in the window. GROUPING marks - -- which of the two sets a row belongs to. + -- One pass computes each user's template count and capped minutes + -- per half hour, plus the templates in the window. GROUPING + -- distinguishes user rows from template rows. SELECT GROUPING(template_id) AS is_user_row, start_time, @@ -139,9 +138,9 @@ WITH templates > 1 ), single_family_usage AS ( - -- Everything but the multi-template buckets, which cannot exceed the - -- cap, so they need no per-user grouping. This also collects the - -- template list per family, for which the cap is irrelevant. + -- Everything but the multi-template buckets. These cannot exceed the + -- cap, so they need no per-user grouping. The template list per + -- family comes from here too, where the cap is irrelevant. SELECT sessions.family, sessions.template_id, @@ -806,10 +805,10 @@ WITH session_digests AS ( -- A stable hash of the bucket's session usage: the ordered set of -- (kind, name, minutes). Names are length-prefixed so embedded - -- delimiters cannot make different row sets encode identically. - -- It is carried on the main row so the upsert's - -- IS DISTINCT FROM guard fires when session usage changes, which is - -- what lets the child writes below skip unchanged buckets. + -- delimiters cannot make different row sets encode identically. The + -- main row stores it, so the upsert's IS DISTINCT FROM guard fires on + -- session usage changes and the child writes below skip unchanged + -- buckets. -- -- INVARIANT: the digest must cover every column the child tables store. -- A column added to either table but left out of the digest would leave @@ -976,13 +975,13 @@ WITH -- the delete matches names the recomputed bucket no longer has, the insert -- only the names it does have. -- - -- The deletes test membership with NOT IN rather than NOT EXISTS on purpose. - -- The planner has no statistics for the recomputed CTE and estimates it at - -- a few rows, which turns NOT EXISTS into a nested loop that rescans the - -- CTE per candidate row, measured at 20 seconds per rollup. NOT IN is - -- planned as a hashed subplan built once, whatever the estimate. Every - -- column in the subquery is non-null, so the two forms delete the same - -- rows; the IS NOT NULL guard keeps that true if the CTE ever changes. + -- The deletes use NOT IN rather than NOT EXISTS on purpose. The planner + -- has no statistics for the recomputed CTE and estimates a few rows, + -- making NOT EXISTS a nested loop that rescans the CTE per candidate row: + -- 20 seconds per rollup. NOT IN is planned as a hashed subplan built + -- once, whatever the estimate. Every column in the subquery is non-null, + -- so both forms delete the same rows; the IS NOT NULL guard keeps that + -- true if the CTE changes. delete_families AS ( DELETE FROM template_usage_stats_session_families AS families diff --git a/coderd/insights.go b/coderd/insights.go index a24ba697238..c94913bbd5c 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -573,11 +573,9 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, resp) } -// sessionFamilySFTP is the session family the insights queries report SFTP -// usage under. No app name maps to it, so nothing in the app family registry -// produces it, but template_usage_stats carries historical sftp minutes under -// this key and the API has always exposed them as a builtin app. Families the -// query does not report simply have no usage. +// sessionFamilySFTP is the family historical sftp minutes are stored under. No +// app name maps to it, so the registry never produces it, but the API has +// always exposed sftp as a builtin app. const sessionFamilySFTP codersdk.AppFamilyName = "sftp" // convertTemplateInsightsApps builds the list of builtin apps and template apps From a9c1629855759c1bd26ad19c7427cd60bea8e2e5 Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Thu, 10 Sep 2026 16:46:16 +0000 Subject: [PATCH 6/7] refactor: tighten error messages --- coderd/database/dbauthz/dbauthz_test.go | 22 +++++++++---------- coderd/database/dbauthz/sessioncountparams.go | 14 ++++++------ coderd/insights.go | 4 ++-- .../insights_session_family_internal_test.go | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 6bab77a8a45..928b9e696b4 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -7857,9 +7857,9 @@ func TestSessionCountAppFamiliesRequired(t *testing.T) { ctx := dbauthz.As(context.Background(), coderdtest.RandomRBACSubject()) _, err := q.GetTemplateInsightsByTemplate(ctx, database.GetTemplateInsightsByTemplateParams{}) - require.ErrorContains(t, err, "developer error") + require.ErrorContains(t, err, "app family registry is empty") err = q.UpsertTemplateUsageStats(ctx, nil) - require.ErrorContains(t, err, "developer error") + require.ErrorContains(t, err, "app family registry is empty") } // TestSessionCountAppFamiliesShape covers registries that are present but @@ -7875,15 +7875,15 @@ func TestSessionCountAppFamiliesShape(t *testing.T) { appFamilies json.RawMessage errContains string }{ - {"EmptyObject", json.RawMessage(`{}`), "must not be empty"}, - {"JSONNull", json.RawMessage(`null`), "must not be empty"}, - {"NotAnObject", json.RawMessage(`["vscode"]`), "must be a JSON object"}, - {"FamilyToAppNames", json.RawMessage(`{"vscode":["cursor"]}`), "must be a JSON object"}, - {"UnnormalizedAppName", json.RawMessage(`{"VSCode-Insiders":"vscode"}`), "is not normalized"}, - {"UnnormalizedFamily", json.RawMessage(`{"vscode":"VS Code"}`), "is not normalized"}, - {"HyphenatedFamily", json.RawMessage(`{"vscode":"vs-code"}`), "is not normalized"}, - {"PaddedFamily", json.RawMessage(`{"vscode":" vscode "}`), "is not normalized"}, - {"UnknownFamily", json.RawMessage(`{"vscode":"unknown"}`), `unknown family for app "vscode"`}, + {"EmptyObject", json.RawMessage(`{}`), "app family registry is empty"}, + {"JSONNull", json.RawMessage(`null`), "app family registry is empty"}, + {"NotAnObject", json.RawMessage(`["vscode"]`), "invalid app family registry"}, + {"FamilyToAppNames", json.RawMessage(`{"vscode":["cursor"]}`), "invalid app family registry"}, + {"UnnormalizedAppName", json.RawMessage(`{"VSCode-Insiders":"vscode"}`), "not normalized"}, + {"UnnormalizedFamily", json.RawMessage(`{"vscode":"VS Code"}`), "not normalized"}, + {"HyphenatedFamily", json.RawMessage(`{"vscode":"vs-code"}`), "not normalized"}, + {"PaddedFamily", json.RawMessage(`{"vscode":" vscode "}`), "not normalized"}, + {"UnknownFamily", json.RawMessage(`{"vscode":"unknown"}`), `app "vscode" maps to unknown family`}, {"EmptyAppName", json.RawMessage(`{"":"vscode"}`), "empty app name"}, {"EmptyFamily", json.RawMessage(`{"vscode":""}`), `no family for app "vscode"`}, {"WhitespaceFamily", json.RawMessage(`{"vscode":" "}`), `no family for app "vscode"`}, diff --git a/coderd/database/dbauthz/sessioncountparams.go b/coderd/database/dbauthz/sessioncountparams.go index 435b14cb82d..e4fc0df542f 100644 --- a/coderd/database/dbauthz/sessioncountparams.go +++ b/coderd/database/dbauthz/sessioncountparams.go @@ -28,32 +28,32 @@ func validateSessionCountAppFamilies(appFamilies json.RawMessage) error { var families map[string]codersdk.AppFamilyName if len(appFamilies) > 0 { if err := json.Unmarshal(appFamilies, &families); err != nil { - return xerrors.Errorf("developer error: session count app families must be a JSON object of app name to family, populate them with codersdk.SessionCountAppFamiliesJSON(): %w", err) + return xerrors.Errorf("invalid app family registry: %w", err) } } if len(families) == 0 { - return xerrors.New("developer error: session count app families must not be empty, populate them with codersdk.SessionCountAppFamiliesJSON()") + return xerrors.New("app family registry is empty") } for appName, family := range families { if appName == "" { - return xerrors.New("developer error: session count app families has an empty app name, which no session can match") + return xerrors.New("empty app name") } // Stored app names are normalized, so an unnormalized key matches no // session and that app's activity falls back to the unknown family. if normalized := codersdk.NormalizeAppName(appName); normalized != appName { - return xerrors.Errorf("developer error: session count app families app name %q is not normalized, expected %q", appName, normalized) + return xerrors.Errorf("app name %q not normalized, want %q", appName, normalized) } // A blank family is not an attribution, so its apps would report under // no usable name at all. if strings.TrimSpace(string(family)) == "" { - return xerrors.Errorf("developer error: session count app families has no family for app %q, so its sessions would be misattributed", appName) + return xerrors.Errorf("no family for app %q", appName) } if normalized := codersdk.NormalizeAppName(string(family)); normalized != string(family) { - return xerrors.Errorf("developer error: session count app families family %q for app %q is not normalized, expected %q", family, appName, normalized) + return xerrors.Errorf("family %q for app %q not normalized, want %q", family, appName, normalized) } if family == codersdk.AppFamilyUnknown { - return xerrors.Errorf("developer error: session count app families has unknown family for app %q, so its sessions would be misattributed", appName) + return xerrors.Errorf("app %q maps to unknown family", appName) } } return nil diff --git a/coderd/insights.go b/coderd/insights.go index c94913bbd5c..3563f3450a6 100644 --- a/coderd/insights.go +++ b/coderd/insights.go @@ -584,11 +584,11 @@ const sessionFamilySFTP codersdk.AppFamilyName = "sftp" func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) ([]codersdk.TemplateAppUsage, error) { usageSeconds, err := codersdk.DecodeAppFamilyMap[int64](usage.SessionFamilyUsageSeconds) if err != nil { - return nil, xerrors.Errorf("convert template insights apps: decode session family usage seconds: %w", err) + return nil, xerrors.Errorf("decode session family usage seconds: %w", err) } templateIDsByFamily, err := codersdk.DecodeAppFamilyMap[[]uuid.UUID](usage.SessionFamilyTemplateIds) if err != nil { - return nil, xerrors.Errorf("convert template insights apps: decode session family template ids: %w", err) + return nil, xerrors.Errorf("decode session family template ids: %w", err) } // Keep serializing empty template lists as [] instead of null. templateIDs := func(family codersdk.AppFamilyName) []uuid.UUID { diff --git a/coderd/insights_session_family_internal_test.go b/coderd/insights_session_family_internal_test.go index 50392c2214a..736b7113fe6 100644 --- a/coderd/insights_session_family_internal_test.go +++ b/coderd/insights_session_family_internal_test.go @@ -57,7 +57,7 @@ func TestConvertTemplateInsightsApps(t *testing.T) { t.Parallel() apps, err := convertTemplateInsightsApps(usage, nil) - require.ErrorContains(t, err, "convert template insights apps: decode session family") + require.ErrorContains(t, err, "decode session family") require.Nil(t, apps) }) } From d17e5b7706ad4935f7bba111b253203906d4941c Mon Sep 17 00:00:00 2001 From: Ehab Younes Date: Mon, 14 Sep 2026 18:01:55 +0000 Subject: [PATCH 7/7] fix(coderd/database/migrations): renumber session usage migration Move the session usage migration and fixtures to 000595 after main's task removal migrations. Update migration tests to use the preceding schema and remove the stale fixture migration reference. Generated by Coder Agents for @EhabY. --- ...mplate_usage_stats_session_usage.down.sql} | 0 ...template_usage_stats_session_usage.up.sql} | 0 ...000592_test.go => migration000595_test.go} | 30 +++++++++---------- ...template_usage_stats_session_usage.up.sql} | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) rename coderd/database/migrations/{000592_template_usage_stats_session_usage.down.sql => 000595_template_usage_stats_session_usage.down.sql} (100%) rename coderd/database/migrations/{000592_template_usage_stats_session_usage.up.sql => 000595_template_usage_stats_session_usage.up.sql} (100%) rename coderd/database/migrations/{migration000592_test.go => migration000595_test.go} (93%) rename coderd/database/migrations/testdata/fixtures/{000592_template_usage_stats_session_usage.up.sql => 000595_template_usage_stats_session_usage.up.sql} (92%) diff --git a/coderd/database/migrations/000592_template_usage_stats_session_usage.down.sql b/coderd/database/migrations/000595_template_usage_stats_session_usage.down.sql similarity index 100% rename from coderd/database/migrations/000592_template_usage_stats_session_usage.down.sql rename to coderd/database/migrations/000595_template_usage_stats_session_usage.down.sql diff --git a/coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/000595_template_usage_stats_session_usage.up.sql similarity index 100% rename from coderd/database/migrations/000592_template_usage_stats_session_usage.up.sql rename to coderd/database/migrations/000595_template_usage_stats_session_usage.up.sql diff --git a/coderd/database/migrations/migration000592_test.go b/coderd/database/migrations/migration000595_test.go similarity index 93% rename from coderd/database/migrations/migration000592_test.go rename to coderd/database/migrations/migration000595_test.go index 5d02bc444e2..f418d6ed387 100644 --- a/coderd/database/migrations/migration000592_test.go +++ b/coderd/database/migrations/migration000595_test.go @@ -55,20 +55,20 @@ func sessionRows(t *testing.T, tx *sql.Tx, table, nameColumn string) []familyRow return got } -// TestMigration000592TemplateUsageStatsSessionUsage covers the conversion of +// TestMigration000595TemplateUsageStatsSessionUsage covers the conversion of // the fixed per-family minute columns into the family child table, which the // testdata/fixtures run does not reach: its template_usage_stats rows record // no session minutes, so the backfill matches zero rows in CI. // //nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. -func TestMigration000592TemplateUsageStatsSessionUsage(t *testing.T) { +func TestMigration000595TemplateUsageStatsSessionUsage(t *testing.T) { t.Parallel() sqlDB := testSQLDB(t) - stepTo(t, sqlDB, 591) + stepTo(t, sqlDB, 594) ctx := testutil.Context(t, testutil.WaitSuperLong) - migrationSQL, err := os.ReadFile("000592_template_usage_stats_session_usage.up.sql") + migrationSQL, err := os.ReadFile("000595_template_usage_stats_session_usage.up.sql") require.NoError(t, err) // insertUsageStats writes one row per minute set, keyed by // (ssh, sftp, reconnecting_pty, vscode, jetbrains). @@ -209,14 +209,14 @@ func TestMigration000592TemplateUsageStatsSessionUsage(t *testing.T) { }) } -// TestMigration000592ChainFrom589 walks the whole window this change spans, -// 589 up to 592 and back down to 589, with data present at every step. The -// isolated 592 tests start at 591, so they never see 590 converting raw +// TestMigration000595ChainFrom589 walks the whole window this change spans, +// 589 up to 595 and back down to 589, with data present at every step. The +// isolated 595 tests start at 594, so they never see 590 converting raw // session counts the rollup has not consumed, which is the state an upgrade // actually finds. // //nolint:tparallel,paralleltest // Subtests share one database with transaction-local fixtures. -func TestMigration000592ChainFrom589(t *testing.T) { +func TestMigration000595ChainFrom589(t *testing.T) { t.Parallel() sqlDB := testSQLDB(t) @@ -227,9 +227,9 @@ func TestMigration000592ChainFrom589(t *testing.T) { require.NoError(t, err) down590, err := os.ReadFile("000590_workspace_agent_session_counts.down.sql") require.NoError(t, err) - up592, err := os.ReadFile("000592_template_usage_stats_session_usage.up.sql") + up595, err := os.ReadFile("000595_template_usage_stats_session_usage.up.sql") require.NoError(t, err) - down592, err := os.ReadFile("000592_template_usage_stats_session_usage.down.sql") + down595, err := os.ReadFile("000595_template_usage_stats_session_usage.down.sql") require.NoError(t, err) // backlogHours spans more than a day, and two backlogged rows sit inside @@ -248,7 +248,7 @@ func TestMigration000592ChainFrom589(t *testing.T) { t.Cleanup(func() { _ = tx.Rollback() }) // One rolled-up half hour, so 590 has a watermark to measure the backlog - // against, and so 592 has a row whose fixed family minutes must convert. + // against, and so 595 has a row whose fixed family minutes must convert. _, err = tx.ExecContext(ctx, ` INSERT INTO template_usage_stats ( start_time, end_time, template_id, user_id, median_latency_ms, @@ -289,7 +289,7 @@ func TestMigration000592ChainFrom589(t *testing.T) { `, backlogHours) require.NoError(t, err) - for _, step := range []chainStep{{"590 up", up590}, {"592 up", up592}} { + for _, step := range []chainStep{{"590 up", up590}, {"595 up", up595}} { _, err = tx.ExecContext(ctx, string(step.sql)) require.NoError(t, err, "%s", step.name) } @@ -313,14 +313,14 @@ func TestMigration000592ChainFrom589(t *testing.T) { require.JSONEq(t, `{"vscode": 4, "ssh": 2}`, gotCounts[1], "backlogged, over a day old") require.JSONEq(t, `{"vscode": 2, "ssh": 1}`, gotCounts[2], "backlogged, recent") - // 592 converted the fixed family minutes and recorded no per-app usage. + // 595 converted the fixed family minutes and recorded no per-app usage. require.Equal(t, []familyRow{{"sftp", 2}, {"ssh", 3}, {"vscode", 4}}, sessionRows(t, tx, "template_usage_stats_session_families", "family")) require.Empty(t, sessionRows(t, tx, "template_usage_stats_session_apps", "app_name")) - // Back down: 592 restores the fixed columns, then 590 restores the fixed + // Back down: 595 restores the fixed columns, then 590 restores the fixed // session counts, landing on the 589 schema. - for _, step := range []chainStep{{"592 down", down592}, {"590 down", down590}} { + for _, step := range []chainStep{{"595 down", down595}, {"590 down", down590}} { _, err = tx.ExecContext(ctx, string(step.sql)) require.NoError(t, err, "%s", step.name) } diff --git a/coderd/database/migrations/testdata/fixtures/000592_template_usage_stats_session_usage.up.sql b/coderd/database/migrations/testdata/fixtures/000595_template_usage_stats_session_usage.up.sql similarity index 92% rename from coderd/database/migrations/testdata/fixtures/000592_template_usage_stats_session_usage.up.sql rename to coderd/database/migrations/testdata/fixtures/000595_template_usage_stats_session_usage.up.sql index f268426f33e..e23250ad3ad 100644 --- a/coderd/database/migrations/testdata/fixtures/000592_template_usage_stats_session_usage.up.sql +++ b/coderd/database/migrations/testdata/fixtures/000595_template_usage_stats_session_usage.up.sql @@ -1,4 +1,4 @@ --- The 000591 backfill produces family rows only, because the fixed columns it +-- The backfill produces family rows only, because the fixed columns it -- converts never recorded an app name. App rows exist only for buckets the -- rollup wrote after this migration, so the fixture seeds one directly. INSERT INTO template_usage_stats (