diff --git a/coderd/coderd.go b/coderd/coderd.go index d64727567720d..4c7ff4fe081ff 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -628,7 +628,8 @@ func New(options *Options) *API { CurrentVersion: buildinfo.Version(), CurrentAPIMajorVersion: proto.CurrentMajor, Store: options.Database, - // TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go + StaleInterval: provisionerdserver.StaleInterval, + // TimeNow set to default, see healthcheck/provisioner.go }, }) } diff --git a/coderd/healthcheck/provisioner.go b/coderd/healthcheck/provisioner.go index 370a5ad04de86..ae3220170dd69 100644 --- a/coderd/healthcheck/provisioner.go +++ b/coderd/healthcheck/provisioner.go @@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae now := opts.TimeNow() if opts.StaleInterval == 0 { - opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3 + opts.StaleInterval = provisionerdserver.StaleInterval } if opts.CurrentVersion == "" { diff --git a/coderd/healthcheck/provisioner_test.go b/coderd/healthcheck/provisioner_test.go index 37530f9f8c747..93871f4a709ad 100644 --- a/coderd/healthcheck/provisioner_test.go +++ b/coderd/healthcheck/provisioner_test.go @@ -15,15 +15,21 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/healthcheck" "github.com/coder/coder/v2/coderd/healthcheck/health" + "github.com/coder/coder/v2/coderd/provisionerdserver" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/healthsdk" "github.com/coder/coder/v2/provisionerd/proto" + "github.com/coder/coder/v2/testutil" ) func TestProvisionerDaemonReport(t *testing.T) { t.Parallel() - now := dbtime.Now() + var ( + now = dbtime.Now() + oneHourAgo = now.Add(-time.Hour) + staleThreshold = now.Add(-provisionerdserver.StaleInterval).Add(-time.Second) + ) for _, tt := range []struct { name string @@ -65,7 +71,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentVersion: "v1.2.3", currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityOK, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -88,7 +96,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -116,7 +126,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeUnknown, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-invalid-version"), withVersion("invalid"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -144,7 +156,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeUnknown, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-api", "v1.2.3", "invalid", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-invalid-api"), withVersion("v1.2.3"), withAPIVersion("invalid"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -172,7 +186,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: 2, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonAPIMajorVersionDeprecated, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-old-api"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -200,7 +216,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -241,7 +260,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + fakeProvisionerDaemon(t, withName("pd-new"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -281,7 +303,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentVersion: "v2.3.4", currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityOK, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-stale", "v1.2.3", "0.9", now.Add(-5*time.Minute), now), fakeProvisionerDaemon(t, "pd-ok", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)), + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -304,8 +329,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeProvisionerDaemonsNoProvisionerDaemons, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", now.Add(-5*time.Minute), now)}, - expectedItems: []healthsdk.ProvisionerDaemonsReportItem{}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)), + }, + expectedItems: []healthsdk.ProvisionerDaemonsReportItem{}, }, } { tt := tt @@ -353,25 +380,52 @@ func TestProvisionerDaemonReport(t *testing.T) { } } -func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string, now time.Time) database.ProvisionerDaemon { +func withName(s string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.Name = s + } +} + +func withCreatedAt(at time.Time) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.CreatedAt = at + } +} + +func withLastSeenAt(at time.Time) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.LastSeenAt.Valid = true + pd.LastSeenAt.Time = at + } +} + +func withVersion(v string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.Version = v + } +} + +func withAPIVersion(v string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.APIVersion = v + } +} + +func fakeProvisionerDaemon(t *testing.T, opts ...func(*database.ProvisionerDaemon)) database.ProvisionerDaemon { t.Helper() - return database.ProvisionerDaemon{ + pd := database.ProvisionerDaemon{ ID: uuid.Nil, - Name: name, - CreatedAt: now, - LastSeenAt: sql.NullTime{Time: now, Valid: true}, + Name: testutil.GetRandomName(t), + CreatedAt: time.Time{}, + LastSeenAt: sql.NullTime{}, Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, ReplicaID: uuid.NullUUID{}, Tags: map[string]string{}, - Version: version, - APIVersion: apiVersion, + Version: "", + APIVersion: "", } -} - -func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt, now time.Time) database.ProvisionerDaemon { - t.Helper() - d := fakeProvisionerDaemon(t, name, version, apiVersion, now) - d.LastSeenAt.Valid = true - d.LastSeenAt.Time = lastSeenAt - return d + for _, o := range opts { + o(&pd) + } + return pd } diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 71847b0562d0b..0e9892b892172 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -57,6 +57,10 @@ const ( // DefaultHeartbeatInterval is the interval at which the provisioner daemon // will update its last seen at timestamp in the database. DefaultHeartbeatInterval = time.Minute + + // StaleInterval is the amount of time after the last heartbeat for which + // the provisioner will be reported as 'stale'. + StaleInterval = 90 * time.Second ) type Options struct { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index a0609c42c33f9..55f3191cc4c57 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1515,7 +1515,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // Check for eligible provisioners. This allows us to log a message warning deployment administrators // of users submitting jobs for which no provisioners are available. - matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) + matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags) if err != nil { api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) } else if matchedProvisioners.Count == 0 { @@ -1819,7 +1819,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID) } } -func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) { +func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) { // Check for eligible provisioners. This allows us to return a warning to the user if they // submit a job for which no provisioner is available. eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ @@ -1831,7 +1831,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err) } - threePollsAgo := time.Now().Add(-3 * pollInterval) + staleInterval := time.Now().Add(-provisionerdserver.StaleInterval) mostRecentlySeen := codersdk.NullTime{} var matched codersdk.MatchedProvisioners for _, provisioner := range eligibleProvisioners { @@ -1839,7 +1839,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI continue } matched.Count++ - if provisioner.LastSeenAt.Time.After(threePollsAgo) { + if provisioner.LastSeenAt.Time.After(staleInterval) { matched.Available++ } if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {