Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bda8f29

Browse files
DanielleMaywoodDanielleMaywood
committed
chore: add tests for unknown state on memory monitor
1 parent 7998f89 commit bda8f29

File tree

3 files changed

+114
-4
lines changed

3 files changed

+114
-4
lines changed

‎coderd/agentapi/resources_monitoring.go

+15-3
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
136136
debouncedUntil = a.Clock.Now().Add(a.Debounce)
137137
}
138138

139+
//nolint:gocritic // We need to be able to update the resource monitor here.
139140
err = a.Database.UpdateMemoryResourceMonitor(dbauthz.AsResourceMonitor(ctx), database.UpdateMemoryResourceMonitorParams{
140141
AgentID: a.AgentID,
141142
State: newState,
@@ -152,7 +153,7 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
152153
return xerrors.Errorf("get workspace by id: %w", err)
153154
}
154155

155-
_, err = a.NotificationsEnqueuer.Enqueue(
156+
_, err = a.NotificationsEnqueuer.EnqueueWithData(
156157
// nolint:gocritic // We need to be able to send the notification.
157158
dbauthz.AsNotifier(ctx),
158159
workspace.OwnerID,
@@ -161,6 +162,12 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
161162
"workspace": workspace.Name,
162163
"threshold": fmt.Sprintf("%d%%", monitor.Threshold),
163164
},
165+
map[string]any{
166+
// NOTE(DanielleMaywood):
167+
// We are injecting a timestamp to circumvent the notification
168+
// deduplication logic.
169+
"timestamp": a.Clock.Now(),
170+
},
164171
"workspace-monitor-memory",
165172
)
166173
if err != nil {
@@ -217,6 +224,7 @@ func (a *ResourcesMonitoringAPI) monitorVolumes(ctx context.Context, datapoints
217224
})
218225
}
219226

227+
//nolint:gocritic // We need to be able to update the resource monitor here.
220228
if err := a.Database.UpdateVolumeResourceMonitor(dbauthz.AsResourceMonitor(ctx), database.UpdateVolumeResourceMonitorParams{
221229
AgentID: a.AgentID,
222230
Path: monitor.Path,
@@ -244,6 +252,10 @@ func (a *ResourcesMonitoringAPI) monitorVolumes(ctx context.Context, datapoints
244252
},
245253
map[string]any{
246254
"volumes": outOfDiskVolumes,
255+
// NOTE(DanielleMaywood):
256+
// We are injecting a timestamp to circumvent the notification
257+
// deduplication logic.
258+
"timestamp": a.Clock.Now(),
247259
},
248260
"workspace-monitor-volumes",
249261
); err != nil {
@@ -269,9 +281,9 @@ func (a *ResourcesMonitoringAPI) calculateNextState(
269281
for _, state := range states {
270282
switch state {
271283
case resourcesmonitor.StateOK:
272-
okCount += 1
284+
okCount++
273285
case resourcesmonitor.StateNOK:
274-
nokCount += 1
286+
nokCount++
275287
}
276288
}
277289

‎coderd/agentapi/resources_monitoring_test.go

+98
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,104 @@ func TestMemoryResourceMonitor(t *testing.T) {
317317
}
318318
}
319319

320+
func TestMemoryResourceMonitorMissingData(t *testing.T) {
321+
t.Parallel()
322+
323+
t.Run("UnknownPreventsMovingIntoAlertState", func(t *testing.T) {
324+
t.Parallel()
325+
326+
api, _, clock, notifyEnq := resourceMonitorAPI(t)
327+
api.ConsecutiveNOKsToAlert = 2
328+
api.MinimumNOKsToAlert = 10
329+
330+
// Given: A monitor in an OK state.
331+
dbgen.WorkspaceAgentMemoryResourceMonitor(t, api.Database, database.WorkspaceAgentMemoryResourceMonitor{
332+
AgentID: api.AgentID,
333+
State: database.WorkspaceAgentMonitorStateOK,
334+
Threshold: 80,
335+
})
336+
337+
// When: A datapoint is missing, surrounded by two NOK datapoints.
338+
_, err := api.PushResourcesMonitoringUsage(context.Background(), &agentproto.PushResourcesMonitoringUsageRequest{
339+
Datapoints: []*agentproto.PushResourcesMonitoringUsageRequest_Datapoint{
340+
{
341+
CollectedAt: timestamppb.New(clock.Now()),
342+
Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
343+
Used: 10,
344+
Total: 10,
345+
},
346+
},
347+
{
348+
CollectedAt: timestamppb.New(clock.Now().Add(10 * time.Second)),
349+
Memory: nil,
350+
},
351+
{
352+
CollectedAt: timestamppb.New(clock.Now().Add(20 * time.Second)),
353+
Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
354+
Used: 10,
355+
Total: 10,
356+
},
357+
},
358+
},
359+
})
360+
require.NoError(t, err)
361+
362+
// Then: We expect no notifications, as this unknown prevents us knowing we should alert.
363+
sent := notifyEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateWorkspaceOutOfMemory))
364+
require.Len(t, sent, 0)
365+
366+
// Then: We expect the monitor to still be in an OK state.
367+
monitor, err := api.Database.FetchMemoryResourceMonitorsByAgentID(context.Background(), api.AgentID)
368+
require.NoError(t, err)
369+
require.Equal(t, database.WorkspaceAgentMonitorStateOK, monitor.State)
370+
})
371+
372+
t.Run("UnknownPreventsMovingOutOfAlertState", func(t *testing.T) {
373+
t.Parallel()
374+
375+
api, _, clock, _ := resourceMonitorAPI(t)
376+
api.ConsecutiveNOKsToAlert = 2
377+
api.MinimumNOKsToAlert = 10
378+
379+
// Given: A monitor in a NOK state.
380+
dbgen.WorkspaceAgentMemoryResourceMonitor(t, api.Database, database.WorkspaceAgentMemoryResourceMonitor{
381+
AgentID: api.AgentID,
382+
State: database.WorkspaceAgentMonitorStateNOK,
383+
Threshold: 80,
384+
})
385+
386+
// When: A datapoint is missing, surrounded by two OK datapoints.
387+
_, err := api.PushResourcesMonitoringUsage(context.Background(), &agentproto.PushResourcesMonitoringUsageRequest{
388+
Datapoints: []*agentproto.PushResourcesMonitoringUsageRequest_Datapoint{
389+
{
390+
CollectedAt: timestamppb.New(clock.Now()),
391+
Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
392+
Used: 1,
393+
Total: 10,
394+
},
395+
},
396+
{
397+
CollectedAt: timestamppb.New(clock.Now().Add(10 * time.Second)),
398+
Memory: nil,
399+
},
400+
{
401+
CollectedAt: timestamppb.New(clock.Now().Add(20 * time.Second)),
402+
Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
403+
Used: 1,
404+
Total: 10,
405+
},
406+
},
407+
},
408+
})
409+
require.NoError(t, err)
410+
411+
// Then: We expect the monitor to still be in a NOK state.
412+
monitor, err := api.Database.FetchMemoryResourceMonitorsByAgentID(context.Background(), api.AgentID)
413+
require.NoError(t, err)
414+
require.Equal(t, database.WorkspaceAgentMonitorStateNOK, monitor.State)
415+
})
416+
}
417+
320418
func TestVolumeResourceMonitorDebounce(t *testing.T) {
321419
t.Parallel()
322420

‎coderd/agentapi/resourcesmonitor/resources_monitor.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func CalculateConsecutiveNOK(states []State) int {
6969

7070
for _, state := range states {
7171
if state == StateNOK {
72-
curLength += 1
72+
curLength++
7373
} else {
7474
maxLength = max(maxLength, curLength)
7575
curLength = 0

0 commit comments

Comments
 (0)