chore: add tests for unknown state on memory monitor

DanielleMaywood · DanielleMaywood · commit bda8f298242d · 2025-02-14T12:05:16.000Z
diff --git a/coderd/agentapi/resources_monitoring.go b/coderd/agentapi/resources_monitoring.go
@@ -136,6 +136,7 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
 		debouncedUntil = a.Clock.Now().Add(a.Debounce)
 	}
 
+	//nolint:gocritic // We need to be able to update the resource monitor here.
 	err = a.Database.UpdateMemoryResourceMonitor(dbauthz.AsResourceMonitor(ctx), database.UpdateMemoryResourceMonitorParams{
 		AgentID:        a.AgentID,
 		State:          newState,
@@ -152,7 +153,7 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
 			return xerrors.Errorf("get workspace by id: %w", err)
 		}
 
-		_, err = a.NotificationsEnqueuer.Enqueue(
+		_, err = a.NotificationsEnqueuer.EnqueueWithData(
 			// nolint:gocritic // We need to be able to send the notification.
 			dbauthz.AsNotifier(ctx),
 			workspace.OwnerID,
@@ -161,6 +162,12 @@ func (a *ResourcesMonitoringAPI) monitorMemory(ctx context.Context, datapoints [
 				"workspace": workspace.Name,
 				"threshold": fmt.Sprintf("%d%%", monitor.Threshold),
 			},
+			map[string]any{
+				// NOTE(DanielleMaywood):
+				// We are injecting a timestamp to circumvent the notification
+				// deduplication logic.
+				"timestamp": a.Clock.Now(),
+			},
 			"workspace-monitor-memory",
 		)
 		if err != nil {
@@ -217,6 +224,7 @@ func (a *ResourcesMonitoringAPI) monitorVolumes(ctx context.Context, datapoints
 			})
 		}
 
+		//nolint:gocritic // We need to be able to update the resource monitor here.
 		if err := a.Database.UpdateVolumeResourceMonitor(dbauthz.AsResourceMonitor(ctx), database.UpdateVolumeResourceMonitorParams{
 			AgentID:        a.AgentID,
 			Path:           monitor.Path,
@@ -244,6 +252,10 @@ func (a *ResourcesMonitoringAPI) monitorVolumes(ctx context.Context, datapoints
 			},
 			map[string]any{
 				"volumes": outOfDiskVolumes,
+				// NOTE(DanielleMaywood):
+				// We are injecting a timestamp to circumvent the notification
+				// deduplication logic.
+				"timestamp": a.Clock.Now(),
 			},
 			"workspace-monitor-volumes",
 		); err != nil {
@@ -269,9 +281,9 @@ func (a *ResourcesMonitoringAPI) calculateNextState(
 	for _, state := range states {
 		switch state {
 		case resourcesmonitor.StateOK:
-			okCount += 1
+			okCount++
 		case resourcesmonitor.StateNOK:
-			nokCount += 1
+			nokCount++
 		}
 	}
 
diff --git a/coderd/agentapi/resources_monitoring_test.go b/coderd/agentapi/resources_monitoring_test.go
@@ -317,6 +317,104 @@ func TestMemoryResourceMonitor(t *testing.T) {
 	}
 }
 
+func TestMemoryResourceMonitorMissingData(t *testing.T) {
+	t.Parallel()
+
+	t.Run("UnknownPreventsMovingIntoAlertState", func(t *testing.T) {
+		t.Parallel()
+
+		api, _, clock, notifyEnq := resourceMonitorAPI(t)
+		api.ConsecutiveNOKsToAlert = 2
+		api.MinimumNOKsToAlert = 10
+
+		// Given: A monitor in an OK state.
+		dbgen.WorkspaceAgentMemoryResourceMonitor(t, api.Database, database.WorkspaceAgentMemoryResourceMonitor{
+			AgentID:   api.AgentID,
+			State:     database.WorkspaceAgentMonitorStateOK,
+			Threshold: 80,
+		})
+
+		// When: A datapoint is missing, surrounded by two NOK datapoints.
+		_, err := api.PushResourcesMonitoringUsage(context.Background(), &agentproto.PushResourcesMonitoringUsageRequest{
+			Datapoints: []*agentproto.PushResourcesMonitoringUsageRequest_Datapoint{
+				{
+					CollectedAt: timestamppb.New(clock.Now()),
+					Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
+						Used:  10,
+						Total: 10,
+					},
+				},
+				{
+					CollectedAt: timestamppb.New(clock.Now().Add(10 * time.Second)),
+					Memory:      nil,
+				},
+				{
+					CollectedAt: timestamppb.New(clock.Now().Add(20 * time.Second)),
+					Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
+						Used:  10,
+						Total: 10,
+					},
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// Then: We expect no notifications, as this unknown prevents us knowing we should alert.
+		sent := notifyEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateWorkspaceOutOfMemory))
+		require.Len(t, sent, 0)
+
+		// Then: We expect the monitor to still be in an OK state.
+		monitor, err := api.Database.FetchMemoryResourceMonitorsByAgentID(context.Background(), api.AgentID)
+		require.NoError(t, err)
+		require.Equal(t, database.WorkspaceAgentMonitorStateOK, monitor.State)
+	})
+
+	t.Run("UnknownPreventsMovingOutOfAlertState", func(t *testing.T) {
+		t.Parallel()
+
+		api, _, clock, _ := resourceMonitorAPI(t)
+		api.ConsecutiveNOKsToAlert = 2
+		api.MinimumNOKsToAlert = 10
+
+		// Given: A monitor in a NOK state.
+		dbgen.WorkspaceAgentMemoryResourceMonitor(t, api.Database, database.WorkspaceAgentMemoryResourceMonitor{
+			AgentID:   api.AgentID,
+			State:     database.WorkspaceAgentMonitorStateNOK,
+			Threshold: 80,
+		})
+
+		// When: A datapoint is missing, surrounded by two OK datapoints.
+		_, err := api.PushResourcesMonitoringUsage(context.Background(), &agentproto.PushResourcesMonitoringUsageRequest{
+			Datapoints: []*agentproto.PushResourcesMonitoringUsageRequest_Datapoint{
+				{
+					CollectedAt: timestamppb.New(clock.Now()),
+					Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
+						Used:  1,
+						Total: 10,
+					},
+				},
+				{
+					CollectedAt: timestamppb.New(clock.Now().Add(10 * time.Second)),
+					Memory:      nil,
+				},
+				{
+					CollectedAt: timestamppb.New(clock.Now().Add(20 * time.Second)),
+					Memory: &agentproto.PushResourcesMonitoringUsageRequest_Datapoint_MemoryUsage{
+						Used:  1,
+						Total: 10,
+					},
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// Then: We expect the monitor to still be in a NOK state.
+		monitor, err := api.Database.FetchMemoryResourceMonitorsByAgentID(context.Background(), api.AgentID)
+		require.NoError(t, err)
+		require.Equal(t, database.WorkspaceAgentMonitorStateNOK, monitor.State)
+	})
+}
+
 func TestVolumeResourceMonitorDebounce(t *testing.T) {
 	t.Parallel()
 
diff --git a/coderd/agentapi/resourcesmonitor/resources_monitor.go b/coderd/agentapi/resourcesmonitor/resources_monitor.go
@@ -69,7 +69,7 @@ func CalculateConsecutiveNOK(states []State) int {
 
 	for _, state := range states {
 		if state == StateNOK {
-			curLength += 1
+			curLength++
 		} else {
 			maxLength = max(maxLength, curLength)
 			curLength = 0
