fix: rate-limit CSI sends and add ENOMEM backoff to prevent crash (ruvnet#132)

ruvnet · web-flow · commit ce171696b266 · 2026-03-03T16:00:40.000-05:00
The CSI callback fires for every WiFi frame in promiscuous mode (100-500+ fps). Each call invoked sendto() synchronously, exhausting lwIP packet buffers (errno 12 = ENOMEM). The rapid-fire failures cascaded into a LoadProhibited guru meditation crash. Two fixes: 1. csi_collector.c: Rate-limit UDP sends to 50 Hz (20ms interval). CSI frames arriving between sends are dropped — the sensing pipeline only needs 20-50 Hz. 2. stream_sender.c: When sendto fails with ENOMEM, suppress further sends for 100ms to let lwIP reclaim buffers. Logs the backoff event and resumes automatically. Closes ruvnet#127
diff --git a/firmware/esp32-csi-node/main/csi_collector.c b/firmware/esp32-csi-node/main/csi_collector.c
@@ -27,6 +27,16 @@ static uint32_t s_sequence = 0;
 static uint32_t s_cb_count = 0;
 static uint32_t s_send_ok = 0;
 static uint32_t s_send_fail = 0;
+static uint32_t s_rate_skip = 0;
+
+/**
+ * Minimum interval between UDP sends in microseconds.
+ * CSI callbacks can fire hundreds of times per second in promiscuous mode.
+ * We cap the send rate to avoid exhausting lwIP packet buffers (ENOMEM).
+ * Default: 20 ms = 50 Hz max send rate.
+ */
+#define CSI_MIN_SEND_INTERVAL_US  (20 * 1000)
+static int64_t s_last_send_us = 0;
 
 /* ---- ADR-029: Channel-hop state ---- */
 
@@ -143,14 +153,23 @@ static void wifi_csi_callback(void *ctx, wifi_csi_info_t *info)
     size_t frame_len = csi_serialize_frame(info, frame_buf, sizeof(frame_buf));
 
     if (frame_len > 0) {
-        int ret = stream_sender_send(frame_buf, frame_len);
-        if (ret > 0) {
-            s_send_ok++;
-        } else {
-            s_send_fail++;
-            if (s_send_fail <= 5) {
-                ESP_LOGW(TAG, "sendto failed (fail #%lu)", (unsigned long)s_send_fail);
+        /* Rate-limit UDP sends to avoid ENOMEM from lwIP pbuf exhaustion.
+         * In promiscuous mode, CSI callbacks can fire 100-500+ times/sec.
+         * We only need 20-50 Hz for the sensing pipeline. */
+        int64_t now = esp_timer_get_time();
+        if ((now - s_last_send_us) >= CSI_MIN_SEND_INTERVAL_US) {
+            int ret = stream_sender_send(frame_buf, frame_len);
+            if (ret > 0) {
+                s_send_ok++;
+                s_last_send_us = now;
+            } else {
+                s_send_fail++;
+                if (s_send_fail <= 5) {
+                    ESP_LOGW(TAG, "sendto failed (fail #%lu)", (unsigned long)s_send_fail);
+                }
             }
+        } else {
+            s_rate_skip++;
         }
     }
 
diff --git a/firmware/esp32-csi-node/main/stream_sender.c b/firmware/esp32-csi-node/main/stream_sender.c
@@ -9,6 +9,7 @@
 
 #include <string.h>
 #include "esp_log.h"
+#include "esp_timer.h"
 #include "lwip/sockets.h"
 #include "lwip/netdb.h"
 #include "sdkconfig.h"
@@ -18,6 +19,17 @@ static const char *TAG = "stream_sender";
 static int s_sock = -1;
 static struct sockaddr_in s_dest_addr;
 
+/**
+ * ENOMEM backoff state.
+ * When sendto fails with ENOMEM (errno 12), we suppress further sends for
+ * a cooldown period to let lwIP reclaim packet buffers.  Without this,
+ * rapid-fire CSI callbacks can exhaust the pbuf pool and crash the device.
+ */
+static int64_t s_backoff_until_us = 0;       /* esp_timer timestamp to resume */
+#define ENOMEM_COOLDOWN_MS  100              /* suppress sends for 100 ms */
+#define ENOMEM_LOG_INTERVAL 50               /* log every Nth suppressed send */
+static uint32_t s_enomem_suppressed = 0;
+
 static int sender_init_internal(const char *ip, uint16_t port)
 {
     s_sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
@@ -57,10 +69,37 @@ int stream_sender_send(const uint8_t *data, size_t len)
         return -1;
     }
 
+    /* ENOMEM backoff: if we recently exhausted lwIP buffers, skip sends
+     * until the cooldown expires.  This prevents the cascade of failed
+     * sendto calls that leads to a guru meditation crash. */
+    if (s_backoff_until_us > 0) {
+        int64_t now = esp_timer_get_time();
+        if (now < s_backoff_until_us) {
+            s_enomem_suppressed++;
+            if ((s_enomem_suppressed % ENOMEM_LOG_INTERVAL) == 1) {
+                ESP_LOGW(TAG, "sendto suppressed (ENOMEM backoff, %lu dropped)",
+                         (unsigned long)s_enomem_suppressed);
+            }
+            return -1;
+        }
+        /* Cooldown expired — resume sending */
+        ESP_LOGI(TAG, "ENOMEM backoff expired, resuming sends (%lu were suppressed)",
+                 (unsigned long)s_enomem_suppressed);
+        s_backoff_until_us = 0;
+        s_enomem_suppressed = 0;
+    }
+
     int sent = sendto(s_sock, data, len, 0,
                       (struct sockaddr *)&s_dest_addr, sizeof(s_dest_addr));
     if (sent < 0) {
-        ESP_LOGW(TAG, "sendto failed: errno %d", errno);
+        if (errno == ENOMEM) {
+            /* Start backoff to let lwIP reclaim buffers */
+            s_backoff_until_us = esp_timer_get_time() +
+                                 (int64_t)ENOMEM_COOLDOWN_MS * 1000;
+            ESP_LOGW(TAG, "sendto ENOMEM — backing off for %d ms", ENOMEM_COOLDOWN_MS);
+        } else {
+            ESP_LOGW(TAG, "sendto failed: errno %d", errno);
+        }
         return -1;
     }
 
