ESP8266: Add esf_buf monitor to detect a WiFi TX stall

It detects a WiFi TX stall that sometimes happens
and that cannot be recovered from other than by rebooting.

Enable by defining ESP_ESF_BUF_MONITOR_INTERVAL_MS to something reasonable, e.g. 5000.
It takes at most 2 intervals to detect the stall.

`void esp_esf_buf_monitor_failure(void)` by default reboots the device
but can be overridden if extra cleanup is required.

Author: rojer

platforms/esp8266/Makefile.build

@@ -104,6 +104,7 @@ MGOS_SRCS += cs_file.c cs_hex.c cs_rbuf.c \
 MGOS_SRCS += esp_config.c \
              esp_coredump.c \
              esp_debug.c \
+             esp_esf_buf_monitor.c \
              esp_exc.c \
              esp_flash_writer.c \
              esp_gpio.c \
@@ -208,15 +209,12 @@ BOOT_LOADER_BIN = $(RBOOT_BUILD_DIR)/rboot.bin
 APP_BIN = $(BUILD_DIR)/$(APP).bin
 APP_ELF = $(BUILD_DIR)/$(APP).elf
 LD_SCRIPT = $(GEN_DIR)/$(APP0_ADDR).ld
-LD_WRAPPERS =
+LD_WRAPPERS = esf_buf_alloc esf_buf_recycle lmacProcessTXStartData
 HEAP_LOG_FLAGS =
 
 ifneq "${MGOS_ENABLE_HEAP_LOG}${MGOS_ENABLE_CALL_TRACE}" "00"
   HEAP_LOG_FLAGS += -DMGOS_ENABLE_HEAP_LOG
-  LD_WRAPPERS += -Wl,--wrap=umm_calloc \
-                 -Wl,--wrap=umm_malloc \
-                 -Wl,--wrap=umm_realloc \
-                 -Wl,--wrap=umm_free
+  LD_WRAPPERS += umm_calloc umm_malloc umm_realloc umm_free
 endif
 ifeq "${MGOS_ENABLE_CALL_TRACE}" "1"
   MGOS_SRCS += cs_heap_trace.c
@@ -351,13 +349,15 @@ $(APP_BIN): $(APP_ELF) $(BUILD_DIR)/esptool2
 
 # -- Linking.
 
+WLWRAP := -Wl,--wrap=
+
 # Link the main ELF output file.
 BIN_PARTS = $(BUILD_DIR)/esp_cache.c.o $(APP_OBJS) $(FFI_EXPORTS_O) $(BUILD_INFO_O) $(MG_BUILD_INFO_O) $(MGOS_LIB) $(APP_BIN_LIB_FILES)
 $(APP_ELF): $(BIN_PARTS) $(LD_SCRIPT)
 ifeq "$(MGOS_ESP8266_RTOS)" "1"
 	$(OBJCOPY) --weaken-symbol printf --weaken-symbol puts /opt/Espressif/esp-open-sdk/xtensa-lx106-elf/xtensa-lx106-elf/lib/libc.a
 endif
-	$(call link,$(BIN_PARTS) $(LIBS) $(LD_WRAPPERS))
+	$(call link,$(BIN_PARTS) $(LIBS) $(addprefix $(WLWRAP),$(LD_WRAPPERS)))
 
 $(BUILD_DIR)/dummy_main.c: $(APP_OBJS) $(MGOS_ESP8266_PATH)/Makefile.build
 	$(vecho) "GEN   $@"

platforms/esp8266/src/esp_esf_buf_monitor.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021 Deomid "rojer" Ryabkov
+ * All rights reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "esp_esf_buf_monitor.h"
+
+#include <stdlib.h>
+
+#include "mgos.h"
+
+struct pbuf;
+
+struct esf_buf *g_pending_bufs[8] = {0};
+uint32_t g_esf_buf_canary_ctr = 0;
+uint8_t g_esf_buf_canary_strikes = 0;
+
+extern struct esf_buf *__real_esf_buf_alloc(struct pbuf *p, uint32_t type,
+                                            uint32_t a4);
+struct esf_buf *__wrap_esf_buf_alloc(struct pbuf *p, uint32_t type,
+                                     uint32_t a4) {
+  struct esf_buf *eb = __real_esf_buf_alloc(p, type, a4);
+  if (eb != NULL) {
+    for (int i = 0; i < 8; i++) {
+      if (g_pending_bufs[i] == NULL) {
+        g_pending_bufs[i] = eb;
+        break;
+      }
+    }
+  }
+  return eb;
+}
+
+extern void __real_esf_buf_recycle(struct esf_buf *eb, uint32_t type);
+void __wrap_esf_buf_recycle(struct esf_buf *eb, uint32_t type) {
+  for (int i = 0; i < 8; i++) {
+    if (g_pending_bufs[i] == eb) {
+      g_pending_bufs[i] = NULL;
+      g_esf_buf_canary_ctr++;
+    }
+  }
+  __real_esf_buf_recycle(eb, type);
+}
+
+static uint32_t s_esf_buf_last_canary_ctr = 0;
+
+static void esp_esf_buf_monitor_timer_cb(void *arg) {
+  int num_pending = 0;
+  for (int i = 0; i < 8; i++) {
+    if (g_pending_bufs[i] != NULL) num_pending++;
+  }
+  LOG(LL_DEBUG, ("np %d ctr %lu %lu", num_pending, s_esf_buf_last_canary_ctr,
+                 g_esf_buf_canary_ctr));
+  if (num_pending == 0) {
+    // All clear, no problem.
+    return;
+  }
+  if (g_esf_buf_canary_ctr != s_esf_buf_last_canary_ctr) {
+    // Things are moving along, all good.
+    s_esf_buf_last_canary_ctr = g_esf_buf_canary_ctr;
+    return;
+  }
+  LOG(LL_ERROR, ("TX is stuck!"));
+  static bool cb_invoked = false;
+  if (!cb_invoked) {
+    esp_esf_buf_monitor_failure();
+    cb_invoked = true;
+  }
+  (void) arg;
+}
+
+void esp_esf_buf_monitor_failure(void) WEAK;
+void esp_esf_buf_monitor_failure(void) {
+  mgos_system_restart_after(1000);
+}
+
+extern void __real_lmacProcessTXStartData(uint8_t id);
+IRAM void __wrap_lmacProcessTXStartData(uint8_t id) {
+#ifdef ESP_ESF_BUF_TRIGGER_BUG
+  // Introducing a delay here triggers the condition reasonably quickly
+  // under moderate traffic load (3 x curl; sleep 1 in parallel).
+  if (id == 0) {
+    ets_delay_us(500);
+  }
+#endif
+  __real_lmacProcessTXStartData(id);
+}
+
+void esp_esf_buf_monitor_init(void) {
+  mgos_set_timer(ESP_ESF_BUF_MONITOR_INTERVAL_MS, MGOS_TIMER_REPEAT,
+                 esp_esf_buf_monitor_timer_cb, NULL);
+}

platforms/esp8266/src/esp_esf_buf_monitor.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2021 Deomid "rojer" Ryabkov
+ * All rights reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+// esf_buf monitor detects a WiFi TX stall that sometimes happens
+// and that cannot be recovered from other than by rebooting.
+
+#ifndef ESP_ESF_BUF_MONITOR_INTERVAL_MS
+#define ESP_ESF_BUF_MONITOR_INTERVAL_MS 0  // Disabled by default
+#endif
+
+// This callback will be invoked once stuck TX is detected.
+// Default (weak) implementation is to reboot the device.
+void esp_esf_buf_monitor_failure(void);
+
+void esp_esf_buf_monitor_init(void);

platforms/esp8266/src/esp_main.c

@@ -38,6 +38,7 @@
 #include "esp_adc.h"
 #endif
 #include "esp_coredump.h"
+#include "esp_esf_buf_monitor.h"
 #include "esp_exc.h"
 #include "esp_features.h"
 #include "esp_fs.h"
@@ -173,6 +174,10 @@ enum mgos_init_result esp_mgos_init2(void) {
     return ir;
   }
 
+#if ESP_ESF_BUF_MONITOR_INTERVAL_MS > 0
+  esp_esf_buf_monitor_init();
+#endif
+
   return MGOS_INIT_OK;
 }