diff mbox series

[Concept,08/17] test: Reset malloc backtrace collection before each test

Message ID 20260316183050.3855921-9-sjg@u-boot.org
State New
Headers
Series Add automatic memory-leak detection to U-Boot tests |

Commit Message

Simon Glass March 16, 2026, 6:30 p.m. UTC 
  From: Simon Glass <sjg@chromium.org>

The stack-protector test and __stack_chk_fail() call
malloc_backtrace_skip(true) to prevent crashes when collecting
backtraces from a corrupted stack. However, they never reset it, so all
subsequent allocations in a long-running session (e.g. pytest) have
empty caller backtraces.

Add malloc_backtrace_unbusy() to clear the reentrant guard which can
also get stuck if backtrace collection crashes. Reset both flags at the
start of each test in test_pre_run()

Signed-off-by: Simon Glass <sjg@chromium.org>
---

 common/dlmalloc.c | 24 ++++++++++++++++++++++++
 include/malloc.h  | 30 ++++++++++++++++++++++++++++++
 test/test-main.c  |  8 ++++++++
 3 files changed, 62 insertions(+)
diff mbox series

Patch

diff --git a/common/dlmalloc.c b/common/dlmalloc.c
index e61f565bbab..f05d7ae83c5 100644
--- a/common/dlmalloc.c
+++ b/common/dlmalloc.c
@@ -5986,6 +5986,30 @@  void malloc_backtrace_skip(bool skip)
 #endif
 }
 
+void malloc_backtrace_unbusy(void)
+{
+#if CONFIG_IS_ENABLED(MCHECK_BACKTRACE)
+	in_backtrace = false;
+#endif
+}
+
+bool malloc_backtrace_is_active(bool *skipp, bool *busyp)
+{
+#if CONFIG_IS_ENABLED(MCHECK_BACKTRACE)
+	if (skipp)
+		*skipp = mcheck_skip_backtrace;
+	if (busyp)
+		*busyp = in_backtrace;
+	return !mcheck_skip_backtrace && !in_backtrace;
+#else
+	if (skipp)
+		*skipp = false;
+	if (busyp)
+		*busyp = false;
+	return false;
+#endif
+}
+
 static const char *mcheck_caller(void)
 {
 #if CONFIG_IS_ENABLED(MCHECK_BACKTRACE)
diff --git a/include/malloc.h b/include/malloc.h
index ded5520ca23..05d4c06fb89 100644
--- a/include/malloc.h
+++ b/include/malloc.h
@@ -827,10 +827,40 @@  int malloc_log_entry(uint idx, struct mlog_entry **entryp);
  *
  * @skip: true to skip backtrace collection, false to enable it
  */
+
+/**
+ * malloc_backtrace_unbusy() - Clear the backtrace reentrant guard
+ *
+ * The malloc backtrace collector sets a guard flag while collecting a
+ * backtrace to prevent re-entrancy. If a crash or longjmp occurs during
+ * collection, the guard stays set and all subsequent backtraces are
+ * silently skipped. Call this to reset it.
+ */
+
+/**
+ * malloc_backtrace_is_active() - Check whether backtrace collection works
+ *
+ * @skipp: If non-NULL, returns true if collection is disabled via
+ *	malloc_backtrace_skip()
+ * @busyp: If non-NULL, returns true if the reentrant guard is stuck
+ * Return: true if backtrace collection is active (neither skipped nor busy)
+ */
 #if CONFIG_IS_ENABLED(MCHECK_HEAP_PROTECTION)
 void malloc_backtrace_skip(bool skip);
+void malloc_backtrace_unbusy(void);
+bool malloc_backtrace_is_active(bool *skipp, bool *busyp);
 #else
 static inline void malloc_backtrace_skip(bool skip) {}
+static inline void malloc_backtrace_unbusy(void) {}
+static inline bool malloc_backtrace_is_active(bool *skipp, bool *busyp)
+{
+	if (skipp)
+		*skipp = false;
+	if (busyp)
+		*busyp = false;
+
+	return false;
+}
 #endif
 
 /**
diff --git a/test/test-main.c b/test/test-main.c
index 09458fa91da..06bb0ff6e1b 100644
--- a/test/test-main.c
+++ b/test/test-main.c
@@ -464,6 +464,14 @@  static int dm_test_restore(struct device_node *of_root)
  */
 static int test_pre_run(struct unit_test_state *uts, struct unit_test *test)
 {
+	/*
+	 * Reset backtrace collection in case a previous test disabled it
+	 * (e.g. stack-protector test via __stack_chk_fail) or a crash
+	 * left the reentrant guard set.
+	 */
+	malloc_backtrace_skip(false);
+	malloc_backtrace_unbusy();
+
 	ut_assertok(event_init());
 
 	/*