[PATCH] malloc: Simplify tst-free-errno munmap failure test

[Arjun Shankar]

The Linux specific test-case in tst-free-errno was backing up malloc
metadata for a large mmap'd block, overwriting the block with its own
mmap, then restoring malloc metadata and calling free to force an munmap
failure.  However, the backed up pages containing metadata can
occasionally be overlapped by the overwriting mmap, leading to a
metadata corruption.

This commit replaces the test case with a simpler three block
allocation, expecting the kernel to coalesce the VMAs, then cause a
fragmentation to trigger the same failure.
---
Context that won't go into the commit message:

I caught this when investigating failures in a threaded copy of the test
introduced by my patch being discussed here:
https://inbox.sourceware.org/libc-alpha/20251110145716.3147101-1-arjun@redhat.com/
The -threaded-worker copy (which runs this test in an alternate thread while
the main thread waits) frequently fails when firstpage_backup is overwritten
by the MAP_FIXED mmap, leading to chunksize getting set to 0 and a
subsequent abort upon free.

I have tested this new version on an x86_64 Fedora 42 after reducing the VMA
limit to 65536 (it's a lot higher these days) and the munmap failure is
indeed triggered as expected.
---
 malloc/tst-free-errno.c | 80 ++++++++++++++++++-----------------------
 1 file changed, 35 insertions(+), 45 deletions(-)

diff --git a/malloc/tst-free-errno.c b/malloc/tst-free-errno.c
index 1c50860e7e..944108dc7c 100644
--- a/malloc/tst-free-errno.c
+++ b/malloc/tst-free-errno.c
@@ -74,54 +74,44 @@ do_test (void)
   #if defined __linux__
   if (xopen ("/proc/sys/vm/max_map_count", O_RDONLY, 0) >= 0)
     {
-      /* Preparations.  */
-      size_t pagesize = getpagesize ();
-      void *firstpage_backup = xmalloc (pagesize);
-      void *lastpage_backup = xmalloc (pagesize);
-      /* Allocate a large memory area, as a bumper, so that the MAP_FIXED
-         allocation later will not overwrite parts of the memory areas
-         allocated to ld.so or libc.so.  */
-      xmmap (NULL, 0x1000000, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1);
-      /* A file descriptor pointing to a regular file.  */
-      int fd = create_temp_file ("tst-free-errno", NULL);
-      if (fd < 0)
-	FAIL_EXIT1 ("cannot create temporary file");
-
-      /* Do a large memory allocation.  */
+      /* We expect the kernel to coalesce the VMAs for these large mallocs
+         (which will be mmap'd by malloc due to their size).  */
       size_t big_size = 0x3000000;
-      void * volatile ptr = xmalloc (big_size - 0x100);
-      char *ptr_aligned = (char *) ((uintptr_t) ptr & ~(pagesize - 1));
-      /* This large memory allocation allocated a memory area
-	 from ptr_aligned to ptr_aligned + big_size.
-	 Enlarge this memory area by adding a page before and a page
-	 after it.  */
-      memcpy (firstpage_backup, ptr_aligned, pagesize);
-      memcpy (lastpage_backup, ptr_aligned + big_size - pagesize,
-	      pagesize);
-      xmmap (ptr_aligned - pagesize, pagesize + big_size + pagesize,
-	     PROT_READ | PROT_WRITE,
-	     MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1);
-      memcpy (ptr_aligned, firstpage_backup, pagesize);
-      memcpy (ptr_aligned + big_size - pagesize, lastpage_backup,
-	      pagesize);
+      void * volatile block1 = xmalloc (big_size - 100);
+      void * volatile block2 = xmalloc (big_size - 100);
+      void * volatile block3 = xmalloc (big_size - 100);
+
+      /* If block2 lands between block1 and block3, we can continue the test
+         since it depends on being able to free block2 to cause an munmap
+         failure.  */
+      if (((block1 > block2) && (block2 > block3))
+          || ((block1 < block2) && (block2 < block3)))
+        {
+          /* We will map this fd repeatedly to consume VMA mappings.  */
+          int fd = create_temp_file ("tst-free-errno", NULL);
+          if (fd < 0)
+            FAIL_EXIT1 ("cannot create temporary file for mmap'ing");
 
-      /* Now add as many mappings as we can.
-	 Stop at 65536, in order not to crash the machine (in case the
-	 limit has been increased by the system administrator).  */
-      for (int i = 0; i < 65536; i++)
-	if (mmap (NULL, pagesize, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0)
-	    == MAP_FAILED)
-	  break;
-      /* Now the number of VMAs of this process has hopefully attained
-	 its limit.  */
+          /* Now add as many mappings as we can.
+             Stop at 65536, in order not to crash the machine (in case the
+             limit has been increased by the system administrator).  */
+          size_t pagesize = getpagesize ();
+          for (int i = 0; i < 65536; i++)
+            if (mmap (NULL, pagesize, PROT_READ, MAP_FILE | MAP_PRIVATE,
+                      fd, 0)
+                == MAP_FAILED)
+              break;
+          /* Now the number of VMAs of this process has hopefully attained
+             its limit.  */
 
-      errno = 1789;
-      /* This call to free() is supposed to call
-	   munmap (ptr_aligned, big_size);
-	 which increases the number of VMAs by 1, which is supposed
-	 to fail.  */
-      free (ptr);
-      TEST_VERIFY (get_errno () == 1789);
+          errno = 1789;
+          /* This call to free() is supposed to call munmap, which should
+             fail because the fragmentation of a bigger coalesced VMA will
+             lead to an increase in the number of VMAs which we already
+             maxed out.  */
+          free (block2);
+          TEST_VERIFY (get_errno () == 1789);
+        }
     }
   #endif
 
-- 
2.51.1