malloc patch for Emacs-20.3 MALLOC_CHECK_ problem

[wmglo@dent.med.uni-muenchen.de]

Hello,

I think this should fix the small remaining Emacs-20.3 problem when
MALLOC_CHECK_ is set.

Regards,
Wolfram.

1999-06-05  Wolfram Gloger  <wmglo@dent.med.uni-muenchen.de>

	* malloc/malloc.c (check_action): Change into bitmap so that both
	diagnostic and abort can be requested by setting it to 3.
	(mALLOC_SET_STATe): Disable malloc checking if necessary.

--- /home/wg/src/malloc/glibc-malloc/malloc.c	Sun May 16 23:21:43 1999
+++ malloc.c	Sat Jun  5 12:40:45 1999
@@ -1780,7 +1780,7 @@
   __free_hook = free_check;
   __realloc_hook = realloc_check;
   __memalign_hook = memalign_check;
-  if(check_action == 1)
+  if(check_action & 1)
     fprintf(stderr, "malloc: using debugging hooks\n");
 }
 
@@ -4068,7 +4068,11 @@
    the heap contents are saved/restored via some other method.  The
    primary example for this is GNU Emacs with its `dumping' procedure.
    `Hook' function pointers are never saved or restored by these
-   functions. */
+   functions, with two exceptions: If malloc checking was in use when
+   malloc_get_state() was called, then malloc_set_state() calls
+   __malloc_check_init() if possible; if malloc checking was not in
+   use in the recorded state but the user requested malloc checking,
+   then the hooks are reset to 0.  */
 
 #define MALLOC_STATE_MAGIC   0x444c4541l
 #define MALLOC_STATE_VERSION (0*0x100l + 1l) /* major*0x100 + minor */
@@ -4196,10 +4200,18 @@
   /* add version-dependent code here */
   if (ms->version >= 1) {
 #if defined _LIBC || defined MALLOC_HOOKS
-    /* Check whether it is safe to enable malloc checking.  */
+    /* Check whether it is safe to enable malloc checking, or whether
+       it is necessary to disable it.  */
     if (ms->using_malloc_checking && !using_malloc_checking &&
 	!disallow_malloc_check)
       __malloc_check_init ();
+    else if (!ms->using_malloc_checking && using_malloc_checking) {
+      __malloc_hook = 0;
+      __free_hook = 0;
+      __realloc_hook = 0;
+      __memalign_hook = 0;
+      using_malloc_checking = 0;
+    }
 #endif
   }
 
@@ -4319,13 +4331,11 @@
   if((char*)t + chunksize(t) == sbrk_base + sbrked_mem ||
      t == initial_top(&main_arena)) return 0;
 
-  switch(check_action) {
-  case 1:
+  if(check_action & 1)
     fprintf(stderr, "malloc: top chunk is corrupt\n");
-    break;
-  case 2:
+  if(check_action & 2)
     abort();
-  }
+
   /* Try to set up a new top chunk. */
   brk = MORECORE(0);
   front_misalign = (unsigned long)chunk2mem(brk) & MALLOC_ALIGN_MASK;
@@ -4374,13 +4384,10 @@
   p = mem2chunk_check(mem);
   if(!p) {
     (void)mutex_unlock(&main_arena.mutex);
-    switch(check_action) {
-    case 1:
+    if(check_action & 1)
       fprintf(stderr, "free(): invalid pointer %p!\n", mem);
-      break;
-    case 2:
+    if(check_action & 2)
       abort();
-    }
     return;
   }
 #if HAVE_MMAP
@@ -4413,13 +4420,10 @@
   oldp = mem2chunk_check(oldmem);
   if(!oldp) {
     (void)mutex_unlock(&main_arena.mutex);
-    switch(check_action) {
-    case 1:
+    if(check_action & 1)
       fprintf(stderr, "realloc(): invalid pointer %p!\n", oldmem);
-      break;
-    case 2:
+    if(check_action & 2)
       abort();
-    }
     return malloc_check(bytes, NULL);
   }
   oldsize = chunksize(oldp);