i386 Linux signal trampoline support for gdb 4.18

[Ian Lance Taylor <ian@zembu.com>]

These patches add support for the signal trampoline code used by the
i386 Linux kernel.  These patches are against gdb 4.18.

These patches permit gdb to do backtraces through signal handlers on
i386 GNU/Linux.  They also permit gdb to do correct stepping when
signals invoke signal handlers.

On most systems signal trampoline code is simply a libc function, so
it is comparatively easy to detect one.  The Linux kernel, however,
builds the signal trampoline on the stack.  This patch works by
looking for the particular set of instructions the Linux kernel places
on the stack.  This test should rarely give a false positive, because
the kernel builds instructions which invoke the sigreturn system call,
which is only useful when returning from a signal handler.  It is of
course possible that the kernel behaviour will change at some later
date, in which case this test will have to be modified.

Ian


1999-07-14  Ian Lance Taylor  <ian@zembu.com>

	* config/i386/tm-linux.h (LINUX_SIGCONTEXT_SIZE): Define.
	(LINUX_SIGCONTEXT_PC_OFFSET): Define.
	(LINUX_SIGCONTEXT_SP_OFFSET): Define.
	(SIGCONTEXT_PC_OFFSET): Don't define.
	(I386_LINUX_SIGTRAMP): Define.
	(IN_SIGTRAMP): Define.
	(i386_linux_sigtramp): Declare.
	(sigtramp_saved_pc): Define.
	(i386_linux_sigtramp_saved_pc): Declare.
	(FRAMELESS_SIGNAL): Define.
	(FRAME_CHAIN, FRAME_SAVED_PC): Define after #undef.
	* i386-tdep.c (i386_linux_sigtramp_start): New static function if
	I386_LINUX_SIGTRAMP.
	(i386_linux_sigtramp): New function if I386_LINUX_SIGTRAMP.
	(i386_linux_sigtramp_saved_pc): Likewise.
	(i386_linux_sigtramp_saved_sp): Likewise.


Index: config/i386/tm-linux.h
===================================================================
RCS file: /zembu/cvsfiles/tools/gdb/gdb/config/i386/tm-linux.h,v
retrieving revision 1.1.1.1
diff -u -p -r1.1.1.1 tm-linux.h
--- tm-linux.h	1998/10/05 00:07:57	1.1.1.1
+++ tm-linux.h	1999/07/14 04:36:10
@@ -25,14 +25,82 @@ Foundation, Inc., 59 Temple Place - Suit
 
 #include "i386/tm-i386.h"
 
-/* Offset to saved PC in sigcontext, from <linux/signal.h>.  */
-#define SIGCONTEXT_PC_OFFSET 38
+/* Size of sigcontext, from <asm/sigcontext.h>.  */
+#define LINUX_SIGCONTEXT_SIZE (88)
 
+/* Offset to saved PC in sigcontext, from <asm/sigcontext.h>.  */
+#define LINUX_SIGCONTEXT_PC_OFFSET (56)
+
+/* Offset to saved SP in sigcontext, from <asm/sigcontext.h>.  */
+#define LINUX_SIGCONTEXT_SP_OFFSET (28)
+
 /* We need this file for the SOLIB_TRAMPOLINE stuff. */
 
 #include "tm-sysv4.h"
 
 /* The following works around a problem with /usr/include/sys/procfs.h  */
 #define sys_quotactl 1
+
+/* When the i386 Linux kernel calls a signal handler, the return
+   address points to a bit of code on the stack.  These definitions
+   are used to identify this bit of code as a signal trampoline in
+   order to support backtracing through calls to signal handlers.  */
+
+#define I386_LINUX_SIGTRAMP
+#define IN_SIGTRAMP(pc, name) ((name) == NULL && i386_linux_sigtramp (pc))
+
+extern int i386_linux_sigtramp PARAMS ((CORE_ADDR));
+
+/* We need our own version of sigtramp_saved_pc to get the saved PC in
+   a sigtramp routine.  */
+
+#define sigtramp_saved_pc i386_linux_sigtramp_saved_pc
+extern CORE_ADDR i386_linux_sigtramp_saved_pc PARAMS ((struct frame_info *));
+
+/* Signal trampolines don't have a meaningful frame.  As in tm-i386.h,
+   the frame pointer value we use is actually the frame pointer of the
+   calling frame--that is, the frame which was in progress when the
+   signal trampoline was entered.  gdb mostly treats this frame
+   pointer value as a magic cookie.  We detect the case of a signal
+   trampoline by looking at the SIGNAL_HANDLER_CALLER field, which is
+   set based on IN_SIGTRAMP.
+
+   When a signal trampoline is invoked from a frameless function, we
+   essentially have two frameless functions in a row.  In this case,
+   we use the same magic cookie for three frames in a row.  We detect
+   this case by seeing whether the next frame has
+   SIGNAL_HANDLER_CALLER set, and, if it does, checking whether the
+   current frame is actually frameless.  In this case, we need to get
+   the PC by looking at the SP register value stored in the signal
+   context.
+
+   This should work in most cases except in horrible situations where
+   a signal occurs just as we enter a function but before the frame
+   has been set up.  */
+
+#define FRAMELESS_SIGNAL(FRAME)					\
+  ((FRAME)->next != NULL					\
+   && (FRAME)->next->signal_handler_caller			\
+   && frameless_look_for_prologue (FRAME))
+
+#undef FRAME_CHAIN
+#define FRAME_CHAIN(FRAME)					\
+  ((FRAME)->signal_handler_caller				\
+   ? (FRAME)->frame						\
+    : (FRAMELESS_SIGNAL (FRAME)					\
+       ? (FRAME)->frame						\
+       : (!inside_entry_file ((FRAME)->pc)			\
+	  ? read_memory_integer ((FRAME)->frame, 4)		\
+	  : 0)))
+
+#undef FRAME_SAVED_PC
+#define FRAME_SAVED_PC(FRAME)					\
+  ((FRAME)->signal_handler_caller				\
+   ? sigtramp_saved_pc (FRAME)					\
+   : (FRAMELESS_SIGNAL (FRAME)					\
+      ? read_memory_integer (i386_linux_sigtramp_saved_sp ((FRAME)->next), 4) \
+      : read_memory_integer ((FRAME)->frame + 4, 4)))
+
+extern CORE_ADDR i386_linux_sigtramp_saved_sp PARAMS ((struct frame_info *));
 
 #endif  /* #ifndef TM_LINUX_H */
Index: i386-tdep.c
===================================================================
RCS file: /zembu/cvsfiles/tools/gdb/gdb/i386-tdep.c,v
retrieving revision 1.1.1.2
diff -u -p -r1.1.1.2 i386-tdep.c
--- i386-tdep.c	1999/07/02 19:07:52	1.1.1.2
+++ i386-tdep.c	1999/07/14 04:36:10
@@ -711,6 +711,140 @@ i386v4_sigtramp_saved_pc (frame)
 }
 #endif /* I386V4_SIGTRAMP_SAVED_PC */
 
+#ifdef I386_LINUX_SIGTRAMP
+
+/* When the i386 Linux kernel calls a signal handler, the return
+   address points to a bit of code on the stack.  This function
+   returns whether the PC appears to be within this bit of code.
+
+   The instruction sequence is
+       pop    %eax
+       mov    $0x77,%eax
+       int    $0x80
+   or 0x58 0xb8 0x77 0x00 0x00 0x00 0xcd 0x80.
+
+   Checking for the code sequence should be somewhat reliable, because
+   the effect is to call the system call sigreturn.  This is unlikely
+   to occur anywhere other than a signal trampoline.
+
+   It kind of sucks that we have to read memory from the process in
+   order to identify a signal trampoline, but there doesn't seem to be
+   any other way.  The IN_SIGTRAMP macro in tm-linux.h arranges to
+   only call us if no function name could be identified, which should
+   be the case since the code is on the stack.  */
+
+#define LINUX_SIGTRAMP_INSN0 (0x58)	/* pop %eax */
+#define LINUX_SIGTRAMP_OFFSET0 (0)
+#define LINUX_SIGTRAMP_INSN1 (0xb8)	/* mov $NNNN,%eax */
+#define LINUX_SIGTRAMP_OFFSET1 (1)
+#define LINUX_SIGTRAMP_INSN2 (0xcd)	/* int */
+#define LINUX_SIGTRAMP_OFFSET2 (6)
+
+static const unsigned char linux_sigtramp_code[] =
+{
+  LINUX_SIGTRAMP_INSN0,					/* pop %eax */
+  LINUX_SIGTRAMP_INSN1, 0x77, 0x00, 0x00, 0x00,		/* mov $0x77,%eax */
+  LINUX_SIGTRAMP_INSN2, 0x80				/* int $0x80 */
+};
+
+#define LINUX_SIGTRAMP_LEN (sizeof linux_sigtramp_code)
+
+/* If PC is in a sigtramp routine, return the address of the start of
+   the routine.  Otherwise, return 0.  */
+
+static CORE_ADDR
+i386_linux_sigtramp_start (pc)
+     CORE_ADDR pc;
+{
+  unsigned char buf[LINUX_SIGTRAMP_LEN];
+
+  /* We only recognize a signal trampoline if PC is at the start of
+     one of the three instructions.  We optimize for finding the PC at
+     the start, as will be the case when the trampoline is not the
+     first frame on the stack.  We assume that in the case where the
+     PC is not at the start of the instruction sequence, there will be
+     a few trailing readable bytes on the stack.  */
+
+  if (read_memory_nobpt (pc, (char *) buf, LINUX_SIGTRAMP_LEN) != 0)
+    return 0;
+
+  if (buf[0] != LINUX_SIGTRAMP_INSN0)
+    {
+      int adjust;
+
+      switch (buf[0])
+	{
+	case LINUX_SIGTRAMP_INSN1:
+	  adjust = LINUX_SIGTRAMP_OFFSET1;
+	  break;
+	case LINUX_SIGTRAMP_INSN2:
+	  adjust = LINUX_SIGTRAMP_OFFSET2;
+	  break;
+	default:
+	  return 0;
+	}
+
+      pc -= adjust;
+
+      if (read_memory_nobpt (pc, (char *) buf, LINUX_SIGTRAMP_LEN) != 0)
+	return 0;
+    }
+
+  if (memcmp (buf, linux_sigtramp_code, LINUX_SIGTRAMP_LEN) != 0)
+    return 0;
+
+  return pc;
+}
+
+/* Return whether PC is in a Linux sigtramp routine.  */
+
+int
+i386_linux_sigtramp (pc)
+     CORE_ADDR pc;
+{
+  return i386_linux_sigtramp_start (pc) != 0;
+}
+
+/* Assuming FRAME is for a Linux sigtramp routine, return the saved
+   program counter.  The Linux kernel will set up a sigcontext
+   structure immediately before the sigtramp routine on the stack.  */
+
+CORE_ADDR
+i386_linux_sigtramp_saved_pc (frame)
+     struct frame_info *frame;
+{
+  CORE_ADDR pc;
+
+  pc = i386_linux_sigtramp_start (frame->pc);
+  if (pc == 0)
+    error ("i386_linux_sigtramp_saved_pc called when no sigtramp");
+  return read_memory_integer ((pc
+			       - LINUX_SIGCONTEXT_SIZE
+			       + LINUX_SIGCONTEXT_PC_OFFSET),
+			      4);
+}
+
+/* Assuming FRAME is for a Linux sigtramp routine, return the saved
+   stack pointer.  The Linux kernel will set up a sigcontext structure
+   immediately before the sigtramp routine on the stack.  */
+
+CORE_ADDR
+i386_linux_sigtramp_saved_sp (frame)
+     struct frame_info *frame;
+{
+  CORE_ADDR pc;
+
+  pc = i386_linux_sigtramp_start (frame->pc);
+  if (pc == 0)
+    error ("i386_linux_sigtramp_saved_sp called when no sigtramp");
+  return read_memory_integer ((pc
+			       - LINUX_SIGCONTEXT_SIZE
+			       + LINUX_SIGCONTEXT_SP_OFFSET),
+			      4);
+}
+
+#endif /* I386_LINUX_SIGTRAMP */
+
 #ifdef STATIC_TRANSFORM_NAME
 /* SunPRO encodes the static variables.  This is not related to C++ mangling,
    it is done for C too.  */