[RFA] epilogue unwinder for i386 (reverse 1/2)

[Michael Snyder <msnyder at vmware dot com>]

This comes out of a discussion with Daniel, about how gcc
does not generate the right dwarf info to allow correct
frame unwinding in function epilogues, causing frame_unwind
to return bad results.

It's necessary for reverse-step, which will frequently step
backward to the return instruction of a function.  But it also
provides an improvement for forward debugging, in that now,
without this change, if you STEPI until you are at the return
instruction, you will get a bad backtrace.

The infrun changes that take advantage of this patch will follow
separately.

Michael


2009-07-03  Michael Snyder  <msnyder@vmware.com>

	* i386-tdep.c: Add a frame unwinder for function epilogues.
	(i386_in_function_epilogue_p): New function.
	(i386_epilogue_frame_sniffer): New function.
	(i386_epilogue_frame_cache): New function.
	(i386_epilogue_frame_this_id): New function.
	(i386_epilogue_frame_unwind): New struct frame_unwind.
	(i386_gdbarch_init): Hook the new unwinder.

Index: i386-tdep.c
===================================================================
RCS file: /cvs/src/src/gdb/i386-tdep.c,v
retrieving revision 1.280
diff -u -p -r1.280 i386-tdep.c
--- i386-tdep.c	2 Jul 2009 17:25:54 -0000	1.280
+++ i386-tdep.c	4 Jul 2009 00:16:03 -0000
@@ -1487,6 +1487,89 @@ static const struct frame_unwind i386_fr
   NULL,
   default_frame_sniffer
 };
+
+/* Normal frames, but in a function epilogue.  */
+
+/* The epilogue is defined here as the RET instruction, which will
+   follow any instruction such as LEAVE or POP EBP that destroys the
+   function's stack frame.  */
+
+static int
+i386_in_function_epilogue_p (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  gdb_byte insn;
+
+  if (target_read_memory (pc, &insn, 1) != 0)
+    return 0;	/* Can't read memory at pc.  */
+
+  if (insn != 0xc3)	/* RET */
+    return 0;
+
+  return 1;
+}
+
+static int
+i386_epilogue_frame_sniffer (const struct frame_unwind *self,
+			     struct frame_info *this_frame,
+			     void **this_prologue_cache)
+{
+  if (frame_relative_level (this_frame) == 0)
+    return i386_in_function_epilogue_p (get_frame_arch (this_frame),
+					get_frame_pc (this_frame));
+  else
+    return 0;
+}
+
+static struct i386_frame_cache *
+i386_epilogue_frame_cache (struct frame_info *this_frame, void **this_cache)
+{
+  struct gdbarch *gdbarch = get_frame_arch (this_frame);
+  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
+  struct i386_frame_cache *cache;
+  gdb_byte buf[4];
+
+  if (*this_cache)
+    return *this_cache;
+
+  cache = i386_alloc_frame_cache ();
+  *this_cache = cache;
+
+  /* Cache base will be ESP plus cache->sp_offset (-4).  */
+  get_frame_register (this_frame, I386_ESP_REGNUM, buf);
+  cache->base = extract_unsigned_integer (buf, 4, 
+					  byte_order) + cache->sp_offset;
+
+  /* Cache pc will be the frame func.  */
+  cache->pc = get_frame_pc (this_frame);
+
+  /* The saved ESP will be at cache->base plus 8.  */
+  cache->saved_sp = cache->base + 8;
+
+  /* The saved EIP will be at cache->base plus 4.  */
+  cache->saved_regs[I386_EIP_REGNUM] = cache->base + 4;
+
+  return cache;
+}
+
+static void
+i386_epilogue_frame_this_id (struct frame_info *this_frame,
+			     void **this_cache,
+			     struct frame_id *this_id)
+{
+  struct i386_frame_cache *cache = i386_epilogue_frame_cache (this_frame,
+							      this_cache);
+
+  (*this_id) = frame_id_build (cache->base + 8, cache->pc);
+}
+
+static const struct frame_unwind i386_epilogue_frame_unwind =
+{
+  NORMAL_FRAME,
+  i386_epilogue_frame_this_id,
+  i386_frame_prev_register,
+  NULL, 
+  i386_epilogue_frame_sniffer
+};
 
 
 /* Signal trampolines.  */
@@ -5328,7 +5411,10 @@ i386_gdbarch_init (struct gdbarch_info i
 
   /* Helper for function argument information.  */
   set_gdbarch_fetch_pointer_argument (gdbarch, i386_fetch_pointer_argument);
-
+#if 0
+  /* Hook the function epilogue frame unwinder.  */
+  frame_unwind_append_unwinder (gdbarch, &i386_epilogue_frame_unwind);
+#endif
   /* Hook in the DWARF CFI frame unwinder.  */
   dwarf2_append_unwinders (gdbarch);