[PATCH] infrun: step through indirect branch thunks

[Markus Metzger <markus dot t dot metzger at intel dot com>]

With version 7.3 GCC supports new options

   -mindirect-branch=<choice>
   -mfunction-return=<choice>

The choices are:

    keep                behaves as before
    thunk               jumps through a thunk
    thunk-external      jumps through an external thunk
    thunk-inline        jumps through an inlined thunk

For thunk and thunk-external, GDB would, on a call to the thunk, step into the
thunk and then resume to its caller assuming that this is an undebuggable
function.  On a return thunk, GDB would stop inside the thunk.

Make GDB step through such thunks instead.

Before:
    Temporary breakpoint 1, main ()
        at gdb.base/step-indirect-call-thunk.c:37
    37        x = apply (inc, 41);
    (gdb) s
    apply (op=0x80483e6 <inc>, x=41)
        at gdb.base/step-indirect-call-thunk.c:29
    29        return op (x);
    (gdb)
    30      }

After:
    Temporary breakpoint 1, main ()
        at gdb.base/step-indirect-call-thunk.c:37
    37        x = apply (inc, 41);
    (gdb) s
    apply (op=0x80483e6 <inc>, x=41)
        at gdb.base/step-indirect-call-thunk.c:29
    29        return op (x);
    (gdb)
    inc (x=41) at gdb.base/step-indirect-call-thunk.c:23
    23        return x + 1;

This is independent of the step-mode.  In order to step into the thunk, you
would need to use stepi.

When stepping over an indirect call thunk, GDB would first step through the
thunk, then recognize that it stepped into a sub-routine and resume to the
caller (of the thunk).  Not sure whether this is worth optimizing.

Thunk detection is implemented via gdbarch.  I implemented the methods for IA.
Other architectures may run into unexpected fails.

The tests assume a fixed number of instruction steps to reach a thunk.  This
depends on the compiler as well as the architecture.  They may need adjustments
when we add support for more architectures.  Or we can simply drop those tests
that cover being able to step into thunks using instruction stepping.

When using an older GCC, the tests will fail and be reported as untested:

    Running .../gdb.base/step-indirect-call-thunk.exp ...
    gdb compile failed, \
    gcc: error: unrecognized command line option '-mindirect-branch=thunk'
    gcc: error: unrecognized command line option '-mfunction-return=thunk'

                    === gdb Summary ===

    # of untested testcases         1

2018-02-16  Markus Metzger  <markus.t.metzger@intel.com>

gdb/
	* infrun.c (in_indirect_branch_thunk): New.
	(process_event_stop_test): Call in_indirect_branch_thunk.
	* gdbarch.sh (in_indirect_branch_thunk): New.
	* gdbarch.c: Regenerated.
	* gdbarch.h: Regenerated.
	* i386-tdep (i386_is_thunk_regiser_name)
	(i386_in_indirect_branch_thunk): New.
	(i386_elf_init_abi): Set in_indirect_branch_thunk gdbarch function.
	* amd64-tdep (amd64_is_thunk_register_name)
	(amd64_in_indirect_branch_thunk): New.
	(amd64_init_abi): Set in_indirect_branch_thunk gdbarch function.

testsuite/
	* gdb.base/step-indirect-call-thunk.exp: New.
	* gdb.base/step-indirect-call-thunk.c: New.
	* gdb.reverse/step-indirect-call-thunk.exp: New.
	* gdb.reverse/step-indirect-call-thunk.c: New.
---
 gdb/amd64-tdep.c                                   | 54 +++++++++++++++++++
 gdb/gdbarch.c                                      | 32 ++++++++++++
 gdb/gdbarch.h                                      |  8 +++
 gdb/gdbarch.sh                                     |  3 ++
 gdb/i386-tdep.c                                    | 54 +++++++++++++++++++
 gdb/infrun.c                                       | 22 ++++++++
 gdb/testsuite/gdb.base/step-indirect-call-thunk.c  | 41 +++++++++++++++
 .../gdb.base/step-indirect-call-thunk.exp          | 33 ++++++++++++
 .../gdb.reverse/step-indirect-call-thunk.c         | 36 +++++++++++++
 .../gdb.reverse/step-indirect-call-thunk.exp       | 60 ++++++++++++++++++++++
 10 files changed, 343 insertions(+)
 create mode 100644 gdb/testsuite/gdb.base/step-indirect-call-thunk.c
 create mode 100644 gdb/testsuite/gdb.base/step-indirect-call-thunk.exp
 create mode 100644 gdb/testsuite/gdb.reverse/step-indirect-call-thunk.c
 create mode 100644 gdb/testsuite/gdb.reverse/step-indirect-call-thunk.exp

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index b589d93..8bd7109 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -3032,6 +3032,57 @@ static const int amd64_record_regmap[] =
   AMD64_DS_REGNUM, AMD64_ES_REGNUM, AMD64_FS_REGNUM, AMD64_GS_REGNUM
 };
 
+/* Check whether NAME is a register used in an indirect branch thunk.  */
+
+static int
+amd64_is_thunk_register_name (const char *name)
+{
+  int reg;
+  for (reg = AMD64_RAX_REGNUM; reg < AMD64_RIP_REGNUM; ++reg)
+    if (strcmp (name, amd64_register_names[reg]) == 0)
+      return 1;
+
+  return 0;
+}
+
+/* Implement the "in_indirect_branch_thunk" gdbarch function.  */
+
+static int
+amd64_in_indirect_branch_thunk (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  struct bound_minimal_symbol bmfun = lookup_minimal_symbol_by_pc (pc);
+  if (bmfun.minsym == nullptr)
+    return 0;
+
+  const char *name = MSYMBOL_LINKAGE_NAME (bmfun.minsym);
+  if (name == nullptr)
+    return 0;
+
+  /* Check the indirect return thunk first.  */
+  if (strcmp (name, "__x86_return_thunk") == 0)
+    return 1;
+
+  /* Then check a family of indirect call/jump thunks.  */
+  static const char thunk[] = "__x86_indirect_thunk";
+  static const size_t length = sizeof (thunk) - 1;
+  if (strncmp (name, thunk, length) != 0)
+    return 0;
+
+  /* If that's the complete name, we're in the memory thunk.  */
+  name += length;
+  if (*name == 0)
+    return 1;
+
+  /* Check for suffixes.  */
+  if (*name++ != '_')
+    return 0;
+
+  if (amd64_is_thunk_register_name (name))
+    return 1;
+
+  return 0;
+}
+
 void
 amd64_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch,
 		const target_desc *default_tdesc)
@@ -3204,6 +3255,9 @@ amd64_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch,
   set_gdbarch_insn_is_call (gdbarch, amd64_insn_is_call);
   set_gdbarch_insn_is_ret (gdbarch, amd64_insn_is_ret);
   set_gdbarch_insn_is_jump (gdbarch, amd64_insn_is_jump);
+
+  set_gdbarch_in_indirect_branch_thunk (gdbarch,
+					amd64_in_indirect_branch_thunk);
 }
 
 
diff --git a/gdb/gdbarch.c b/gdb/gdbarch.c
index fe3c12e..6243a41 100644
--- a/gdb/gdbarch.c
+++ b/gdb/gdbarch.c
@@ -266,6 +266,7 @@ struct gdbarch
   gdbarch_skip_trampoline_code_ftype *skip_trampoline_code;
   gdbarch_skip_solib_resolver_ftype *skip_solib_resolver;
   gdbarch_in_solib_return_trampoline_ftype *in_solib_return_trampoline;
+  gdbarch_in_indirect_branch_thunk_ftype *in_indirect_branch_thunk;
   gdbarch_stack_frame_destroyed_p_ftype *stack_frame_destroyed_p;
   gdbarch_elf_make_msymbol_special_ftype *elf_make_msymbol_special;
   gdbarch_coff_make_msymbol_special_ftype *coff_make_msymbol_special;
@@ -627,6 +628,7 @@ verify_gdbarch (struct gdbarch *gdbarch)
   /* Skip verify of skip_trampoline_code, invalid_p == 0 */
   /* Skip verify of skip_solib_resolver, invalid_p == 0 */
   /* Skip verify of in_solib_return_trampoline, invalid_p == 0 */
+  /* Skip verify of in_indirect_branch_thunk, has predicate.  */
   /* Skip verify of stack_frame_destroyed_p, invalid_p == 0 */
   /* Skip verify of elf_make_msymbol_special, has predicate.  */
   /* Skip verify of coff_make_msymbol_special, invalid_p == 0 */
@@ -1103,6 +1105,12 @@ gdbarch_dump (struct gdbarch *gdbarch, struct ui_file *file)
                       "gdbarch_dump: have_nonsteppable_watchpoint = %s\n",
                       plongest (gdbarch->have_nonsteppable_watchpoint));
   fprintf_unfiltered (file,
+                      "gdbarch_dump: gdbarch_in_indirect_branch_thunk_p() = %d\n",
+                      gdbarch_in_indirect_branch_thunk_p (gdbarch));
+  fprintf_unfiltered (file,
+                      "gdbarch_dump: in_indirect_branch_thunk = <%s>\n",
+                      host_address_to_string (gdbarch->in_indirect_branch_thunk));
+  fprintf_unfiltered (file,
                       "gdbarch_dump: in_solib_return_trampoline = <%s>\n",
                       host_address_to_string (gdbarch->in_solib_return_trampoline));
   fprintf_unfiltered (file,
@@ -3354,6 +3362,30 @@ set_gdbarch_in_solib_return_trampoline (struct gdbarch *gdbarch,
 }
 
 int
+gdbarch_in_indirect_branch_thunk_p (struct gdbarch *gdbarch)
+{
+  gdb_assert (gdbarch != NULL);
+  return gdbarch->in_indirect_branch_thunk != NULL;
+}
+
+int
+gdbarch_in_indirect_branch_thunk (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  gdb_assert (gdbarch != NULL);
+  gdb_assert (gdbarch->in_indirect_branch_thunk != NULL);
+  if (gdbarch_debug >= 2)
+    fprintf_unfiltered (gdb_stdlog, "gdbarch_in_indirect_branch_thunk called\n");
+  return gdbarch->in_indirect_branch_thunk (gdbarch, pc);
+}
+
+void
+set_gdbarch_in_indirect_branch_thunk (struct gdbarch *gdbarch,
+                                      gdbarch_in_indirect_branch_thunk_ftype in_indirect_branch_thunk)
+{
+  gdbarch->in_indirect_branch_thunk = in_indirect_branch_thunk;
+}
+
+int
 gdbarch_stack_frame_destroyed_p (struct gdbarch *gdbarch, CORE_ADDR addr)
 {
   gdb_assert (gdbarch != NULL);
diff --git a/gdb/gdbarch.h b/gdb/gdbarch.h
index 5664c4d..b080717 100644
--- a/gdb/gdbarch.h
+++ b/gdb/gdbarch.h
@@ -741,6 +741,14 @@ typedef int (gdbarch_in_solib_return_trampoline_ftype) (struct gdbarch *gdbarch,
 extern int gdbarch_in_solib_return_trampoline (struct gdbarch *gdbarch, CORE_ADDR pc, const char *name);
 extern void set_gdbarch_in_solib_return_trampoline (struct gdbarch *gdbarch, gdbarch_in_solib_return_trampoline_ftype *in_solib_return_trampoline);
 
+/* Return non-zero if PC lies inside an indirect branch thunk. */
+
+extern int gdbarch_in_indirect_branch_thunk_p (struct gdbarch *gdbarch);
+
+typedef int (gdbarch_in_indirect_branch_thunk_ftype) (struct gdbarch *gdbarch, CORE_ADDR pc);
+extern int gdbarch_in_indirect_branch_thunk (struct gdbarch *gdbarch, CORE_ADDR pc);
+extern void set_gdbarch_in_indirect_branch_thunk (struct gdbarch *gdbarch, gdbarch_in_indirect_branch_thunk_ftype *in_indirect_branch_thunk);
+
 /* A target might have problems with watchpoints as soon as the stack
    frame of the current function has been destroyed.  This mostly happens
    as the first action in a function's epilogue.  stack_frame_destroyed_p()
diff --git a/gdb/gdbarch.sh b/gdb/gdbarch.sh
index a929e13..0b71df7 100755
--- a/gdb/gdbarch.sh
+++ b/gdb/gdbarch.sh
@@ -660,6 +660,9 @@ m;CORE_ADDR;skip_solib_resolver;CORE_ADDR pc;pc;;generic_skip_solib_resolver;;0
 # Some systems also have trampoline code for returning from shared libs.
 m;int;in_solib_return_trampoline;CORE_ADDR pc, const char *name;pc, name;;generic_in_solib_return_trampoline;;0
 
+# Return non-zero if PC lies inside an indirect branch thunk.
+M;int;in_indirect_branch_thunk;CORE_ADDR pc;pc
+
 # A target might have problems with watchpoints as soon as the stack
 # frame of the current function has been destroyed.  This mostly happens
 # as the first action in a function's epilogue.  stack_frame_destroyed_p()
diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index cd56642..36d5855 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -4421,6 +4421,57 @@ i386_gnu_triplet_regexp (struct gdbarch *gdbarch)
 
 
 
+/* Check whether NAME is a register used in an indirect branch thunk.  */
+
+static int
+i386_is_thunk_register_name (const char *name)
+{
+  int reg;
+  for (reg = I386_EAX_REGNUM; reg < I386_EIP_REGNUM; ++reg)
+    if (strcmp (name, i386_register_names[reg]) == 0)
+      return 1;
+
+  return 0;
+}
+
+/* Implement the "in_indirect_branch_thunk" gdbarch function.  */
+
+static int
+i386_in_indirect_branch_thunk (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  struct bound_minimal_symbol bmfun = lookup_minimal_symbol_by_pc (pc);
+  if (bmfun.minsym == nullptr)
+    return 0;
+
+  const char *name = MSYMBOL_LINKAGE_NAME (bmfun.minsym);
+  if (name == nullptr)
+    return 0;
+
+  /* Check the indirect return thunk first.  */
+  if (strcmp (name, "__x86_return_thunk") == 0)
+    return 1;
+
+  /* Then check a family of indirect call/jump thunks.  */
+  static const char thunk[] = "__x86_indirect_thunk";
+  static const size_t length = sizeof (thunk) - 1;
+  if (strncmp (name, thunk, length) != 0)
+    return 0;
+
+  /* If that's the complete name, we're in the memory thunk.  */
+  name += length;
+  if (*name == 0)
+    return 1;
+
+  /* Check for suffixes.  */
+  if (*name++ != '_')
+    return 0;
+
+  if (i386_is_thunk_register_name (name))
+    return 1;
+
+  return 0;
+}
+
 /* Generic ELF.  */
 
 void
@@ -4447,6 +4498,9 @@ i386_elf_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch)
 				      i386_stap_is_single_operand);
   set_gdbarch_stap_parse_special_token (gdbarch,
 					i386_stap_parse_special_token);
+
+  set_gdbarch_in_indirect_branch_thunk (gdbarch,
+					i386_in_indirect_branch_thunk);
 }
 
 /* System V Release 4 (SVR4).  */
diff --git a/gdb/infrun.c b/gdb/infrun.c
index 1bc860b..e819b0d 100644
--- a/gdb/infrun.c
+++ b/gdb/infrun.c
@@ -6186,6 +6186,17 @@ handle_signal_stop (struct execution_control_state *ecs)
   process_event_stop_test (ecs);
 }
 
+/* Check whether PC lies inside an indirect branch thunk.  */
+
+static int
+in_indirect_branch_thunk (struct gdbarch *gdbarch, CORE_ADDR pc)
+{
+  if (!gdbarch_in_indirect_branch_thunk_p (gdbarch))
+    return 0;
+
+  return gdbarch_in_indirect_branch_thunk (gdbarch, pc);
+}
+
 /* Come here when we've got some debug event / signal we can explain
    (IOW, not a random signal), and test whether it should cause a
    stop, or whether we should resume the inferior (transparently).
@@ -6569,6 +6580,17 @@ process_event_stop_test (struct execution_control_state *ecs)
       return;
     }
 
+  /* Step through an indirect branch thunk.  */
+  if (ecs->event_thread->control.step_over_calls != STEP_OVER_NONE
+      && in_indirect_branch_thunk (gdbarch, stop_pc))
+    {
+      if (debug_infrun)
+	 fprintf_unfiltered (gdb_stdlog,
+			     "infrun: stepped into indirect branch thunk\n");
+      keep_going (ecs);
+      return;
+    }
+
   if (ecs->event_thread->control.step_range_end != 1
       && (ecs->event_thread->control.step_over_calls == STEP_OVER_UNDEBUGGABLE
 	  || ecs->event_thread->control.step_over_calls == STEP_OVER_ALL)
diff --git a/gdb/testsuite/gdb.base/step-indirect-call-thunk.c b/gdb/testsuite/gdb.base/step-indirect-call-thunk.c
new file mode 100644
index 0000000..a43b3b3
--- /dev/null
+++ b/gdb/testsuite/gdb.base/step-indirect-call-thunk.c
@@ -0,0 +1,41 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright 2018 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+static int
+inc (int x)
+{                /* inc.1 */
+  return x + 1;  /* inc.2 */
+}                /* inc.3 */
+
+static int
+twice (int (*op)(int), int x)
+{
+  x = op (x);     /* twice.1 */
+  return op (x);  /* twice.2 */
+}                 /* twice.3 */
+
+int
+main ()
+{
+  int x;
+
+  x = twice (inc, 40);
+
+  return x;
+}
diff --git a/gdb/testsuite/gdb.base/step-indirect-call-thunk.exp b/gdb/testsuite/gdb.base/step-indirect-call-thunk.exp
new file mode 100644
index 0000000..4bc04ba
--- /dev/null
+++ b/gdb/testsuite/gdb.base/step-indirect-call-thunk.exp
@@ -0,0 +1,33 @@
+# Copyright 2018 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+standard_testfile
+
+set cflags "-mindirect-branch=thunk -mfunction-return=thunk"
+if { [prepare_for_testing "failed to prepare" $testfile $srcfile \
+        [list debug "additional_flags=$cflags"]] } {
+    return -1
+}
+
+if { ![runto_main] } {
+    untested "failed to run to main"
+    return -1
+}
+
+gdb_test "step" "twice\.1.*" "step into twice ()"
+gdb_test "next" "twice\.2.*" "step through thunks and over inc ()"
+gdb_test "step" "inc\.2.*" "step through call thunk into inc ()"
+gdb_test "step" "inc\.3.*" "step inside inc ()"
+gdb_test "step" "twice\.3.*" "step through return thunk back into twice ()"
diff --git a/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.c b/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.c
new file mode 100644
index 0000000..85464c3
--- /dev/null
+++ b/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.c
@@ -0,0 +1,36 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright 2018 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+static int
+inc (int x)
+{                /* inc.1 */
+  return x + 1;  /* inc.2 */
+}                /* inc.3 */
+
+static int
+apply (int (*op)(int), int x)
+{                 /* apply.1 */
+  return op (x);  /* apply.2 */
+}                 /* apply.3 */
+
+int
+main ()
+{                         /* main.1 */
+  return apply (inc, 41); /* main.2 */
+}                         /* main.3 */
diff --git a/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.exp b/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.exp
new file mode 100644
index 0000000..40a0237
--- /dev/null
+++ b/gdb/testsuite/gdb.reverse/step-indirect-call-thunk.exp
@@ -0,0 +1,60 @@
+# Copyright 2018 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if { ![supports_reverse] } {
+	untested "target does not support record"
+    return -1
+}
+
+standard_testfile
+
+set cflags "-mindirect-branch=thunk -mfunction-return=thunk"
+if { [prepare_for_testing "failed to prepare" $testfile $srcfile \
+        [list debug "additional_flags=$cflags"]] } {
+    return -1
+}
+
+if { ![runto_main] } {
+    untested "failed to run to main"
+    return -1
+}
+
+gdb_test_no_output "record"
+gdb_test "next" ".*" "record trace"
+
+# Normal stepping steps through all thunks.
+gdb_test "reverse-step" "apply\.3.*" "reverse-step into apply ()"
+gdb_test "reverse-step" "inc\.3.*" "reverse-step into inc ()"
+gdb_test "reverse-step" "inc\.2.*" "reverse-step inside inc ()"
+gdb_test "reverse-step" "apply\.2.*" "reverse-step through call thunk into apply ()"
+gdb_test "reverse-step" "main\.2.*" "reverse-step into main ()"
+
+gdb_test "step" "apply\.2.*" "step into apply ()"
+gdb_test "step" "inc\.2.*" "step through call thunk into inc ()"
+
+# We can step into the call thunk using instruction stepping.
+gdb_test "reverse-stepi 5" "indirect_thunk.*" "reverse-stepi into call thunk"
+gdb_test "stepi 5" "inc\.2.*" "stepi out of call thunk into inc ()"
+
+gdb_test "step" "inc\.3.*" "step inside inc ()"
+
+# We can also step into the return thunk using instruction stepping
+gdb_test "stepi 3" "return_thunk.*" "stepi into return thunk"
+gdb_test "reverse-stepi 3" "inc\.3.*" "reverse-stepi out of return thunk into inc ()"
+
+gdb_test "reverse-step" "inc\.2.*" "reverse-step inside inc ()"
+gdb_test "reverse-step" "apply\.2.*" "reverse-step through call thunk into apply ()"
+gdb_test "next" "apply\.3.*" "step through thunks and over inc ()"
+gdb_test "reverse-next" "apply\.2.*" "reverse-step through thunks and over inc ()"
-- 
1.8.3.1