[RFC] i386-tdep.c: Fix handling of ENTER instruction

["Pierre Muller" <muller at ics dot u-strasbg dot fr>]

  This small patch simply aligns the handling
of the enter instruction:
	enter $localsize,0
to its equivalent, i.e.:
	pushl %ebp
	movl  %esp,%ebp
	subl  $localsize,%esp

I also wrote a test showing the problem
that is generated by the missing two lines in i386_analyze_frame_setup.

  I know that enter is a rather odd i386 instruction,
that gcc never uses it, that Linux OS doesn't support it.
(My only directly interaction with Linus Torwald was about 
adding enter specific support into Linux kernel, and
the patch was rejected...)
See
http://groups.google.co.nz/group/comp.os.linux.development.system/browse_thr
ead/thread/a057249198598933/a4f5251c9ef1e7a2

  But I think that gdb, as a general purpose debugger,
should handle the best it can, all instructions, even
if only used rarely and more or less banned for some OS.

  The test that I added gives one failure
with current CVS HEAD that is fixed by
this patch. There are no other FAIL changes
when I ran the testsuite on cygwin.

  I have a few questions, about the test:
 1) Are the names of the tests OK, or should I use simply
i386-enter-XXX?
 2) The current test only checks that main is in the backtrace,
should I write something more precise or is this enough?

  All comments are welcome.


Pierre Muller
Pascal language support maintainer for GDB



gdb/ChangeLog entry:

2008-06-28  Pierre Muller  <muller@ics.u-strasbg.fr>

	i386-tdep.c: (i386_analyze_frame_setup) Fix handling of
	enter instruction.

gdb/testsuite/ChangeLog entry:

2008-06-28  Pierre Muller  <muller@ics.u-strasbg.fr>

	gdb.base/i386-test-enter-s.S: New file.
	gdb.base/i386-test-enter.c: New file.
	gdb.base/i386-test-enter.exp: New test file.


Index: gdb/i386-tdep.c
===================================================================
RCS file: /cvs/src/src/gdb/i386-tdep.c,v
retrieving revision 1.259
diff -u -p -r1.259 i386-tdep.c
--- gdb/i386-tdep.c	11 Jun 2008 22:03:49 -0000	1.259
+++ gdb/i386-tdep.c	27 Jun 2008 14:05:21 -0000
@@ -1005,6 +1005,8 @@ i386_analyze_frame_setup (CORE_ADDR pc, 
     }
   else if (op == 0xc8)		/* enter */
     {
+      cache->saved_regs[I386_EBP_REGNUM] = 0;
+      cache->sp_offset += 4;
       cache->locals = read_memory_unsigned_integer (pc + 1, 2);
       return pc + 4;
     }
Index: gdb/testsuite/gdb.arch/i386-test-enter-s.S
===================================================================
RCS file: gdb/testsuite/gdb.arch/i386-test-enter-s.S
diff -N gdb/testsuite/gdb.arch/i386-test-enter-s.S
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ gdb/testsuite/gdb.arch/i386-test-enter-s.S	27 Jun 2008 14:05:22 -0000
@@ -0,0 +1,37 @@
+	.text
+
+
+.globl _level2
+_level2:
+	pushl	%ebp
+	movl	%esp,%ebp
+	subl	$16,%esp
+	leave
+	ret
+
+.globl _test1
+_test1:
+	pushl	%ebp
+	movl	%esp,%ebp
+	subl	$8,%esp
+	call	_level2
+	leave
+	ret
+
+.globl _test2
+_test2:
+	enter   $8,$0
+	call	_level2
+	leave
+	ret
+
+.globl _testenter
+_testenter:
+	pushl	%ebp
+	movl	%esp,%ebp
+	subl	$8,%esp
+	call	_test1
+	call	_test2
+	movl	$1, %eax
+	leave
+	ret
Index: gdb/testsuite/gdb.arch/i386-test-enter.c
===================================================================
RCS file: gdb/testsuite/gdb.arch/i386-test-enter.c
diff -N gdb/testsuite/gdb.arch/i386-test-enter.c
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ gdb/testsuite/gdb.arch/i386-test-enter.c	27 Jun 2008 14:05:22 -0000
@@ -0,0 +1,11 @@
+
+extern int testenter ();
+
+int
+main ()
+{
+  int res;
+  res = testenter ();
+  return res;
+}
+
Index: gdb/testsuite/gdb.arch/i386-test-enter.exp
===================================================================
RCS file: gdb/testsuite/gdb.arch/i386-test-enter.exp
diff -N gdb/testsuite/gdb.arch/i386-test-enter.exp
--- /dev/null	1 Jan 1970 00:00:00 -0000
+++ gdb/testsuite/gdb.arch/i386-test-enter.exp	27 Jun 2008 14:05:22 -0000
@@ -0,0 +1,78 @@
+# Copyright 2004, 2005, 2007, 2008 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Please email any bugs, comments, and/or additions to this file to:
+# bug-gdb@gnu.org
+
+# This file is part of the gdb testsuite.
+
+if $tracelevel {
+    strace $tracelevel
+}
+
+set prms_id 0
+set bug_id 0
+
+if ![istarget "i?86-*-*"] then {
+    verbose "Skipping i386 ENTER instruction test."
+    return
+}
+
+set testfile "i386-test-enter"
+set srcfile ${testfile}.c
+set testfile2 "i386-test-enter-s"
+set srcfile2 ${testfile2}.S
+set objfile2 ${testfile2}.o
+
+set binfile ${objdir}/${subdir}/${testfile}
+
+if [get_compiler_info ${binfile}] {
+    return -1
+}
+
+if { [gdb_compile "${srcdir}/${subdir}/${srcfile2}" "${objfile2}" object
[list debug $additional_flags]] != "" } {
+    unsupported "compilation failed"
+    return
+}
+
+
+if { [gdb_compile "${srcdir}/${subdir}/${srcfile} ${objfile2}" "${binfile}"
executable [list debug $additional_flags]] != "" } {
+    unsupported "compilation failed"
+    return
+}
+
+gdb_exit
+gdb_start
+gdb_reinitialize_dir $srcdir/$subdir
+gdb_load ${binfile}
+
+gdb_test "break level2" \
+         "Breakpoint .*${srcfile2}.*" \
+         "set breakpoint level2"
+
+gdb_test "run" ".* level2 .*" "run to level2 in test1"
+
+gdb_test "backtrace" \
+	".* main .*" \
+	"Test main in backtrace 1"
+
+gdb_test "continue" "Breakpoint .*${srcfile2}.*" \
+	"continue to level2 in test2"
+
+gdb_test "backtrace" \
+	".* main .*" \
+	"Test main in backtrace 2"
+
+gdb_exit