[patch+7.5] Fix assertion fail on PaX gentoo (PR 14564)

[Jan Kratochvil]

Hi,

Anton Kochkov has bugreported and tested 7.5 regression:
	http://sourceware.org/bugzilla/show_bug.cgi?id=14564
	./common/linux-ptrace.c:117: internal-error: linux_ptrace_test_ret_to_nx: Assertion `WIFSTOPPED (status)' failed.
due to:
	[2396058.354252] PAX: execution attempt in: <anonymous mapping>, aa1d2000-aa1d5000 aa1d2000
	[2396058.354259] PAX: terminating task: /usr/bin/gdb(gdb):15808, uid/euid: 1000/1000, PC: aa1d2000, SP: bb6d5690

I expect it affects all archs but the patch below tests for the warning
message only x86_64.  Tests for other archs would be nice.

It is still just about printing a warning, inferior calls with
stack-non-executable inferior will still not work.

I will check it in, it may be worth 7.5.1 as it blocks 7.5 at least on Gentoo
Hardened, as it has been bugreported.


Thanks,
Jan


2012-09-09  Jan Kratochvil  <jan.kratochvil@redhat.com>

	* common/linux-ptrace.c: Change __i386__ to __i386__ || __x86_64__.
	(linux_ptrace_test_ret_to_nx): Extend comment for x86_64.  Change
	__i386__ to __i386__ || __x86_64__. Extend code also for __x86_64__.
	Extend code also for PaX support.

diff --git a/gdb/common/linux-ptrace.c b/gdb/common/linux-ptrace.c
index fdec5d6..9bfe19f 100644
--- a/gdb/common/linux-ptrace.c
+++ b/gdb/common/linux-ptrace.c
@@ -49,7 +49,7 @@ linux_ptrace_attach_warnings (pid_t pid, struct buffer *buffer)
 		       (int) pid);
 }
 
-#ifdef __i386__
+#if defined __i386__ || defined __x86_64__
 
 /* Address of the 'ret' instruction in asm code block below.  */
 extern void (linux_ptrace_test_ret_to_nx_instr) (void);
@@ -60,15 +60,17 @@ extern void (linux_ptrace_test_ret_to_nx_instr) (void);
 #include <sys/wait.h>
 #include <stdint.h>
 
-#endif /* __i386__ */
+#endif /* defined __i386__ || defined __x86_64__ */
 
 /* Test broken off-trunk Linux kernel patchset for NX support on i386.  It was
-   removed in Fedora kernel 88fa1f0332d188795ed73d7ac2b1564e11a0b4cd.  */
+   removed in Fedora kernel 88fa1f0332d188795ed73d7ac2b1564e11a0b4cd.
+
+   Test also x86_64 arch for PaX support.  */
 
 static void
 linux_ptrace_test_ret_to_nx (void)
 {
-#ifdef __i386__
+#if defined __i386__ || defined __x86_64__
   pid_t child, got_pid;
   gdb_byte *return_address, *pc;
   long l;
@@ -101,11 +103,21 @@ linux_ptrace_test_ret_to_nx (void)
 		 strerror (errno));
       else
 	{
+#if defined __i386__
 	  asm volatile ("pushl %0;"
 			".globl linux_ptrace_test_ret_to_nx_instr;"
 			"linux_ptrace_test_ret_to_nx_instr:"
 			"ret"
 			: : "r" (return_address) : "%esp", "memory");
+#elif defined __x86_64__
+	  asm volatile ("pushq %0;"
+			".globl linux_ptrace_test_ret_to_nx_instr;"
+			"linux_ptrace_test_ret_to_nx_instr:"
+			"ret"
+			: : "r" (return_address) : "%rsp", "memory");
+#else
+# error "!__i386__ && !__x86_64__"
+#endif
 	  gdb_assert_not_reached ("asm block did not terminate");
 	}
 
@@ -114,13 +126,29 @@ linux_ptrace_test_ret_to_nx (void)
 
   got_pid = waitpid (child, &status, 0);
   gdb_assert (got_pid == child);
+
+  if (WIFSIGNALED (status))
+    {
+      gdb_assert (WTERMSIG (status) == SIGKILL);
+
+      warning (_("Cannot call inferior functions, Linux kernel PaX protection "
+		 "forbids return to non-executable pages!"));
+      return;
+    }
+
   gdb_assert (WIFSTOPPED (status));
 
   /* We may get SIGSEGV due to missing PROT_EXEC of the return_address.  */
   gdb_assert (WSTOPSIG (status) == SIGTRAP || WSTOPSIG (status) == SIGSEGV);
 
   errno = 0;
+#if defined __i386__
   l = ptrace (PTRACE_PEEKUSER, child, (void *) (uintptr_t) (EIP * 4), NULL);
+#elif defined __x86_64__
+  l = ptrace (PTRACE_PEEKUSER, child, (void *) (uintptr_t) (RIP * 8), NULL);
+#else
+# error "!__i386__ && !__x86_64__"
+#endif
   gdb_assert (errno == 0);
   pc = (void *) (uintptr_t) l;
 
@@ -154,7 +182,7 @@ linux_ptrace_test_ret_to_nx (void)
 
   warning (_("Cannot call inferior functions, you have broken "
 	     "Linux kernel i386 NX (non-executable pages) support!"));
-#endif /* __i386__ */
+#endif /* defined __i386__ || defined __x86_64__ */
 }
 
 /* Display possible problems on this system.  Display them only once per GDB