[PATCH v2] Fix PR gdb/19250: ptrace prototype is not detected properly in C++ mode

[Pedro Alves <palves at redhat dot com>]

On 04/16/2016 07:47 AM, Andreas Schwab wrote:
> Wouldn't it be easier to split the configure check in two parts, one
> running in C mode that does the detection of the argument and return
> types, and a second part running in C++ mode that probes the need for
> the enum?

Note "easier" is ambiguous.  (Easier to patch vs resulting code easier
to maintain/read.)

I think the resulting code is simpler to understand as I wrote it.

We always need to a separate test for glibc's varargs prototype.
A parameter list with an ellipsis can't match an empty parameter
name list declaration.  So we can't mix that with the argument
types detection for loop.  We could make this:

# Check return type.  Varargs (used on GNU/Linux) conflict with the
# empty argument list, so check for that explicitly.
AC_CACHE_CHECK([return type of ptrace], gdb_cv_func_ptrace_ret,
  AC_TRY_COMPILE($gdb_ptrace_headers,
    [extern long ptrace (enum __ptrace_request, ...);],
    gdb_cv_func_ptrace_ret='long',
    AC_TRY_COMPILE($gdb_ptrace_headers,
      [extern int ptrace ();],
      gdb_cv_func_ptrace_ret='int',
      gdb_cv_func_ptrace_ret='long')))

set gdb_cv_func_ptrace_args as well, instead of duplicating
the enum __ptrace_request test immediately below.  This is
where I think a single gdb_cv_func_ptrace_proto variable
makes things easier to understand.

BUT!  This seems to be all moot and can be relegated to "clean up".

I took another fresh look, and I can't figure out why I needed
to make these tests run in C++ mode in the first place...

So I think that from a correctness perspective, we should push
this patch in instead.

----------

>From dfb5880175911e5ac1616fff9c4c602221e0a811 Mon Sep 17 00:00:00 2001
From: Pedro Alves <palves@redhat.com>
Date: Mon, 18 Apr 2016 15:23:09 +0100
Subject: [PATCH] Fix PR gdb/19250: ptrace prototype is not detected properly
 in C++ mode

The ptrace args/return types detection doesn't work properly in C++
mode, on non-GNU/Linux hosts.  For example, on gcc70 (NetBSD 5.1),
where the prototype is:

 int ptrace(int, __pid_t, void*, int);

configure misdetects it as:

 $ grep PTRACE_TYPE config.h
 #define PTRACE_TYPE_ARG1 int
 #define PTRACE_TYPE_ARG3 int *
 #define PTRACE_TYPE_ARG4 int
 /* #undef PTRACE_TYPE_ARG5 */
 #define PTRACE_TYPE_RET int

resulting in:

 ../../src/gdb/amd64bsd-nat.c: In function 'void amd64bsd_fetch_inferior_registers(target_ops*, regcache*, int)':
 ../../src/gdb/amd64bsd-nat.c:56: warning: dereferencing type-punned pointer will break strict-aliasing rules
 ../../src/gdb/amd64bsd-nat.c: In function 'void amd64bsd_store_inferior_registers(target_ops*, regcache*, int)':
 ../../src/gdb/amd64bsd-nat.c:104: warning: dereferencing type-punned pointer will break strict-aliasing rules
 ../../src/gdb/amd64bsd-nat.c:110: warning: dereferencing type-punned pointer will break strict-aliasing rules

We could address this [1], however despite ptrace.m4's claim:

 # Needs to be tested in C++ mode, to detect whether we need to cast
 # the first argument to enum __ptrace_request.

it appears that there's actually no need to test in C++ mode.  Always
running the ptrace tests in C mode works just the same on GNU/Linux.

I remember experimenting with several different ways to handle the
original issue back then, and maybe that was needed in some other
attempt and then I didn't realize it ended up not really necessary.

Confirmed that this fixes the NetBSD 5.1 C++ build, and confirmed that
C and C++ builds on Fedora 23 are unaffected.

[1] - https://sourceware.org/ml/gdb-patches/2016-04/msg00374.html

gdb/ChangeLog:
2016-04-18  Pedro Alves  <palves@redhat.com>

	* ptrace.m4 (GDB_AC_PTRACE): Don't run tests in C++ mode.
	* configure: Regenerate.

gdb/gdbserver/ChangeLog:
2016-04-18  Pedro Alves  <palves@redhat.com>

	* configure: Regenerate.
---
 gdb/ptrace.m4           | 10 -------
 gdb/configure           | 79 +++++--------------------------------------------
 gdb/gdbserver/configure | 79 +++++--------------------------------------------
 3 files changed, 14 insertions(+), 154 deletions(-)

diff --git a/gdb/ptrace.m4 b/gdb/ptrace.m4
index ca2b7c6..89591d4 100644
--- a/gdb/ptrace.m4
+++ b/gdb/ptrace.m4
@@ -22,12 +22,6 @@ AC_DEFUN([GDB_AC_PTRACE],
 
 AC_CHECK_HEADERS([sys/ptrace.h ptrace.h])
 
-# Needs to be tested in C++ mode, to detect whether we need to cast
-# the first argument to enum __ptrace_request.
-if test "$enable_build_with_cxx" = "yes"; then
-    AC_LANG_PUSH([C++])
-fi
-
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -97,8 +91,4 @@ if test -n "$[5]"; then
   AC_DEFINE_UNQUOTED(PTRACE_TYPE_ARG5, $[5],
     [Define to the type of arg 5 for ptrace.])
 fi
-
-if test "$enable_build_with_cxx" = "yes"; then
-    AC_LANG_POP([C++])
-fi
 ])
diff --git a/gdb/configure b/gdb/configure
index b523deb..8c2ec0f 100755
--- a/gdb/configure
+++ b/gdb/configure
@@ -2401,51 +2401,6 @@ $as_echo "$ac_res" >&6; }
   eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
 
 } # ac_fn_c_check_member
-
-# ac_fn_cxx_check_decl LINENO SYMBOL VAR
-# --------------------------------------
-# Tests whether SYMBOL is declared, setting cache variable VAR accordingly.
-ac_fn_cxx_check_decl ()
-{
-  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
-  as_decl_name=`echo $2|sed 's/ *(.*//'`
-  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
-  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
-$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
-if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then :
-  $as_echo_n "(cached) " >&6
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$4
-int
-main ()
-{
-#ifndef $as_decl_name
-#ifdef __cplusplus
-  (void) $as_decl_use;
-#else
-  (void) $as_decl_name;
-#endif
-#endif
-
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  eval "$3=yes"
-else
-  eval "$3=no"
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-eval ac_res=\$$3
-	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
-$as_echo "$ac_res" >&6; }
-  eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
-
-} # ac_fn_cxx_check_decl
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
@@ -12351,7 +12306,6 @@ fi
 # Check the return and argument types of ptrace.
 
 
-
 for ac_header in sys/ptrace.h ptrace.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
@@ -12367,17 +12321,6 @@ fi
 done
 
 
-# Needs to be tested in C++ mode, to detect whether we need to cast
-# the first argument to enum __ptrace_request.
-if test "$enable_build_with_cxx" = "yes"; then
-    ac_ext=cpp
-ac_cpp='$CXXCPP $CPPFLAGS'
-ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
-
-fi
-
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -12388,7 +12331,7 @@ gdb_ptrace_headers='
 #endif
 '
 # There is no point in checking if we don't have a prototype.
-ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
+ac_fn_c_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
 "
 if test "x$ac_cv_have_decl_ptrace" = x""yes; then :
   ac_have_decl=1
@@ -12426,7 +12369,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='long'
 else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -12440,7 +12383,7 @@ extern int ptrace ();
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='int'
 else
   gdb_cv_func_ptrace_ret='long'
@@ -12474,7 +12417,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
 else
 
@@ -12496,7 +12439,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
     break 4;
 fi
@@ -12516,7 +12459,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
 
 gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
     break 5;
@@ -12562,15 +12505,6 @@ _ACEOF
 
 fi
 
-if test "$enable_build_with_cxx" = "yes"; then
-    ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-fi
-
 
 if test "$cross_compiling" = no; then
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether setpgrp takes no argument" >&5
@@ -14265,6 +14199,7 @@ _ACEOF
 
 
 
+
 # Check whether --enable-werror was given.
 if test "${enable_werror+set}" = set; then :
   enableval=$enable_werror; case "${enableval}" in
diff --git a/gdb/gdbserver/configure b/gdb/gdbserver/configure
index bb01922..2082335 100755
--- a/gdb/gdbserver/configure
+++ b/gdb/gdbserver/configure
@@ -1939,51 +1939,6 @@ $as_echo "$ac_res" >&6; }
 
 } # ac_fn_c_check_decl
 
-# ac_fn_cxx_check_decl LINENO SYMBOL VAR
-# --------------------------------------
-# Tests whether SYMBOL is declared, setting cache variable VAR accordingly.
-ac_fn_cxx_check_decl ()
-{
-  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
-  as_decl_name=`echo $2|sed 's/ *(.*//'`
-  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
-  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
-$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
-if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then :
-  $as_echo_n "(cached) " >&6
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-$4
-int
-main ()
-{
-#ifndef $as_decl_name
-#ifdef __cplusplus
-  (void) $as_decl_use;
-#else
-  (void) $as_decl_name;
-#endif
-#endif
-
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
-  eval "$3=yes"
-else
-  eval "$3=no"
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-eval ac_res=\$$3
-	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
-$as_echo "$ac_res" >&6; }
-  eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
-
-} # ac_fn_cxx_check_decl
-
 # ac_fn_c_compute_int LINENO EXPR VAR INCLUDES
 # --------------------------------------------
 # Tries to find the compile-time value of EXPR in a program that includes
@@ -6030,7 +5985,6 @@ fi
 # Check the return and argument types of ptrace.
 
 
-
 for ac_header in sys/ptrace.h ptrace.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
@@ -6046,17 +6000,6 @@ fi
 done
 
 
-# Needs to be tested in C++ mode, to detect whether we need to cast
-# the first argument to enum __ptrace_request.
-if test "$enable_build_with_cxx" = "yes"; then
-    ac_ext=cpp
-ac_cpp='$CXXCPP $CPPFLAGS'
-ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
-
-fi
-
 gdb_ptrace_headers='
 #include <sys/types.h>
 #if HAVE_SYS_PTRACE_H
@@ -6067,7 +6010,7 @@ gdb_ptrace_headers='
 #endif
 '
 # There is no point in checking if we don't have a prototype.
-ac_fn_cxx_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
+ac_fn_c_check_decl "$LINENO" "ptrace" "ac_cv_have_decl_ptrace" "$gdb_ptrace_headers
 "
 if test "x$ac_cv_have_decl_ptrace" = x""yes; then :
   ac_have_decl=1
@@ -6105,7 +6048,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='long'
 else
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -6119,7 +6062,7 @@ extern int ptrace ();
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_ret='int'
 else
   gdb_cv_func_ptrace_ret='long'
@@ -6153,7 +6096,7 @@ extern long ptrace (enum __ptrace_request, ...);
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args='enum __ptrace_request,int,long,long'
 else
 
@@ -6175,7 +6118,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
   gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4";
     break 4;
 fi
@@ -6195,7 +6138,7 @@ extern $gdb_cv_func_ptrace_ret
   return 0;
 }
 _ACEOF
-if ac_fn_cxx_try_compile "$LINENO"; then :
+if ac_fn_c_try_compile "$LINENO"; then :
 
 gdb_cv_func_ptrace_args="$gdb_arg1,$gdb_arg2,$gdb_arg3,$gdb_arg4,$gdb_arg5";
     break 5;
@@ -6241,15 +6184,6 @@ _ACEOF
 
 fi
 
-if test "$enable_build_with_cxx" = "yes"; then
-    ac_ext=c
-ac_cpp='$CPP $CPPFLAGS'
-ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-fi
-
 
 # Check for UST
 ustlibs=""
@@ -6332,6 +6266,7 @@ fi
 
 
 
+
 # Check whether --enable-werror was given.
 if test "${enable_werror+set}" = set; then :
   enableval=$enable_werror; case "${enableval}" in
-- 
2.5.5