Hi,
This patch adds FMA support in fma. I added __libc_cpu_features and
__libc_init_cpu_features so that I can use them in libm. Tested on
AVX emulator.
Thanks.
H.J.
---
2009-07-29 H.J. Lu <hongjiu.lu@intel.com>
* math/s_fma.c (FMA): New.
(__fma): Renamed to ...
(FMA): This.
* sysdeps/x86_64/multiarch/Makefile (CFLAGS-init-arch.c): New.
(libm-sysdep_routines): Add s_fma-c.
* sysdeps/x86_64/multiarch/init-arch.c (__libc_cpu_features): New.
(__libc_init_cpu_features): Likewise.
* sysdeps/x86_64/multiarch/init-arch.h (__libc_cpu_features): New.
(__libc_init_cpu_features): Likewise.
(INIT_ARCH): Use __libc_cpu_features and __libc_init_cpu_features
if NOT_IN_libc is defined.
(HAS_POPCOUNT): Use __libc_cpu_features.cpuid if NOT_IN_libc is
defined.
(HAS_SSE4_2): Likewise.
(HAS_FMA): New.
* sysdeps/x86_64/multiarch/Versions: New.
* sysdeps/x86_64/multiarch/s_fma-c.c: Likewise.
* sysdeps/x86_64/multiarch/s_fma.c: Likewise.
diff --git a/math/s_fma.c b/math/s_fma.c
index e5ff5a7..407acdd 100644
--- a/math/s_fma.c
+++ b/math/s_fma.c
@@ -1,5 +1,5 @@
/* Compute x * y + z as ternary operation.
- Copyright (C) 1997, 2001 Free Software Foundation, Inc.
+ Copyright (C) 1997, 2001, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -20,8 +20,12 @@
#include <math.h>
+#ifndef FMA
+# define FMA __fma
+#endif
+
double
-__fma (double x, double y, double z)
+FMA (double x, double y, double z)
{
return (x * y) + z;
}
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index b066402..dad0731 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -1,5 +1,6 @@
ifeq ($(subdir),csu)
aux += init-arch
+CFLAGS-init-arch.c += -fno-common
gen-as-const-headers += ifunc-defines.sym
endif
@@ -14,3 +15,9 @@ CFLAGS-strstr.c += -msse4
CFLAGS-strcasestr.c += -msse4
endif
endif
+
+ifeq ($(subdir),math)
+ifeq (yes,$(config-cflags-avx))
+libm-sysdep_routines += s_fma-c
+endif
+endif
diff --git a/sysdeps/x86_64/multiarch/Versions b/sysdeps/x86_64/multiarch/Versions
new file mode 100644
index 0000000..0d2d71a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/Versions
@@ -0,0 +1,6 @@
+libc {
+ GLIBC_PRIVATE {
+ # Internal libc interface to libm
+ __libc_init_cpu_features; __libc_cpu_features;
+ }
+}
diff --git a/sysdeps/x86_64/multiarch/init-arch.c b/sysdeps/x86_64/multiarch/init-arch.c
index 35fd19a..365c8ac 100644
--- a/sysdeps/x86_64/multiarch/init-arch.c
+++ b/sysdeps/x86_64/multiarch/init-arch.c
@@ -86,3 +86,6 @@ __init_cpu_features (void)
else
__cpu_features.kind = arch_kind_other;
}
+
+strong_alias (__cpu_features, __libc_cpu_features)
+strong_alias (__init_cpu_features, __libc_init_cpu_features)
diff --git a/sysdeps/x86_64/multiarch/init-arch.h b/sysdeps/x86_64/multiarch/init-arch.h
index 48a2127..7d49acd 100644
--- a/sysdeps/x86_64/multiarch/init-arch.h
+++ b/sysdeps/x86_64/multiarch/init-arch.h
@@ -45,19 +45,43 @@ extern struct cpu_features
unsigned int family;
unsigned int model;
} __cpu_features attribute_hidden;
+extern struct cpu_features __libc_cpu_features;
extern void __init_cpu_features (void) attribute_hidden;
-#define INIT_ARCH()\
+extern void __libc_init_cpu_features (void);
+#ifdef NOT_IN_libc
+# define INIT_ARCH()\
+ do \
+ if (__libc_cpu_features.kind == arch_kind_unknown) \
+ __libc_init_cpu_features (); \
+ while (0)
+#else
+# define INIT_ARCH()\
do \
if (__cpu_features.kind == arch_kind_unknown) \
__init_cpu_features (); \
while (0)
+#endif
/* Following are the feature tests used throughout libc. */
-#define HAS_POPCOUNT \
+#ifdef NOT_IN_libc
+# define HAS_POPCOUNT \
+ ((__libc_cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 23)) != 0)
+
+# define HAS_SSE4_2 \
+ ((__libc_cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 20)) != 0)
+
+# define HAS_FMA \
+ ((__libc_cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 12)) != 0)
+#else
+# define HAS_POPCOUNT \
((__cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 23)) != 0)
-#define HAS_SSE4_2 \
+# define HAS_SSE4_2 \
((__cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 20)) != 0)
+
+# define HAS_FMA \
+ ((__cpu_features.cpuid[COMMON_CPUID_INDEX_1].ecx & (1 << 12)) != 0)
+#endif
diff --git a/sysdeps/x86_64/multiarch/s_fma-c.c b/sysdeps/x86_64/multiarch/s_fma-c.c
new file mode 100644
index 0000000..68837a5
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/s_fma-c.c
@@ -0,0 +1,32 @@
+/* FMA version of fma
+ Copyright (C) 2009 Free Software Foundation, Inc.
+ Contributed by Intel Corporation.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include <config.h>
+
+#ifdef HAVE_AVX_SUPPORT
+
+double
+__fma_fma (double x, double y, double z)
+{
+ asm ("vfmadd213sd %3, %2, %0" : "=x" (x) : "0" (x), "x" (y), "xm" (z));
+ return x;
+}
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/s_fma.c b/sysdeps/x86_64/multiarch/s_fma.c
new file mode 100644
index 0000000..205bb1c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/s_fma.c
@@ -0,0 +1,14 @@
+#include <config.h>
+
+#ifdef HAVE_AVX_SUPPORT
+# include "init-arch.h"
+# define FMA __fma_sse2
+
+extern double __fma (double, double, double);
+extern double __fma_fma (double, double, double);
+extern double __fma_sse2 (double, double, double);
+
+libc_ifunc (__fma, HAS_FMA ? __fma_fma : __fma_sse2);
+#endif
+
+#include <math/s_fma.c>