[PATCH v2] libstdc++: Implement C++26 [simd]
Matthias Kretz
m.kretz@gsi.de
Mon Oct 27 11:40:56 GMT 2025
Also at https://forge.sourceware.org/gcc/gcc-TEST/pulls/117
------------------ 8< -----------------
This implementation differs a lot from the std::experimental
implementation. I tried my best to reduce the number of template
instantiations wrt. what std::experimental::simd does.
A basic_vec/basic_mask that uses multiple registers is built as a tree
of smaller specializations, where the left branch always stores a
power-of-2 number of elements.
Missing with this commit: [simd.permute.dynamic], [simd.permute.mask],
[simd.permute.memory], [simd.bit], [simd.math], and everything related
to making complex<T> vectorizable.
Minimal unit tests will follow. (An extensive test suite that does not
easily integrate with the GCC test framework already exists.)
libstdc++-v3/ChangeLog:
* include/bits/simd_alg.h: New file.
* include/bits/simd_details.h: New file.
* include/bits/simd_flags.h: New file.
* include/bits/simd_iterator.h: New file.
* include/bits/simd_loadstore.h: New file.
* include/bits/simd_mask.h: New file.
* include/bits/simd_mask_reductions.h: New file.
* include/bits/simd_reductions.h: New file.
* include/bits/simd_vec.h: New file.
* include/bits/simd_x86.h: New file.
* include/bits/vec_ops.h: New file.
* include/bits/version.def: Add simd.
* include/bits/version.h: Regenerate.
* include/std/simd: New file.
Signed-off-by: Matthias Kretz <m.kretz@gsi.de>
---
libstdc++-v3/include/bits/simd_alg.h | 76 +
libstdc++-v3/include/bits/simd_details.h | 1563 ++++++++++++
libstdc++-v3/include/bits/simd_flags.h | 246 ++
libstdc++-v3/include/bits/simd_iterator.h | 153 ++
libstdc++-v3/include/bits/simd_loadstore.h | 387 +++
libstdc++-v3/include/bits/simd_mask.h | 1677 +++++++++++++
.../include/bits/simd_mask_reductions.h | 94 +
libstdc++-v3/include/bits/simd_reductions.h | 112 +
libstdc++-v3/include/bits/simd_vec.h | 2091 +++++++++++++++++
libstdc++-v3/include/bits/simd_x86.h | 1338 +++++++++++
libstdc++-v3/include/bits/vec_ops.h | 612 +++++
libstdc++-v3/include/bits/version.def | 11 +
libstdc++-v3/include/bits/version.h | 9 +
libstdc++-v3/include/std/simd | 48 +
14 files changed, 8417 insertions(+)
create mode 100644 libstdc++-v3/include/bits/simd_alg.h
create mode 100644 libstdc++-v3/include/bits/simd_details.h
create mode 100644 libstdc++-v3/include/bits/simd_flags.h
create mode 100644 libstdc++-v3/include/bits/simd_iterator.h
create mode 100644 libstdc++-v3/include/bits/simd_loadstore.h
create mode 100644 libstdc++-v3/include/bits/simd_mask.h
create mode 100644 libstdc++-v3/include/bits/simd_mask_reductions.h
create mode 100644 libstdc++-v3/include/bits/simd_reductions.h
create mode 100644 libstdc++-v3/include/bits/simd_vec.h
create mode 100644 libstdc++-v3/include/bits/simd_x86.h
create mode 100644 libstdc++-v3/include/bits/vec_ops.h
create mode 100644 libstdc++-v3/include/std/simd
diff --git a/libstdc++-v3/include/bits/simd_alg.h b/libstdc++-v3/include/bits/
simd_alg.h
new file mode 100644
index 00000000000..3e9652bae94
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_alg.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+ */
+
+#ifndef _GLIBCXX_SIMD_ALG_H
+#define _GLIBCXX_SIMD_ALG_H 1
+
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#if __cplusplus >= 202400L
+
+#include "simd_vec.h"
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+// [simd.alg]
-----------------------------------------------------------------
+namespace std::simd
+{
+ template<typename _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ constexpr basic_vec<_Tp, _Ap>
+ min(const basic_vec<_Tp, _Ap>& __a, const basic_vec<_Tp, _Ap>& __b)
noexcept
+ { return __select_impl(__a < __b, __a, __b); }
+
+ template<typename _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ constexpr basic_vec<_Tp, _Ap>
+ max(const basic_vec<_Tp, _Ap>& __a, const basic_vec<_Tp, _Ap>& __b)
noexcept
+ { return __select_impl(__a < __b, __b, __a); }
+ constexpr pair<basic_vec<_Tp, _Ap>, basic_vec<_Tp, _Ap>>
+ minmax(const basic_vec<_Tp, _Ap>& __a, const basic_vec<_Tp, _Ap>& __b)
noexcept
+ { return {min(__a, __b), max(__a, __b)}; }
+ constexpr basic_vec<_Tp, _Ap>
+ clamp(const basic_vec<_Tp, _Ap>& __v, const basic_vec<_Tp, _Ap>& __lo,
+ const basic_vec<_Tp, _Ap>& __hi)
+ {
+ __glibcxx_simd_precondition(none_of(__lo > __hi), "lower bound is
larger than upper bound");
+ return max(__lo, min(__hi, __v));
+ constexpr auto
+ select(bool __c, const _Tp& __a, const _Up& __b)
+ -> remove_cvref_t<decltype(__c ? __a : __b)>
+ { return __c ? __a : __b; }
+ template<size_t _Bytes, typename _Ap, typename _Tp, typename _Up>
+ [[__gnu__::__always_inline__]]
+ constexpr auto
+ select(const basic_mask<_Bytes, _Ap>& __c, const _Tp& __a, const _Up&
__b)
+ noexcept -> decltype(__select_impl(__c, __a, __b))
+ { return __select_impl(__c, __a, __b); }
+ using simd::minmax;
+ using simd::clamp;
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_ALG_H
diff --git a/libstdc++-v3/include/bits/simd_details.h b/libstdc++-v3/include/
bits/simd_details.h
new file mode 100644
index 00000000000..c900100601f
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_details.h
@@ -0,0 +1,1563 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+//#include <bits/c++config.h>
+#include <bits/utility.h> // integer_sequence, etc.
+#include <cmath> // for math_errhandling :(
+#include <concepts>
+#include <cstdint>
+#include <limits>
+#include <span> // for dynamic_extent
+
+// There are simply too many constants and bit operators that currently
depend on CHAR_BIT == 8.
+// Generalization to CHAR_BIT != 8 does not make sense without testability
(i.e. a test target).
+#error "<simd> is not supported for CHAR_BIT != 8"
+#endif
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+#error "Use of SSE2 is required on x86-64"
+#endif
+
+#endif
+
+#error "Not implemented yet. Only supported on x86 for now."
+#endif
+
+#ifndef _GLIBCXX_SIMD_NOEXCEPT
+/** @internal
+ * For unit-testing preconditions, use this macro to remove noexcept.
+ */
+#define _GLIBCXX_SIMD_NOEXCEPT noexcept
+#endif
+
+#define __glibcxx_simd_precondition(expr, msg, ...)
\
+ __glibcxx_assert(expr)
+
+ __iota = [] { static_assert(false, "invalid __iota specialization"); }();
+
+ template <typename _Cp, auto __re, auto __im, typename _Tp = typename
_Cp::value_type>
+ constexpr _Cp __complex_object = _Cp {_Tp(__re), _Tp(__im)};
+#endif
+
+ = same_as<_Tp, complex<typename _Tp::value_type>>;
+#else
+ typename _Tp::value_type;
+ { __x.real() } -> same_as<typename _Tp::value_type>;
+ { __x.imag() } -> same_as<typename _Tp::value_type>;
+ { real(__x) } -> same_as<typename _Tp::value_type>;
+ { imag(__x) } -> same_as<typename _Tp::value_type>;
+ { +__x } -> same_as<_Tp>;
+ { -__x } -> same_as<_Tp>;
+ { __x + __x } -> same_as<_Tp>;
+ { __x - __x } -> same_as<_Tp>;
+ { __x * __x } -> same_as<_Tp>;
+ { __x / __x } -> same_as<_Tp>;
+ { __x += __x } -> same_as<_Tp&>;
+ { __x -= __x } -> same_as<_Tp&>;
+ { __x *= __x } -> same_as<_Tp&>;
+ { __x /= __x } -> same_as<_Tp&>;
+ { abs(__x) } -> same_as<typename _Tp::value_type>;
+ { arg(__x) } -> same_as<typename _Tp::value_type>;
+ { norm(__x) } -> same_as<typename _Tp::value_type>;
+ { conj(__x) } -> same_as<_Tp>;
+ { proj(__x) } -> same_as<_Tp>;
+ }
+ // not constexpr: && (abs(__complex_object<_Tp, 3, 4>) == typename
_Tp::value_type(5))
+ && (norm(__complex_object<_Tp, 5, 5>) == typename
_Tp::value_type(50))
+ && (2 * sizeof(typename _Tp::value_type) == sizeof(_Tp))
+ && (__builtin_bit_cast(_Arr2<typename _Tp::value_type>,
__complex_object<_Tp, 1, 2>)
+ ._M_data[0] == 1);
+#endif
+
+ concept __complex_like = __complex_like_impl<remove_cvref_t<_Tp>>;
+
+ // FIXME: needs to exclude bfloat16_t
+ template <typename _Tp>
+ = same_as<remove_cv_t<_Tp>, _Tp>
+ && ((integral<_Tp> && sizeof(_Tp) <= sizeof(0ULL) && !same_as<_Tp,
bool>)
+ || (floating_point<_Tp> && sizeof(_Tp) <= sizeof(double)));
+
+ // [simd.general] p2
+ template <typename _Tp>
+ || (__complex_like_impl<_Tp> && __vectorizable_scalar<typename
_Tp::value_type>
+ && floating_point<typename _Tp::value_type>);
+
+ enum class _AbiVariant : unsigned long long
+ {
+ _BitMask = 0x01, // AVX512 bit-masks
+ _MaskVariants = 0x0f, // vector masks if bits [0:3] are 0
+ consteval _AbiVariant
+ __filter_abi_variant(_AbiVariant __in, same_as<_AbiVariant> auto...
__to_keep)
+ {
+ using _Up = underlying_type_t<_AbiVariant>;
+ return static_cast<_AbiVariant>(static_cast<_Up>(__in) &
(static_cast<_Up>(__to_keep) | ...));
+ }
+ * C++26 [simd.expos.defn]
+ */
+ template <size_t _Bytes>
+ using __integer_from
+ return static_cast<signed char>(0);
+ else if constexpr (sizeof(signed short) == _Bytes)
+ return static_cast<signed short>(0);
+ else if constexpr (sizeof(signed int) == _Bytes)
+ return static_cast<signed int>(0);
+ else if constexpr (sizeof(signed long long) == _Bytes)
+ return static_cast<signed long long>(0);
+ else
+ template <size_t _Bytes>
+ using _UInt = make_unsigned_t<__integer_from<_Bytes>>;
+
+ template <std::same_as<long> _Tp>
+ struct __canonical_vec_type<_Tp>
+ template <std::same_as<unsigned long> _Tp>
+ struct __canonical_vec_type<_Tp>
+ template <std::same_as<long> _Tp>
+ struct __canonical_vec_type<_Tp>
+ template <std::same_as<unsigned long> _Tp>
+ struct __canonical_vec_type<_Tp>
+#endif
+
+ requires std::is_enum_v<_Tp>
+ struct __canonical_vec_type<_Tp>
+ { using type = __canonical_vec_type<std::underlying_type_t<_Tp>>::type;
};
+
+#endif
+
+ struct __canonical_vec_type<char8_t>
+ { using type = unsigned char; };
+ struct __canonical_vec_type<char16_t>
+ { using type = uint_least16_t; };
+
+ struct __canonical_vec_type<char32_t>
+ { using type = uint_least32_t; };
+
+ struct __canonical_vec_type<wchar_t>
+ {
+ using type = std::conditional_t<std::is_signed_v<wchar_t>,
+ simd::__integer_from<sizeof(wchar_t)>,
+ simd::_UInt<sizeof(wchar_t)>>;
+ };
+ * This ABI tag describes basic_vec objects that store one element per data
member and basic_mask
+ * objects that store one bool data members.
+ *
+ * @tparam _Np The number of elements, which also matches the number of
data members in
+ * basic_vec and basic_mask.
+ */
+ static constexpr bool _S_is_vecmask = false;
+
+ // in principle a bool is a 1-bit bitmask, but this is asking for an
AVX512 bitmask
+ static constexpr bool _S_is_bitmask = false;
+
+ template <size_t>
+ using _MaskDataType = bool;
+
+ consteval _ScalarAbi<_N2>
+ _M_resize() const
+ static_assert(_N2 == _Nreg2);
+ return {};
+ * This ABI tag describes basic_vec objects that store one or more objects
declared with the
+ * [[gnu::vector_size(N)]] attribute.
+ * Applied to basic_mask objects, this ABI tag either describes
corresponding vector-mask objects
+ * or bit-mask objects. Which one is used is determined via @p _Var.
+ * @tparam _Np The number of elements.
+ * @tparam _Nreg The number of registers needed to store @p _Np elements.
+ * @tparam _Var Determines how complex value-types are layed out and
whether mask types use
+ * bit-masks or vector-masks.
+ template <int _Np, int _Nreg, underlying_type_t<_AbiVariant> _Var>
+ struct _Abi
+ * The number of registers needed to represent one basic_vec for the
element type that was
+ * used on ABI deduction.
+ *
+ static constexpr _AbiVariant _S_variant =
static_cast<_AbiVariant>(_Var);
+
+ static constexpr bool _S_is_bitmask
+ = __filter_abi_variant(_S_variant, _AbiVariant::_BitMask) ==
_AbiVariant::_BitMask;
+ static constexpr bool _S_is_vecmask = !_S_is_bitmask;
+
+ static_assert(_S_nreg == 1);
+ constexpr int __n =
__bit_ceil(unsigned(_S_size));
+ template <size_t _Bytes>
+ using _MaskDataType
+ consteval auto
+ _M_resize() const
+ * value.
+ *
+ * Rationale: Consider diagnostic output and mangling of e.g. vec<int, 4>
with AVX512. That's an
+ * alias for std::simd::basic_vec<int, std::simd::_Abi<4, 1, 1ull>>. If
_AbiVariant were the
+ * template argument type of _Abi, the diagnostic output would be
'std::simd::basic_vec<int,
+ * std::simd::_Abi<4, 1,
(std::simd::_AbiVariant)std::simd::_AbiVariant::_BitMask>>'. That's a lot
+ * longer, requires longer mangled names, and bakes the names of the
enumerators into the ABI. As
+ * soon as bits of multiple _AbiVariants are combined, this becomes hard to
parse for humans
+ using _Abi_t = _Abi<_Np, _Nreg,
(static_cast<underlying_type_t<_AbiVariant>>(_Vs) | ... | 0)>;
+
+ * This type is used whenever ABI tag deduction can't give a useful answer.
+ */
+ * Satisfied if @p _Tp is a valid simd ABI tag. This is a necessary but not
sufficient condition
+ * for an enabled basic_vec/basic_mask specialization.
+ */
+ = same_as<decltype(_Tp::_S_variant), const _AbiVariant>
+ && (_Tp::_S_size >= _Tp::_S_nreg) && (_Tp::_S_nreg >= 1)
+ { __x.template _M_resize<_Tp::_S_size, _Tp::_S_nreg>() } ->
same_as<_Tp>;
+ };
+ = __abi_tag<_Tp> && same_as<_Tp, _ScalarAbi<_Tp::_S_size>>;
+
+ // math_errhandling may expand to an extern symbol, in which case we must
assume fp exceptions
+ // need to be considered.
+ requires requires { typename bool_constant<0 != (math_errhandling &
MATH_ERREXCEPT)>; }
+ consteval bool
+ __handle_fpexcept_impl(int)
+ { return 0 != (math_errhandling & MATH_ERREXCEPT); }
+
+ // Fallback if math_errhandling doesn't work: implement correct exception
behavior.
+ consteval bool
+ __handle_fpexcept_impl(float)
+ * This type can be used as a template parameter for avoiding ODR
violations, where code needs to
+ * differ depending on optimization flags (mostly fp-math related).
+ consteval bool
+ _M_test(int __bit) const
+ // true iff floating-point operations can signal an exception (allow non-
default handler)
+ consteval bool
+ _M_fp_may_signal() const
+ // true iff floating-point operations can raise an exception flag
+ consteval bool
+ _M_fp_may_raise() const
+ consteval bool
+ _M_fast_math() const
+ consteval bool
+ _M_finite_math_only() const
+ consteval bool
+ _M_no_signed_zeros() const
+ consteval bool
+ _M_signed_zeros() const
+ consteval bool
+ _M_reciprocal_math() const
+ consteval bool
+ _M_no_math_errno() const
+ consteval bool
+ _M_math_errno() const
+ consteval bool
+ _M_associative_math() const
+ consteval bool
+ _M_conforming_to_STDC_annex_G() const
+ consteval bool
+ _M_support_snan() const
+#endif
+ + (__handle_fpexcept_impl(0) << 12)
+#endif
+#if __FINITE_MATH_ONLY__
+#endif
+#if __NO_SIGNED_ZEROS__
+#endif
+#if __RECIPROCAL_MATH__
+#endif
+#if __NO_MATH_ERRNO__
+#endif
+#if __ASSOCIATIVE_MATH__
+#endif
+ // bits 7, 8, and 9 reserved for __FLT_EVAL_METHOD__
+#elif __FLT_EVAL_METHOD__ == 2
+ + (2 << 7)
+#elif __FLT_EVAL_METHOD__ != 0
+ + (3 << 7)
+#endif
+
+ // C Annex G defines the behavior of complex<T> where T is IEC60559
floating-point. If
+ // __STDC_IEC_60559_COMPLEX__ is defined then Annex G is implemented
- and simd<complex>
+#endif
+#if __SUPPORT_SNAN__
+#endif
+ ;
+ * Return true iff @p __s equals "1".
+ */
+ consteval bool
+ __streq_to_1(const char* __s)
+ { return __s != nullptr && __s[0] == '1' && __s[1] == '\0'; }
+
+
(static_cast<__UINT64_TYPE__>(std::simd::__streq_to_1(_GLIBCXX_SIMD_TOSTRING_IMPL(feat)))
<< off)
+
+#endif
+#if __ARM_FEATURE_SVE_BITS > 0 && __ARM_FEATURE_SVE_VECTOR_OPERATORS == 1
+#endif
+
+ struct _ArchTraits
+ {
+ consteval bool
+ _M_test(int __bit) const
+ consteval bool
+ _M_have_neon() const
+ consteval bool
+ _M_have_neon_a32() const
+ consteval bool
+ _M_have_neon_a64() const
+ consteval bool
+ _M_have_sve() const
+ consteval bool
+ _M_eval_as_f32() const
+ { return is_same_v<_Tp, _Float16>; }
+ };
+#elif __powerpc__
+
+ struct _ArchTraits
+ {
+ consteval bool _M_altivec()
+ { return (_M_flags & 0xf) >= 1; }
+ consteval bool _M_vsx()
+ { return (_M_flags & 0xf) >= 2; }
+ consteval bool _M_power8()
+ { return (_M_flags & 0xf) >= 3; }
+ consteval bool _M_power9()
+ { return (_M_flags & 0xf) >= 4; }
+ consteval bool
+ _M_eval_as_f32() const
+ { return is_same_v<_Tp, _Float16>; }
+ };
+#elif _GLIBCXX_X86
+
+ // no ODR issue? The same could be said about several other flags above
that are not checked
+ // anywhere.
+ struct _ArchTraits
+ {
+ consteval bool
+ _M_test(int __bit) const
+ consteval bool
+ _M_have_mmx() const
+ consteval bool
+ _M_have_sse() const
+ consteval bool
+ _M_have_sse2() const
+ consteval bool
+ _M_have_sse3() const
+ consteval bool
+ _M_have_ssse3() const
+ consteval bool
+ _M_have_sse4_1() const
+ consteval bool
+ _M_have_sse4_2() const
+ consteval bool
+ _M_have_popcnt() const
+ consteval bool
+ _M_have_avx() const
+ consteval bool
+ _M_have_f16c() const
+ consteval bool
+ _M_have_bmi() const
+ consteval bool
+ _M_have_bmi2() const
+ consteval bool
+ _M_have_lzcnt() const
+ consteval bool
+ _M_have_avx2() const
+ consteval bool
+ _M_have_fma() const
+ consteval bool
+ _M_have_avx512f() const
+ consteval bool
+ _M_have_avx512cd() const
+ consteval bool
+ _M_have_avx512dq() const
+ consteval bool
+ _M_have_avx512bw() const
+ consteval bool
+ _M_have_avx512vl() const
+ consteval bool
+ _M_have_avx512bitalg() const
+ consteval bool
+ _M_have_avx512vbmi() const
+ consteval bool
+ _M_have_avx512vbmi2() const
+ consteval bool
+ _M_have_avx512ifma() const
+ consteval bool
+ _M_have_avx512vnni() const
+ consteval bool
+ _M_have_avx512vpopcntdq() const
+ consteval bool
+ _M_have_avx512fp16() const
+ consteval bool
+ _M_have_avx512bf16() const
+ consteval bool
+ _M_have_avxifma() const
+ consteval bool
+ _M_have_avxneconvert() const
+ consteval bool
+ _M_have_avxvnni() const
+ consteval bool
+ _M_have_avxvnniint8() const
+ consteval bool
+ _M_have_avxvnniint16() const
+ consteval bool
+ _M_have_avx10_1() const
+ consteval bool
+ _M_have_avx10_2() const
+ consteval bool
+ _M_have_avx512vp2intersect() const
+ consteval bool
+ _M_have_sse4a() const
+ consteval bool
+ _M_have_fma4() const
+ consteval bool
+ _M_have_xop() const
+ consteval bool
+ _M_eval_as_f32() const
+ { return is_same_v<_Tp, _Float16> && !_M_have_avx512fp16(); }
+ };
+ consteval auto
+ __native_abi()
+ constexpr int __adj_sizeof = sizeof(_Tp) * (1 + is_same_v<_Tp,
_Float16>);
+ if constexpr (!__vectorizable<_Tp>)
+ static_assert(false);
+ }
+ else if constexpr (is_same_v<_Tp, _Float16> && !_Traits._M_have_f16c())
+ return _ScalarAbi<1>();
+ else if constexpr (_Traits._M_have_avx() && is_floating_point_v<_Tp>)
+ return _Abi_t<32 / __adj_sizeof, 1>();
+ else if constexpr (_Traits._M_have_sse() && is_floating_point_v<_Tp>
+ && sizeof(_Tp) == sizeof(float))
+ // TODO: add more targets
+ struct _ArchTraits
+ {
+ constexpr bool
+ _M_test(int __bit) const
+ consteval auto
+ __native_abi()
+#endif
+
+ * always_inline (to avoid issues when linking code compiled with different
compiler flags).
+ */
+ * Alias for an ABI tag such that basic_vec<_Tp, __native_abi_t_<_Tp>>
stores one SIMD register of
+ * optimal width.
+ * @tparam _Tp A vectorizable type.
+ *
+ * C++26 [simd.expos.abi]
+ */
+ consteval auto
+ __deduce_abi()
+ * Alias for an ABI tag @c A such that <tt>basic_vec<_Tp, A></tt> stores @p
_Np elements.
+ *
+ * C++26 [simd.expos.abi]
+ */
+ * \c rebind implementation detail for basic_vec, and basic_mask where we
know the destination
+ * value-type
+ consteval auto
+ __abi_rebind()
+ static_assert(0 != __native._S_size);
+ constexpr int __nreg = __div_ceil(_Np, __native._S_size);
+ static_assert(false);
+
+ * @c rebind implementation detail for basic_mask.
+ *
+ * than its @c sizeof. So <tt>_Bytes == 8</tt> could mean
<tt>complex<float></tt>, @c double, or
+ * @c int64_t. E.g. <tt>_Np == 4</tt> with AVX w/o AVX2 that's
<tt>vector(4) int</tt>,
+ * <tt>vector(4) long long</tt>, or <tt>2x vector(2) long long</tt>.
+ * That's why this overload has the additional @p _IsOnlyResize parameter,
which tells us that the
+ template <size_t _Bytes, int _Np, __abi_tag _A0, bool _IsOnlyResize,
_ArchTraits _Traits = {}>
+ consteval auto
+ __abi_rebind()
+ // AVX w/o AVX2:
+ // e.g. resize_t<8, mask<float, Whatever>> needs to be _Abi<8, 1> not
_Abi<8, 2>
+ // We determine whether _A0 identifies an AVX vector by looking at the
size of a native
+ // register. If it's 32, it's a YMM register, otherwise it's 16 or
less.
+ else if constexpr (_IsOnlyResize
+ static_assert(false);
+ }
+#endif
+
+ * On IvyBridge, (vec<float> == 0.f) == (rebind_t<int, vec<float>> == 0)
does not compile. It does
+ * compile on basically every other target, though. This is due to the
difference in ABI tag:
+ * _Abi<8, 1, 1> vs. _Abi<8, 2, 1>. I know how to define this funtion for
libstdc++ to avoid
+ * interconvertible masks. The question is whether we can specify this in
general for C++29.
+ */
+ consteval bool
+ __is_mask_conversion_explicit([[maybe_unused]] size_t __b0,
[[maybe_unused]] size_t __b1)
+ {
+ static_assert(__n == _From::_S_size);
+#ifndef _GLIBCXX_SIMD_COND_EXPLICIT_MASK_CONVERSION
+ /// C++26 [simd.mask.ctor] uses unconditional explicit
+ return true;
+ // everything is better than _ScalarAbi, except when converting to a
single bool
+ if constexpr (__scalar_abi_tag<_To>)
+ // with vec-masks, fewer registers is better
+ else if constexpr (_From::_S_nreg != _To::_S_nreg)
+#endif
+ }
+ * libstdc++ unconditionally uses @c int here, since it matches the return
type of
+ * 'Bit Operation Builtins' in GCC.
+ *
+ * C++26 [simd.expos.defn]
+ */
+ * The width of <tt>basic_vec<T, Abi></tt> if the specialization
<tt>basic_vec<T, Abi></tt> is
+ * enabled, or @c 0 otherwise.
+ * C++26 [simd.expos.defn]
+ */
+ class basic_vec;
+
+ using vec = basic_vec<_Tp, __deduce_abi_t<_Tp, _Np>>;
+
+ template <size_t _Bytes, typename _Ap =
__native_abi_t<__integer_from<_Bytes>>>
+ class basic_mask;
+
+ using mask = basic_mask<sizeof(_Tp), __deduce_abi_t<_Tp, _Np>>;
+
+ = __vectorizable<typename _Tp::value_type>
+ && __abi_tag<typename _Tp::abi_type>
+ && _Tp::size() >= 1;
+ // [simd.ctor] load constructor constraints
+#ifdef __clang__
+ template <typename _Tp>
+ static constexpr remove_cvref_t<_Tp> __static_sized_range_obj = {};
+#endif
+
+ template <typename _Tp, size_t _Np = 0>
+ concept __static_sized_range
+ = ranges::contiguous_range<_Tp> && ranges::sized_range<_Tp>
+ && requires(_Tp&& __r) {
+ typename integral_constant<size_t, ranges::size(__r)>;
+#ifdef __clang__
+ requires (_Np == 0 || ranges::size(__static_sized_range_obj<_Tp>)
== _Np);
+#endif
+ };
+ consteval size_t
+ __static_range_size(_Rg&& __r)
+ if constexpr (requires { typename integral_constant<size_t,
ranges::size(__r)>; })
+ return ranges::size(__r);
+ return dynamic_extent;
+ }
+ = convertible_to<_From, _To> && is_arithmetic_v<_From> &&
is_arithmetic_v<_To>
+ && !(is_signed_v<_From> && is_unsigned_v<_To>)
+ && numeric_limits<_From>::digits <= numeric_limits<_To>::digits
+ && numeric_limits<_From>::max() <= numeric_limits<_To>::max()
+ && numeric_limits<_From>::lowest() >=
numeric_limits<_To>::lowest();
+
+ _From, typename _To::value_type>);
+
+ * The value of the @c _Bytes template argument to a @c basic_mask
specialization.
+ *
+ * C++26 [simd.expos.defn]
+ */
+ constexpr size_t __mask_element_size = 0;
+
+ = convertible_to<_Tp, decltype(_Tp::value)>
+ && equality_comparable_with<_Tp, decltype(_Tp::value)>
+ && bool_constant<_Tp() == _Tp::value>::value
+ && bool_constant<static_cast<decltype(_Tp::value)>(_Tp()) ==
_Tp::value>::value;
+
+ // [simd.ctor] explicit(...) of broadcast ctor
+ template <typename _From, typename _To>
+ = __constexpr_wrapper_like<_From> && convertible_to<_From, _To>
+ && requires { { _From::value } -> std::convertible_to<_To>; }
+ && static_cast<decltype(_From::value)>(_To(_From::value)) ==
_From::value
+ && !(std::unsigned_integral<_To> && _From::value <
decltype(_From::value)())
+ && _From::value <= std::numeric_limits<_To>::max()
+ && _From::value >= std::numeric_limits<_To>::lowest();
+
+ // [simd.ctor] p4
+ template <typename _From, typename _To>
+ = convertible_to<_From, _To> // 4
+ && ((!is_arithmetic_v<remove_cvref_t<_From>>
+ && !__constexpr_wrapper_like<remove_cvref_t<_From>>) // 4.1
+ || __value_preserving_convertible_to<remove_cvref_t<_From>,
_To> // 4.2
+ ||
__non_narrowing_constexpr_conversion<remove_cvref_t<_From>, _To>); // 4.3
+
+ = floating_point<_From> && floating_point<_To>
+ && same_as<common_type_t<_From, _To>, _From>;
+
+ && (sizeof(_From) > sizeof(_To) || same_as<common_type_t<_From,
_To>, _From>);
+
+ = same_as<_From, _To>
+ || (__vectorizable<_From> && __vectorizable<_To>
+ || (std::convertible_to<_From, _To>
+ && (std::same_as<_Traits, __convert_flag> ||
...))));
+
+ = std::convertible_to<_From, _To>
+ && (!is_arithmetic_v<_From> ||
__value_preserving_convertible_to<_From, _To>);
+
+ decltype(declval<_Fp>()(__simd_size_constant<_Is>)), _Tp> &&
...)
+ constexpr void
+ __simd_generator_invokable_impl(integer_sequence<__simd_size_type,
_Is...>);
+
+ __simd_generator_invokable_impl<_Fp,
_Tp>(make_integer_sequence<__simd_size_type, _Np>());
+ };
+ template <size_t _Bytes, __abi_tag _Ap>
+ constexpr size_t __mask_element_size<basic_mask<_Bytes, _Ap>> = _Bytes;
+
+ = same_as<_Vp, basic_vec<typename _Vp::value_type, typename
_Vp::abi_type>>
+ && is_default_constructible_v<_Vp>;
+
+ = same_as<_Vp, basic_mask<__mask_element_size<_Vp>, typename
_Vp::abi_type>>
+ && is_default_constructible_v<_Vp>;
+
+ = __simd_vec_type<_Vp> && floating_point<typename _Vp::value_type>;
+
+ = __simd_vec_type<_Vp> && integral<typename _Vp::value_type>;
+
+ using _Up = decltype(declval<const _Tp&>() + declval<const
_Tp&>());
+ if constexpr (__data_parallel_type<_Up>)
+ static_assert(is_same_v<__deduced_vec_t<int>, void>);
+
+ using _Up = decltype(declval<const _Tp&>() + declval<const
_Tp&>());
+ if constexpr (__simd_vec_type<_Up>)
+ return vec<_Up, _Vp::size()>();
+ }());
+ return common_type_t<__deduced_vec_t<_T0>,
__deduced_vec_t<_T1>>();
+ else if constexpr (__math_floating_point<_T0>)
+ return common_type_t<__deduced_vec_t<_T0>, _T1>();
+ else if constexpr (__math_floating_point<_T1>)
+ return common_type_t<_T0, __deduced_vec_t<_T1>>();
+ // else void
+ { using type = common_type_t<__math_common_simd_t<_T0, _T1>, _TRest...>;
};
+
+ requires (sizeof...(_TRest) > 0) && is_void_v<__math_common_simd_t<_T0,
_T1>>
+ struct __math_common_simd_impl<_T0, _T1, _TRest...>
+ { using type = common_type_t<__math_common_simd_t<_TRest...>, _T0, _T1>;
};
+
+ = requires (const _BinaryOperation __binary_op, const vec<_Tp, 1> __v)
{
+ { __binary_op(__v, __v) } -> same_as<vec<_Tp, 1>>;
+ };
+ * Returns the highest index @c i where <tt>(__bits >> i) & 1</tt> equals
@c 1.
+ */
+ __highest_bit(std::unsigned_integral auto __bits)
+ {
+ using __similar_mask = basic_mask<sizeof(_Tp), decltype(__abi_rebind<_Tp,
_Np, _Ap>())>;
+
+ using __similar_vec = basic_vec<_Tp, decltype(__abi_rebind<_Tp, _Np,
_Ap>())>;
+
+ // LWG???? [simd.expos]
+ template <size_t _Bytes, typename _Ap>
+ using __simd_vec_from_mask_t = __similar_vec<__integer_from<_Bytes>,
_Ap::_S_size, _Ap>;
+#ifdef _GLIBCXX_SIMD_CONSTEVAL_BROADCAST
+ class bad_value_preserving_cast
+ {};
+ template <typename _To, signed_integral _From>
+ constexpr void
+ using _Up = make_unsigned_t<_From>;
+ if (static_cast<_Up>(static_cast<_To>(__x)) != static_cast<_Up>(__x))
+ __builtin_unreachable(); //throw bad_value_preserving_cast();
+ }
+ requires (is_arithmetic_v<_From> && !signed_integral<_From>)
+ constexpr void
+ if (static_cast<_From>(static_cast<_To>(__x)) != __x)
+ __builtin_unreachable(); //throw bad_value_preserving_cast();
+ }
+ return static_cast<_To>(__x);
+ }
+#ifndef PREFER_CONSTEVAL
+#define PREFER_CONSTEVAL 1
+#endif
+
+#if PREFER_CONSTEVAL
+ template <typename _From, typename _To>
+ concept __simd_vec_bcast = constructible_from<_To, _From>;
+
+ = __simd_vec_bcast<_From, _To> && convertible_to<_From, _To>
+ && is_arithmetic_v<remove_cvref_t<_From>>
+ && !__value_preserving_convertible_to<remove_cvref_t<_From>, _To>;
+#else
+ concept __simd_vec_bcast_consteval = constructible_from<_To, _From>;
+
+#endif
+#else
+ concept __simd_vec_bcast = constructible_from<_To, _From>;
+#endif
+
+ concept __converts_trivially = convertible_to<_From, _To>
+ && sizeof(_From) == sizeof(_To)
+ && is_integral_v<_From> ==
is_integral_v<_To>
+ && is_floating_point_v<_From> ==
is_floating_point_v<_To>;
+
+ __bit_foreach(unsigned_integral auto __bits, auto&& __fun)
+ {
+ static_assert(sizeof(__bits) >= sizeof(int)); // avoid promotion to int
+ while (__bits)
+ * Optimized memcpy for use in partial loads and stores.
+ *
+ * @tparam _Chunk Copies @p __n times @p _Chunk bytes.
+ * @tparam _Max Copy no more than @p _Max bytes.
+ *
+ * @param __dst The destination pointer.
+ * @param __src The source pointer.
+ * @param __n Thu number of chunks that need to be copied.
+ */
+ template <size_t _Chunk, size_t _Max>
+ inline void
+ __memcpy_chunks(byte* __restrict__ __dst, const byte* __restrict__ __src,
+ size_t __n)
+ {
+ static_assert(_Max <= 64);
+ static_assert(__has_single_bit(_Chunk) && _Chunk <= 8);
+ size_t __bytes = _Chunk * __n;
+ if (__bytes > 32 && _Max > 32)
+ constexpr bool
+ __ptr_is_aligned_to(const void* __ptr, size_t __align)
+ {
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_DETAILS_H
diff --git a/libstdc++-v3/include/bits/simd_flags.h b/libstdc++-v3/include/
bits/simd_flags.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_flags.h
@@ -0,0 +1,246 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_details.h"
+#include <bits/align.h> // assume_aligned
+
+ template <typename _Tp, typename _Up = typename _Tp::value_type>
+ struct alignment
+ struct alignment<basic_vec<_Tp, _Ap>, _Up>
+ : integral_constant<size_t, alignof(basic_vec<_Tp, _Ap>)>
+ {};
+ template <typename _Tp, typename _Up = typename _Tp::value_type>
+ constexpr size_t alignment_v = alignment<_Tp, _Up>::value;
+
+ * <tt>struct convert-flag</tt>
+ *
+ * <tt>struct aligned-flag</tt>
+ *
+ { return assume_aligned<simd::alignment_v<_Tp,
remove_cv_t<_Up>>>(__ptr); }
+ };
+ * <tt>template<size_t N> struct overaligned-flag</tt>
+ *
+ * @tparam _Np alignment in bytes
+ *
+ template <size_t _Np>
+ struct __overaligned_flag
+ static_assert(__has_single_bit(_Np));
+
+ { return assume_aligned<_Np>(__ptr); }
+ };
+ // L1: 0, 1
+ // L2: 0, 2
+ // L3: 0, 3
+ // (exclusive cache line) for writing: 1, 0 / 1, 1
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_T0);
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_T1);
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_T2);
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_ET0);
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_ET1);
+ _mm_prefetch(reinterpret_cast<char const*>(__ptr), _MM_HINT_NTA);*/
+ return __ptr;
+#endif
+
+ concept __loadstore_tag = is_base_of_v<_LoadStoreTag, _Tp>;
+
+ consteval bool
+ _M_is_equal(flags) const
+ consteval bool
+ _M_is_equal(flags<_Other...> __y) const
+ { return same_as<flags<>, decltype(_M_xor(__y))>; }
+
+ consteval bool
+ _M_test(flags<_Other...> __x) const noexcept
+ { return __x._M_is_equal(_M_and(__x)); }
+ friend consteval auto
+ operator|(flags, flags<>)
+ friend consteval auto
+ operator|(flags, flags<_T0, _More...>)
+ if constexpr ((same_as<_Traits, _T0> || ...))
+ return flags<_Traits...>{} | flags<_More...>{};
+ consteval auto
+ _M_and(flags<>) const
+ consteval auto
+ _M_and(flags<_T0, _More...>) const
+ if constexpr ((same_as<_Traits, _T0> || ...))
+ return flags<_T0>{} | (flags{}._M_and(flags<_More...>{}));
+ consteval auto
+ _M_xor(flags<>) const
+ consteval auto
+ _M_xor(flags<_T0, _More...>) const
+ if constexpr ((same_as<_Traits, _T0> || ...))
+ {
+ = (conditional_t<same_as<_Traits, _T0>, flags<>,
+ flags<_Traits>>{} | ...);
+ inline constexpr flags<> flag_default {};
+
+ inline constexpr flags<__convert_flag> flag_convert {};
+
+ inline constexpr flags<__aligned_flag> flag_aligned {};
+
+ template <size_t _Np>
+ requires(__has_single_bit(_Np))
+ inline constexpr flags<__overaligned_flag<_Np>> flag_overaligned {};
+
+ * Pass to unchecked_load or unchecked_store to make it behave like
partial_load / partial_store.
+ */
+#endif
+}
+#endif // C++26
+#endif // _GLIBCXX_SIMD_FLAGS_H
diff --git a/libstdc++-v3/include/bits/simd_iterator.h b/libstdc++-v3/include/
bits/simd_iterator.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_iterator.h
@@ -0,0 +1,153 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_details.h"
+
+ * Iterator type for basic_vec and basic_mask.
+ *
+ friend class basic_vec;
+
+ template <size_t, typename>
+ friend class basic_mask;
+
+ _Vp* _M_data = nullptr;
+
+ public:
+ using value_type = typename _Vp::value_type;
+
+ using iterator_category = input_iterator_tag;
+
+ using iterator_concept = random_access_iterator_tag;
+
+ using difference_type = __simd_size_type;
+
+ __iterator(const __iterator<remove_const_t<_Vp>> &__i) requires
is_const_v<_Vp>
+ : _M_data(__i._M_data), _M_offset(__i._M_offset)
+ constexpr value_type
+ operator*() const
+ operator+=(difference_type __x)
+ {
+ operator-=(difference_type __x)
+ {
+ constexpr value_type
+ operator[](difference_type __i) const
+ { return (*_M_data)[_M_offset + __i]; } // checked in operator[]
+ constexpr friend bool operator==(__iterator __a, __iterator __b) =
default;
+
+ constexpr friend bool operator==(__iterator __a,
std::default_sentinel_t) noexcept
+ { return __a._M_offset == _Vp::size.value; }
+ operator+(const __iterator& __it, difference_type __x)
+ { return __iterator(*__it._M_data, __it._M_offset + __x); }
+ operator+(difference_type __x, const __iterator& __it)
+ { return __iterator(*__it._M_data, __it._M_offset + __x); }
+ operator-(const __iterator& __it, difference_type __x)
+ { return __iterator(*__it._M_data, __it._M_offset - __x); }
+ constexpr friend difference_type
+ operator-(__iterator __a, __iterator __b)
+ constexpr friend difference_type
+ operator-(__iterator __it, std::default_sentinel_t)
+ { return __it._M_offset - difference_type(_Vp::size.value); }
+
+ constexpr friend difference_type
+ operator-(std::default_sentinel_t, __iterator __it)
+ { return difference_type(_Vp::size.value) - __it._M_offset; }
+ };
+#endif // C++26
+#endif // _GLIBCXX_SIMD_ITERATOR_H
diff --git a/libstdc++-v3/include/bits/simd_loadstore.h b/libstdc++-v3/
include/bits/simd_loadstore.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_loadstore.h
@@ -0,0 +1,387 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_vec.h"
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ { using type = basic_vec<_Tp>; };
+
+ using __load_mask_type_t = typename __vec_load_return_t<_Vp,
_Tp>::mask_type;
+
+ = ranges::contiguous_range<_Tp> && ranges::sized_range<_Tp>;
+
+ constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
+ unchecked_load(_Rg&& __r, flags<_Flags...> __f = {})
+ {
+ using _Tp = ranges::range_value_t<_Rg>;
+ using _RV = __vec_load_return_t<_Vp, _Tp>;
+ using _Rp = typename _RV::value_type;
+ static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>,
_Rp, _Flags...>,
+ "The converting load is not value-preserving. "
+ "Pass 'flag_convert' if lossy conversion matches the
intent.");
+
+ constexpr bool __allow_out_of_bounds
+ = (... || is_same_v<_Flags, __partial_loadstore_flag>);
+ constexpr size_t __static_size = __static_range_size(__r);
+
+ static_assert(__static_size >= _RV::size.value || __allow_out_of_bounds
+ || __static_size == dynamic_extent, "Out-of-bounds simd
load");
+
+ "Input range is too small. Did you mean to use 'partial_load'?");
+
+ if consteval
+ {
+ return _RV([&](size_t __i) {
+ return __i < __rg_size ? static_cast<typename
_Rp::value_type>(__r[__i])
+ : _Rp();
+ return _RV([&](size_t __i) {
+ return __i < __rg_size ? static_cast<_Rp>(__r[__i]) :
_Rp();
+ });
+ if constexpr ((__static_size != dynamic_extent && __static_size >=
size_t(_RV::size()))
+ || !__allow_out_of_bounds)
+ constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
+ unchecked_load(_Rg&& __r, const __load_mask_type_t<_Vp,
ranges::range_value_t<_Rg>>& __mask,
+ flags<_Flags...> __f = {})
+ using _RV = __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>;
+ using _Rp = typename _RV::value_type;
+ static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>,
_Rp, _Flags...>,
+ "The converting load is not value-preserving. "
+ "Pass 'flag_convert' if lossy conversion matches the
intent.");
+
+ constexpr bool __allow_out_of_bounds = (... || is_same_v<_Flags,
__partial_loadstore_flag>);
+ constexpr auto __static_size = __static_range_size(__r);
+ static_assert(__static_size >= _RV::size.value || __allow_out_of_bounds
+ || __static_size == dynamic_extent, "Out-of-bounds simd
load");
+
+ ranges::size(__r) >= size_t(_RV::size()),
+ "Input range is too small. Did you mean to use 'partial_load'?");
+
+ const size_t __rg_size = ranges::size(__r);
+ if (__builtin_is_constant_evaluated())
+ return _RV([&](size_t __i) { return __i < __rg_size &&
__mask[int(__i)] ? __r[__i]
+
: _Rp(); });
+ return _RV([&](size_t __i) { return __mask[int(__i)] ? __r[__i] :
_Rp(); });
+ }
+ || (__static_size != dynamic_extent
+ && __static_size >=
size_t(_RV::size.value)))
+ return _RV::_S_masked_load(__ptr, __mask);
+ else if (__rg_size >= size_t(_RV::size()))
+ return _RV::_S_masked_load(__ptr, __mask);
+ __ptr, __mask &&
_RV::mask_type::_S_partial_mask_of_n(int(__rg_size)));
+ else
+ template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
+ [[__gnu__::__always_inline__]]
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ unchecked_load(_It __first, iter_difference_t<_It> __n, flags<_Flags...>
__f = {})
+ { return unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __n),
__f); }
+
+ template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
+ [[__gnu__::__always_inline__]]
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ unchecked_load(_It __first, iter_difference_t<_It> __n,
+ const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
+ flags<_Flags...> __f = {})
+ { return unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __n),
__mask, __f); }
+
+ template <typename _Vp = void, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ unchecked_load(_It __first, _Sp __last, flags<_Flags...> __f = {})
+ { return unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first,
__last), __f); }
+
+ template <typename _Vp = void, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ unchecked_load(_It __first, _Sp __last,
+ const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
+ flags<_Flags...> __f = {})
+ { return unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first,
__last), __mask, __f); }
+
+ constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
+ partial_load(_Rg&& __r, flags<_Flags...> __f = {})
+ { return unchecked_load<_Vp>(__r, __f | __allow_partial_loadstore); }
+
+ constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
+ partial_load(_Rg&& __r, const __load_mask_type_t<_Vp,
ranges::range_value_t<_Rg>>& __mask,
+ flags<_Flags...> __f = {})
+ { return unchecked_load<_Vp>(__r, __mask, __f |
__allow_partial_loadstore); }
+
+ template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
+ [[__gnu__::__always_inline__]]
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ partial_load(_It __first, iter_difference_t<_It> __n, flags<_Flags...>
__f = {})
+ { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __n),
__f); }
+
+ template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
+ [[__gnu__::__always_inline__]]
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ partial_load(_It __first, iter_difference_t<_It> __n,
+ const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
+ flags<_Flags...> __f = {})
+ { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __n),
__mask, __f); }
+
+ template <typename _Vp = void, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ partial_load(_It __first, _Sp __last, flags<_Flags...> __f = {})
+ { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first,
__last), __f); }
+
+ template <typename _Vp = void, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
+ partial_load(_It __first, _Sp __last, const __load_mask_type_t<_Vp,
iter_value_t<_It>>& __mask,
+ flags<_Flags...> __f = {})
+ { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first,
__last), __mask, __f); }
+
+ requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r,
flags<_Flags...> __f = {})
+ {
+ using _TV = basic_vec<_Tp, _Ap>;
+ static_assert(destructible<_TV>);
+ static_assert(__loadstore_convertible_to<_Tp,
ranges::range_value_t<_Rg>, _Flags...>,
+ "The converting store is not value-preserving. "
+ "Pass 'flag_convert' if lossy conversion matches the
intent.");
+
+ constexpr bool __allow_out_of_bounds = (... || is_same_v<_Flags,
__partial_loadstore_flag>);
+ constexpr auto __static_size = __static_range_size(__r);
+ static_assert(__static_size >= _TV::size.value || __allow_out_of_bounds
+ || __static_size == dynamic_extent, "Out-of-bounds simd
store");
+
+ "output range is too small. Did you mean to use 'partial_store'?");
+
+ __ptr[__i] = static_cast<ranges::range_value_t<_Rg>>(__v[__i]);
+ }
+ else if constexpr ((__static_size != dynamic_extent && __static_size >=
_TV::size())
+ || !__allow_out_of_bounds)
+ __ptr[__i] =
static_cast<std::ranges::range_value_t<_Rg>>(__v[__i]);
+ }
+ requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
+ flags<_Flags...> __f = {})
+ using _TV = basic_vec<_Tp, _Ap>;
+ static_assert(__loadstore_convertible_to<_Tp,
ranges::range_value_t<_Rg>, _Flags...>,
+ "The converting store is not value-preserving. "
+ "Pass 'flag_convert' if lossy conversion matches the
intent.");
+
+ constexpr bool __allow_out_of_bounds = (... || is_same_v<_Flags,
__partial_loadstore_flag>);
+ constexpr auto __static_size = __static_range_size(__r);
+ static_assert(__static_size >= _TV::size.value || __allow_out_of_bounds
+ || __static_size == dynamic_extent, "Out-of-bounds simd
store");
+
+ ranges::size(__r) >= size_t(_TV::size()),
+ "output range is too small. Did you mean to use 'partial_store'?");
+
+ const size_t __rg_size = ranges::size(__r);
+ if (__builtin_is_constant_evaluated())
+ if (__mask[__i] && (!__allow_out_of_bounds || size_t(__i) <
__rg_size))
+ __ptr[__i] =
static_cast<ranges::range_value_t<_Rg>>(__v[__i]);
+ }
+ else if (__allow_out_of_bounds && __rg_size < size_t(_TV::size()))
+ _TV::_S_masked_store(__v, __ptr,
+ __mask &&
_TV::mask_type::_S_partial_mask_of_n(int(__rg_size)));
+ else
+ template <typename _Tp, typename _Ap, contiguous_iterator _It, typename...
_Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first,
+ iter_difference_t<_It> __n, flags<_Flags...> __f = {})
+ { unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __n), __f);
}
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It, typename...
_Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first,
iter_difference_t<_It> __n,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
+ flags<_Flags...> __f = {})
+ { unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __n),
__mask, __f); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
+ flags<_Flags...> __f = {})
+ { unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __last),
__f); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
+ flags<_Flags...> __f = {})
+ { unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __last),
__mask, __f); }
+
+ requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r, flags<_Flags...>
__f = {})
+ { unchecked_store(__v, __r, __f | __allow_partial_loadstore); }
+
+ requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
+ flags<_Flags...> __f = {})
+ { unchecked_store(__v, __r, __mask, __f | __allow_partial_loadstore); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It, typename...
_Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first,
iter_difference_t<_It> __n,
+ flags<_Flags...> __f = {})
+ { partial_store(__v, span(__first, __n), __f); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It, typename...
_Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first,
iter_difference_t<_It> __n,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
flags<_Flags...> __f = {})
+ { partial_store(__v, span(__first, __n), __mask, __f); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
+ flags<_Flags...> __f = {})
+ { partial_store(__v, span(__first, __last), __f); }
+
+ template <typename _Tp, typename _Ap, contiguous_iterator _It,
sized_sentinel_for<_It> _Sp,
+ typename... _Flags>
+ requires indirectly_writable<_It, _Tp>
+ [[__gnu__::__always_inline__]]
+ partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
flags<_Flags...> __f = {})
+ { partial_store(__v, span(__first, __last), __mask, __f); }
+}
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_LOADSTORE_H
diff --git a/libstdc++-v3/include/bits/simd_mask.h b/libstdc++-v3/include/
bits/simd_mask.h
new file mode 100644
index 00000000000..ac3a1ea8e94
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_mask.h
@@ -0,0 +1,1677 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_iterator.h"
+#include "vec_ops.h"
+#if _GLIBCXX_X86
+#include "simd_x86.h"
+#endif
+
+#include <bitset>
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ consteval unsigned
+ operator()(unsigned __i, unsigned __size) const
+ __builtin_abort(); // swap_neighbors<N> permutation requires a
multiple of 2N elements
+ else if (std::has_single_bit(_Np))
+ return __i ^ _Np;
+ template <size_t _Np, size_t _Mp>
+ constexpr auto
+ __bitset_split(const bitset<_Mp>& __b)
+ {
+ bitset<_Np> _M_lo;
+ bitset<_Mp - _Np> _M_hi;
+ };
+ static_assert(_Mp <= __bits_per_ullong);
+ using _Lo = _Bitmask<_Np>;
+ return _Tmp {static_cast<_Lo>(__b.to_ullong()),
static_cast<_Hi>(__b.to_ullong() >> _Np)};
+ }
+ { using type = __similar_vec<_Tp, _Vp::size(), typename _Vp::abi_type>;
};
+
+ { using type = __similar_mask<_Tp, _Mp::size(), typename _Mp::abi_type>;
};
+
+ using rebind_t = typename rebind<_Tp, _Vp>::type;
+
+ requires requires { typename _Vp::mask_type; }
+ //requires requires { typename __deduce_abi_t<typename _Vp::value_type,
_Np>; }
+ struct resize<_Np, _Vp, _Traits>
+ { using type = __similar_vec<typename _Vp::value_type, _Np, typename
_Vp::abi_type>; };
+
+ //requires requires { typename __deduce_abi_t<typename _Mp::value_type,
_Np>; }
+ struct resize<_Np, _Mp, _Traits>
+ using _A1 = decltype(__abi_rebind<__mask_element_size<_Mp>, _Np,
typename _Mp::abi_type,
+ true>());
+ static_assert(_Mp::abi_type::_S_variant == _A1::_S_variant ||
__scalar_abi_tag<_A1>
+ || __scalar_abi_tag<typename _Mp::abi_type>);
+
+ using type = basic_mask<__mask_element_size<_Mp>, _A1>;
+ };
+ using resize_t = typename resize<_Np, _Vp>::type;
+
+ static constexpr __simd_size_type zero_element = -1 << (sizeof(int) *
__CHAR_BIT__ - 1);
+
+ static constexpr __simd_size_type uninit_element = zero_element + 1;
+
+ constexpr resize_t<_Np == 0 ? _Vp::size() : _Np, _Vp>
+ permute(const _Vp& __v, _IdxMap&& __idxmap)
+ { return resize_t<_Np == 0 ? _Vp::size() : _Np,
_Vp>::_S_static_permute(__v, __idxmap); }
+
+ constexpr resize_t<_Ip::size(), _Vp>
+ permute(const _Vp& __v, const _Ip& __indices)
+ constexpr auto
+ chunk(const basic_vec<typename _Vp::value_type, _Ap>& __x) noexcept
+ { return __x.template _M_chunk<_Vp>(); }
+ constexpr auto
+ chunk(const basic_mask<__mask_element_size<_Mp>, _Ap>& __x) noexcept
+ { return __x.template _M_chunk<_Mp>(); }
+ constexpr auto
+ chunk(const basic_vec<_Tp, _Ap>& __x) noexcept
+ { return chunk<resize_t<_Np, basic_vec<_Tp, _Ap>>>(__x); }
+
+ template<__simd_size_type _Np, size_t _Bytes, typename _Ap>
+ constexpr auto
+ chunk(const basic_mask<_Bytes, _Ap>& __x) noexcept
+ { return chunk<resize_t<_Np, basic_mask<_Bytes, _Ap>>>(__x); }
+
+ constexpr basic_vec<_Tp, __deduce_abi_t<_Tp, (_Abis::_S_size + ...)>>
+ cat(const basic_vec<_Tp, _Abis>&... __xs) noexcept
+ { return basic_vec<_Tp, __deduce_abi_t<_Tp, (_Abis::_S_size +
...)>>::_S_concat(__xs...); }
+
+ template<size_t _Bytes, typename... _Abis>
+ constexpr basic_mask<_Bytes, __deduce_abi_t<__integer_from<_Bytes>,
(_Abis::_S_size + ...)>>
+ cat(const basic_mask<_Bytes, _Abis>&... __xs) noexcept
+ { static_assert(false, "TODO: cat"); }
+
+ template <size_t _Bytes, typename _Ap>
+ class basic_mask
+ {
+ public:
+ using value_type = bool;
+
+ using abi_type = _Ap;
+
+ "of template arguments to basic_mask.")
+
+ basic_mask() = _GLIBCXX_DELETE_SIMD;
+
+ ~basic_mask() = _GLIBCXX_DELETE_SIMD;
+
+ basic_mask(const basic_mask&) = _GLIBCXX_DELETE_SIMD;
+
+ basic_mask& operator=(const basic_mask&) = _GLIBCXX_DELETE_SIMD;
+
+#undef _GLIBCXX_DELETE_SIMD
+ };
+ template <size_t _Bytes, __abi_tag _Ap>
+ requires (_Ap::_S_nreg == 1)
+ class basic_mask<_Bytes, _Ap>
+ {
+ template <size_t, typename>
+ friend class basic_mask;
+
+ friend class basic_vec;
+
+ static constexpr bool _S_is_scalar = __scalar_abi_tag<_Ap>;
+
+ static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask;
+
+ static constexpr bool _S_is_partial = _S_size != _S_full_size;
+
+ static_assert(destructible<_VecType>);
+
+ static constexpr bool _S_has_bool_member = _S_is_scalar;
+
+ static constexpr size_t _S_padding_bytes = 0;
+
+ public:
+ using value_type = bool;
+
+ using abi_type = _Ap;
+
+ using iterator = __iterator<basic_mask>;
+
+ using const_iterator = __iterator<const basic_mask>;
+
+ begin() noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ begin() const noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ cbegin() const noexcept
+ { return {*this, 0}; }
+ constexpr default_sentinel_t
+ end() const noexcept
+ { return {}; }
+ constexpr default_sentinel_t
+ cend() const noexcept
+ { return {}; }
+ static constexpr basic_mask
+ _S_init(_DataType __x)
+ basic_mask __r;
+ __r._M_data = __x;
+ static constexpr basic_mask
+ _S_init(unsigned_integral auto __bits)
+ { return basic_mask(__bits); }
+
+ * Bit-cast the given object \p __x to basic_mask.
+ *
+ * This is necessary for _S_nreg > 1 where the last element can be bool
or when the sizeof
+ * doesn't match because of different alignment requirements of the
sub-masks.
+ template <size_t _UBytes, typename _UAbi>
+ [[__gnu__::__always_inline__]]
+ static constexpr basic_mask
+ _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
+ { return __builtin_bit_cast(basic_mask, __x._M_concat_data()); }
+
+ constexpr auto
+ _M_concat_data() const
+ static constexpr basic_mask
+ _S_partial_mask_of_n(int __n)
+ static_assert(!_S_is_scalar);
+ if constexpr (!_S_use_bitmask)
+ __glibcxx_simd_precondition(__n >= 0 && __n <=
numeric_limits<_Ip>::max(),
+ "_S_partial_mask_of_n without
_S_use_bitmask requires "
+ "The x86 BZHI instruction requires
__n to "
+ "only use bits 0:7");
+#endif
+#if __has_builtin(__builtin_ia32_bzhi_di)
+#endif
+ if constexpr (_S_size <= 32)
+ static_assert(false);
+ }
+ constexpr basic_mask&
+ _M_and_neighbors()
+ constexpr basic_mask&
+ _M_or_neighbors()
+ constexpr auto _M_chunk() const noexcept
+ {
+ return tuple<basic_mask> {*this};
+ }
+ // convert to whatever _Mp uses first and then recurse into
_M_chunk
+ return resize_t<_S_size, _Mp>(*this).template _M_chunk<_Mp>();
+ else if constexpr (_S_use_bitmask && _Mp::_S_use_bitmask)
+ static_assert(is_unsigned_v<_DataType>);
+ if constexpr (__rem == 0)
+ using _Rest = resize_t<__rem, _Mp>;
+ return tuple {_Mp::_S_init(_M_data >> (__is *
__stride))...,
+ if constexpr (is_same_v<typename
_Rest::_DataType, bool>)
+ return operator[](__n * _Mp::_S_size);
+ static_assert(is_same_v<__vec_value_type<typename
_Mp::_DataType>,
+ __vec_value_type<_DataType>>);
+ using _Rest = resize_t<__rem, _Mp>;
+ return tuple {
+ if constexpr (is_same_v<typename _Rest::_DataType, bool>)
+ return operator[](__n * _Mp::_S_size);
+ basic_mask() = default;
+
+ basic_mask(_DataType __x) requires(!_S_is_scalar && !_S_use_bitmask)
+ : _M_data(__x)
+ // [simd.mask.ctor] broadcast constructor
-------------------------------
+ [[__gnu__::__always_inline__]]
+ basic_mask(same_as<bool> auto __x) noexcept
+ : _M_data(__x ? _S_implicit_mask : _DataType())
+ // [simd.mask.ctor] conversion constructor
------------------------------
+ template <size_t _UBytes, typename _UAbi>
+ requires (_S_size == _UAbi::_S_size)
+ basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
+ : _M_data([&] [[__gnu__::__always_inline__]] {
+ using _UV = basic_mask<_UBytes, _UAbi>;
+ // bool to bool
+ if constexpr (_S_is_scalar)
+ // converting from an "array of bool"
+ else if constexpr (_UV::_S_is_scalar)
+ // vec-/bit-mask to bit-mask | bit-mask to vec-mask
+ else if constexpr (_S_use_bitmask || _UV::_S_use_bitmask)
+ return basic_mask(__x.to_bitset())._M_data;
+
+ // vec-mask to vec-mask
+ else if constexpr (sizeof(__x) == sizeof(_M_data) && _Bytes ==
_UBytes
+ // TODO: turn this into a __vec_mask_cast overload in
simd_x86.h
+ if constexpr (_Bytes == 1 && _UBytes == 2)
+#endif
+ return __vec_mask_cast<_DataType>(__x._M_concat_data());
+ // [simd.mask.ctor] generator constructor
-------------------------------
+ template <__simd_generator_invokable<bool, _S_size> _Fp>
+ [[__gnu__::__always_inline__]]
+ basic_mask(_Fp&& __gen)
+ : _M_data([&] [[__gnu__::__always_inline__]] {
+ // [simd.mask.ctor] bitset constructor
----------------------------------
+ [[__gnu__::__always_inline__]]
+ basic_mask(const same_as<bitset<size()>> auto& __b) noexcept
+ : basic_mask(static_cast<_Bitmask<_S_size>>(__b.to_ullong()))
+ {
+ static_assert(_S_size <= 64); // more than 64 elements in one
register? not yet.
+ }
+ // [simd.mask.ctor] uint constructor
------------------------------------
+ template <unsigned_integral _Tp>
+ requires (!same_as<_Tp, bool>)
+ [[__gnu__::__always_inline__]]
+ basic_mask(_Tp __val) noexcept
+ : _M_data([&] [[__gnu__::__always_inline__]] () {
+ return bool(__val & 1);
+ else if (__is_const_known(__val))
+ using _Ip = typename _VecType::value_type;
+ _VecType __v0 = _Ip(__val);
+ static_assert(_Bytes == 1);
+ static_assert(sizeof(_Ip) == 1);
+ _Bitmask<_S_size> __bits = __val;
+ static_assert(sizeof(_VecType) % sizeof(__bits) == 0);
+ if constexpr (sizeof(_DataType) == 32)
+ using _V1 = vec<_Ip, sizeof(__bits)>;
+ _V1 __v1 = __builtin_bit_cast(_V1, __bits);
+ // [simd.mask.subscr]
---------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr value_type
+ operator[](__simd_size_type __i) const
+ return bool((_M_data >> __i) & 1);
+ else
+ constexpr basic_mask
+ operator!() const noexcept
+ {
+ operator+() const noexcept requires destructible<_VecType>
+ { return operator _VecType(); }
+ operator+() const noexcept = delete;
+
+ operator-() const noexcept requires destructible<_VecType>
+ {
+ using _Ip = typename _VecType::value_type;
+ if constexpr (_S_is_scalar)
+ static_assert(sizeof(_VecType) == sizeof(_M_data));
+ return __builtin_bit_cast(_VecType, _M_data);
+ operator-() const noexcept = delete;
+
+ operator~() const noexcept requires destructible<_VecType>
+ {
+ using _Ip = typename _VecType::value_type;
+ if constexpr (_S_is_scalar)
+ static_assert(sizeof(_VecType) == sizeof(_M_data));
+ return __builtin_bit_cast(_VecType, _M_data) - _Ip(1);
+ operator~() const noexcept = delete;
+
+ // [simd.mask.conv]
-----------------------------------------------------
+ template <typename _Up, typename _UAbi>
+ operator basic_vec<_Up, _UAbi>() const noexcept
+ {
+ // [simd.mask.namedconv]
------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr bitset<_S_size>
+ to_bitset() const noexcept
+ {
+ static_assert(_S_size <= 64); // more than 64 elements in one
register? not yet.
+ return to_ullong();
+ }
+ * \tparam _Offset Adjust the return type & value to start at bit
\p _Offset.
+ */
+ static_assert(__nbits + _Offset <= 64);
+ // before shifting
+ if constexpr (_Bytes != 2) // movmskb would duplicate each
bit
+ __uint = _U0(__x86_movmsk(_M_data));
+ static_assert(false);
+ if constexpr (_S_is_partial)
+ // TODO: with AVX512 use __builtin_ia32_cvt[bwdq]2mask(128|
256|512)
+ // TODO: Ask for compiler builtin to do the best of the
above. This should also
+ // combine with a preceding vector-mask compare to produce
a bit-mask compare (on
+ // AVX512)
+ }
+#endif
+ using _IV = _VecType;
+ static_assert(destructible<_IV>);
+ const typename _IV::mask_type& __k = [&]
[[__gnu__::__always_inline__]] () {
+ if constexpr (is_same_v<typename _IV::mask_type, basic_mask>)
+ return *this;
+ return typename _IV::mask_type(*this);
+ }();
+ ._M_reduce(bit_or<>()))) << _Offset;
+ }
+ return __i * 8 < __n ? __i * 8 : uninit_element;
+ });
+ to_ullong() const
+ { return _M_to_uint(); }
+ friend constexpr basic_mask
+ operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data & __y._M_data); }
+ friend constexpr basic_mask
+ operator||(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data | __y._M_data); }
+ friend constexpr basic_mask
+ operator&(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data & __y._M_data); }
+ friend constexpr basic_mask
+ operator|(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data | __y._M_data); }
+ friend constexpr basic_mask
+ operator^(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data ^ __y._M_data); }
+ // [simd.mask.cassign]
--------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ friend constexpr basic_mask&
+ operator&=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask&
+ operator|=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask&
+ operator^=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask
+ operator==(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !(__x ^ __y); }
+ friend constexpr basic_mask
+ operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x ^ __y; }
+ friend constexpr basic_mask
+ operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x || !__y; }
+ friend constexpr basic_mask
+ operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !__x || __y; }
+ friend constexpr basic_mask
+ operator>(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x && !__y; }
+ friend constexpr basic_mask
+ operator<(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !__x && __y; }
+ // [simd.mask.cond]
-----------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ friend constexpr basic_mask
+ __select_impl(const basic_mask& __k, const basic_mask& __t, const
basic_mask& __f) noexcept
+ {
+ // This pattern, is recognized to match the x86 blend
instructions, which only consider
+ // the sign bit of the mask register. Also, without SSE4, if the
compiler knows that __k
+ // is a vector-mask, then the '< 0' is elided.
+#endif
+ return __k._M_data ? __t._M_data : __f._M_data;
+ friend constexpr basic_mask
+ __select_impl(const basic_mask& __k, same_as<bool> auto __t,
same_as<bool> auto __f) noexcept
+ {
+ return basic_mask(__t);
+ else
+ template <__vectorizable _T0, same_as<_T0> _T1>
+ requires (sizeof(_T0) == _Bytes)
+ friend constexpr vec<_T0, _S_size>
+ __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f)
noexcept
+ {
+ using _Vp = vec<_T0, _S_size>;
+ using _Mp = typename _Vp::mask_type;
+ return __select_impl(_Mp(__k), _Vp(__t), _Vp(__f));
+ constexpr bool
+ _M_all_of() const noexcept
+ {
+ // PR120925 (partial kortest pattern not recognized)
+ return (_M_data & _S_implicit_mask) == _S_implicit_mask;
+#endif
+ else
+ constexpr bool
+ _M_any_of() const noexcept
+ {
+ // PR120925 (partial kortest pattern not recognized)
+ return (_M_data & _S_implicit_mask) != 0;
+#endif
+ else
+ constexpr bool
+ _M_none_of() const noexcept
+ {
+ // PR120925 (partial kortest pattern not recognized)
+ return (_M_data & _S_implicit_mask) == 0;
+#endif
+ else
+ _M_reduce_count() const noexcept
+ {
+ return __builtin_popcountll(to_ullong());
+ }
+ __glibcxx_simd_precondition(__bits, "An empty mask does not have a
min_index.");
+ if constexpr (_S_size == 1)
+ __glibcxx_simd_precondition(__bits, "An empty mask does not have a
max_index.");
+ if constexpr (_S_size == 1)
+ friend constexpr bool
+ __is_const_known(const basic_mask& __x)
+ { return __builtin_constant_p(__x._M_data); }
+ template <size_t _Bytes, __abi_tag _Ap>
+ requires (_Ap::_S_nreg > 1)
+ class basic_mask<_Bytes, _Ap>
+ {
+ template <size_t, typename>
+ friend class basic_mask;
+
+ friend class basic_vec;
+
+ using _Abi0 = conditional_t<_N0 == _Nreg0 || __scalar_abi_tag<_Ap>,
+ _ScalarAbi<_N0>, _Abi_t<_N0, _Nreg0,
_Ap::_S_variant>>;
+ using _Abi1 = conditional_t<_N1 == _Nreg1 || __scalar_abi_tag<_Ap>,
+ _ScalarAbi<_N1>, _Abi_t<_N1, _Nreg1,
_Ap::_S_variant>>;
+ using _Mask0 = basic_mask<_Bytes, _Abi0>;
+
+ using _Mask1 = basic_mask<_Bytes, _Abi1>;
+
+ // _Ap::_S_nreg determines how deep the recursion goes. E.g.
basic_mask<4, _Abi<8, 4>> cannot
+ // use basic_mask<4, _Abi<4, 1>> as _Mask0/1 types.
+ static_assert(_Mask0::abi_type::_S_nreg + _Mask1::abi_type::_S_nreg ==
_Ap::_S_nreg);
+
+ static constexpr bool _S_use_bitmask = _Mask0::_S_use_bitmask;
+
+ static constexpr bool _S_is_scalar = _Mask0::_S_is_scalar;
+
+ static constexpr bool _S_has_bool_member = _Mask1::_S_has_bool_member;
+
+ static constexpr size_t _S_padding_bytes
+ = __alignof__(_Mask0) - sizeof(_Mask1) + _Mask1::_S_padding_bytes;
+ public:
+ using value_type = bool;
+
+ using abi_type = _Ap;
+
+ using iterator = __iterator<basic_mask>;
+
+ using const_iterator = __iterator<const basic_mask>;
+
+ begin() noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ begin() const noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ cbegin() const noexcept
+ { return {*this, 0}; }
+ constexpr default_sentinel_t
+ end() const noexcept
+ { return {}; }
+ constexpr default_sentinel_t
+ cend() const noexcept
+ { return {}; }
+ static constexpr basic_mask
+ _S_init(const _Mask0& __x, const _Mask1& __y)
+ basic_mask __r;
+ __r._M_data0 = __x;
+ static constexpr basic_mask
+ _S_init(unsigned_integral auto __bits)
+ { return basic_mask(__bits); }
+
+ static constexpr basic_mask
+ _S_init(const __trivial_pair<_U0, _U1>& __bits)
+ if constexpr (is_unsigned_v<_U0>)
+ {
+ static_assert(is_unsigned_v<_U1>);
+ return _S_init(_Mask0(__bits._M_first),
_Mask1(__bits._M_second));
+ else if constexpr (is_unsigned_v<_U1>)
+ return _S_init(_Mask0::_S_init(__bits._M_first),
_Mask1(__bits._M_second));
+ template <size_t _UBytes, typename _UAbi>
+ [[__gnu__::__always_inline__]]
+ static constexpr basic_mask
+ _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
+ {
+ using _Mp = basic_mask<_UBytes, _UAbi>;
+ if constexpr (_Mp::_S_has_bool_member || sizeof(basic_mask) >
sizeof(__x))
+ return _S_init(__builtin_bit_cast(_Mask0, __x._M_data0),
+ else if constexpr (sizeof(basic_mask) == sizeof(__x))
+ return __builtin_bit_cast(basic_mask, __x);
+ else
+ { // e.g. on IvyBridge (different alignment => different sizeof)
+ struct _Tmp { alignas(_Mp) basic_mask _M_data; };
+ return __builtin_bit_cast(_Tmp, __x)._M_data;
+ constexpr auto
+ _M_concat_data() const
+ static_assert(_S_size <= sizeof(0ull) * __CHAR_BIT__, "cannot
concat more than 64 bits");
+ using _Up = _Bitmask<_S_size>;
+ static constexpr basic_mask
+ _S_partial_mask_of_n(int __n)
+#endif
+ if (__n < _N0)
+ constexpr basic_mask&
+ _M_and_neighbors()
+ constexpr basic_mask&
+ _M_or_neighbors()
+ constexpr auto
+ _M_chunk() const noexcept
+ {
+ [[maybe_unused]] constexpr auto [...__is] = _IotaArray<__n>;
+ if constexpr (_N0 == _Mp::_S_size)
+ if constexpr (__rem == 0 && is_same_v<_Mp, _Mask0>)
+ return array<_Mp, __n> {_M_data0, _M_data1};
+ return tuple<_Mp, resize_t<__rem, _Mp>> {_M_data0, _M_data1};
+ }
+ static_assert(!_Mp::_S_is_partial);
+ return __builtin_bit_cast(_Rp, *this);
+ using _Rest = resize_t<__rem, _Mp>;
+ // can't bit-cast because the member order of tuple is reversed
+ basic_mask() = default;
+
+ // TODO: any?
+
+ // [simd.mask.ctor] broadcast constructor
-------------------------------
+ [[__gnu__::__always_inline__]]
+ basic_mask(same_as<bool> auto __x) noexcept
+ : _M_data0(__x), _M_data1(__x)
+ // [simd.mask.ctor] conversion constructor
------------------------------
+ template <size_t _UBytes, typename _UAbi>
+ requires (_S_size == _UAbi::_S_size)
+ basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
+ : _M_data0([&] {
+ // [simd.mask.ctor] generator constructor
-------------------------------
+ template <__simd_generator_invokable<bool, _S_size> _Fp>
+ [[__gnu__::__always_inline__]]
+ basic_mask(_Fp&& __gen)
+ : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]]
(auto __i) {
+ // [simd.mask.ctor] bitset constructor
----------------------------------
+ [[__gnu__::__always_inline__]]
+ basic_mask(const same_as<bitset<size()>> auto& __b) noexcept
+ : _M_data0(__bitset_split<_N0>(__b)._M_lo),
_M_data1(__bitset_split<_N0>(__b)._M_hi)
+ // [simd.mask.ctor] uint constructor
------------------------------------------
+ template <unsigned_integral _Tp>
+ requires (!same_as<_Tp, bool>)
+ [[__gnu__::__always_inline__]]
+ basic_mask(_Tp __val) noexcept
+ : _M_data0(static_cast<_Bitmask<_N0>>(__val)),
+ _M_data1(sizeof(_Tp) * __CHAR_BIT__ > _N0
+ ? static_cast<_Bitmask<_N1>>(__val >> _N0) :
_Bitmask<_N1>())
+ {}
+ // [simd.mask.subscr]
---------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr value_type
+ operator[](__simd_size_type __i) const
+ // in some cases the last element can be 'bool' instead of bit-/
vector-mask;
+ // e.g. mask<short, 17> is {mask<short, 16>, mask<short, 1>}, where
the latter uses
+ // _ScalarAbi<1>, which is stored as 'bool'
+ if constexpr (_M_data1._S_has_bool_member)
+ else if constexpr (abi_type::_S_is_bitmask)
+ {
+ alignas(basic_mask) unsigned char _M_bytes[__div_ceil(_S_size,
__CHAR_BIT__)];
+ };
+ return bool((__builtin_bit_cast(_Tmp, *this)
+ ._M_bytes[__i / __CHAR_BIT__] >> (__i %
__CHAR_BIT__)) & 1);
+ alignas(basic_mask) __integer_from<_Bytes> _M_values[_S_size];
+ };
+ constexpr basic_mask
+ operator!() const noexcept
+ { return _S_init(!_M_data0, !_M_data1); }
+ operator+() const noexcept requires destructible<_VecType>
+ { return _VecType::_S_concat(+_M_data0, +_M_data1); }
+ operator+() const noexcept = delete;
+
+ operator-() const noexcept requires destructible<_VecType>
+ { return _VecType::_S_concat(-_M_data0, -_M_data1); }
+ operator-() const noexcept = delete;
+
+ operator~() const noexcept requires destructible<_VecType>
+ { return _VecType::_S_concat(~_M_data0, ~_M_data1); }
+ operator~() const noexcept = delete;
+
+ // [simd.mask.conv]
-----------------------------------------------------
+ template <typename _Up, typename _UAbi>
+ operator basic_vec<_Up, _UAbi>() const noexcept
+ {
+ using _Rp = basic_vec<_Up, _UAbi>;
+ return _Rp::_S_init(_M_data0, _M_data1);
+ // [simd.mask.namedconv]
------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr bitset<_S_size>
+ to_bitset() const noexcept
+ {
+ return to_ullong();
+ else
+ static_assert(_N0 % 64 == 0);
+ struct _Tmp
+ bitset<_N0> _M_lo;
+ bitset<_N1> _M_hi;
+ } __tmp = {_M_data0.to_bitset(), _M_data1.to_bitset()};
+ return __builtin_bit_cast(bitset<_S_size>, __tmp);
+ }
+ constexpr auto
+ _M_to_uint() const
+ static_assert(_Offset == 0);
+ return __trivial_pair {
+#endif
+ auto __uint = _M_data1.template _M_to_uint<_N0x + _Offset>();
+ to_ullong() const
+ {
+ __glibcxx_simd_precondition(_M_data1.to_ullong() == 0,
+ "to_ullong called on mask with 'true'
elements at indices"
+ "higher than 64");
+ return _M_data0.to_ullong();
+ }
+ friend constexpr basic_mask
+ operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data0 && __y._M_data0, __x._M_data1 &&
__y._M_data1); }
+ friend constexpr basic_mask
+ operator||(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data0 || __y._M_data0, __x._M_data1 ||
__y._M_data1); }
+ friend constexpr basic_mask
+ operator&(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data0 & __y._M_data0, __x._M_data1 &
__y._M_data1); }
+ friend constexpr basic_mask
+ operator|(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data0 | __y._M_data0, __x._M_data1 |
__y._M_data1); }
+ friend constexpr basic_mask
+ operator^(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return _S_init(__x._M_data0 ^ __y._M_data0, __x._M_data1 ^
__y._M_data1); }
+ // [simd.mask.cassign]
+ [[__gnu__::__always_inline__]]
+ friend constexpr basic_mask&
+ operator&=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask&
+ operator|=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask&
+ operator^=(basic_mask& __x, const basic_mask& __y) noexcept
+ {
+ friend constexpr basic_mask
+ operator==(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !(__x ^ __y); }
+ friend constexpr basic_mask
+ operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x ^ __y; }
+ friend constexpr basic_mask
+ operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x || !__y; }
+ friend constexpr basic_mask
+ operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !__x || __y; }
+ friend constexpr basic_mask
+ operator>(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return __x && !__y; }
+ friend constexpr basic_mask
+ operator<(const basic_mask& __x, const basic_mask& __y) noexcept
+ { return !__x && __y; }
+ // [simd.mask.cond]
-----------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ friend constexpr basic_mask
+ __select_impl(const basic_mask& __k, const basic_mask& __t, const
basic_mask& __f) noexcept
+ {
+ friend constexpr basic_mask
+ __select_impl(const basic_mask& __k, same_as<bool> auto __t,
same_as<bool> auto __f) noexcept
+ {
+ return basic_mask(__t);
+ else
+ template <__vectorizable _T0, same_as<_T0> _T1>
+ requires (sizeof(_T0) == _Bytes)
+ friend constexpr vec<_T0, _S_size>
+ __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f)
noexcept
+ {
+ using _Vp = vec<_T0, _S_size>;
+ return _Vp::_S_init(__select_impl(__k._M_data0, __t, __f),
+ constexpr bool
+ _M_all_of() const
+ constexpr bool
+ _M_any_of() const
+ constexpr bool
+ _M_none_of() const
+ __glibcxx_simd_precondition(__bits, "An empty mask does not have
a min_index.");
+ if constexpr (_S_size == 1)
+ __glibcxx_simd_precondition(__bits, "An empty mask does not have
a max_index.");
+ if constexpr (_S_size == 1)
+ friend constexpr bool
+ __is_const_known(const basic_mask& __x)
+ { return __is_const_known(__x._M_data0) &&
__is_const_known(__x._M_data1); }
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_MASK_H
diff --git a/libstdc++-v3/include/bits/simd_mask_reductions.h b/libstdc++-v3/
include/bits/simd_mask_reductions.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_mask_reductions.h
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_mask.h"
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ constexpr bool
+ all_of(const basic_mask<_Bytes, _Ap>& __k) noexcept
+ { return __k._M_all_of(); }
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ constexpr bool
+ any_of(const basic_mask<_Bytes, _Ap>& __k) noexcept
+ { return __k._M_any_of(); }
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ constexpr bool
+ none_of(const basic_mask<_Bytes, _Ap>& __k) noexcept
+ { return __k._M_none_of(); }
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_count(const basic_mask<_Bytes, _Ap>& __k) noexcept
+ {
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_min_index(const basic_mask<_Bytes, _Ap>& __k)
+ { return __k._M_reduce_min_index(); }
+ template <size_t _Bytes, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_max_index(const basic_mask<_Bytes, _Ap>& __k)
+ { return __k._M_reduce_max_index(); }
+ constexpr bool
+ all_of(same_as<bool> auto __x) noexcept
+ { return __x; }
+ constexpr bool
+ any_of(same_as<bool> auto __x) noexcept
+ { return __x; }
+ constexpr bool
+ none_of(same_as<bool> auto __x) noexcept
+ { return !__x; }
+ reduce_count(same_as<bool> auto __x) noexcept
+ { return __x; }
+ reduce_min_index(same_as<bool> auto __x)
+ { return 0; }
+ reduce_max_index(same_as<bool> auto __x)
+ { return 0; }
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_MASK_REDUCTIONS_H
diff --git a/libstdc++-v3/include/bits/simd_reductions.h b/libstdc++-v3/
include/bits/simd_reductions.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_reductions.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_vec.h"
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ inline constexpr _Tp __identity_element_for = nullptr;
+
+ inline constexpr _Tp __identity_element_for<_Tp, bit_and<>> =
_Tp(~_Tp());
+
+ inline constexpr _Tp __identity_element_for<_Tp, bit_or<>> = _Tp(0);
+
+ inline constexpr _Tp __identity_element_for<_Tp, bit_xor<>> = _Tp(0);
+
+ requires same_as<_BinaryOperation, plus<>>
+ || same_as<_BinaryOperation, multiplies<>>
+ || same_as<_BinaryOperation, bit_and<>>
+ || same_as<_BinaryOperation, bit_or<>>
+ || same_as<_BinaryOperation, bit_xor<>>
+ consteval _Tp __default_identity_element()
+ { return __identity_element_for<_Tp, _BinaryOperation>; }
+ reduce(const basic_vec<_Tp, _Ap>& __x, _BinaryOperation __binary_op = {})
+ { return __x._M_reduce(__binary_op); }
+ reduce(const basic_vec<_Tp, _Ap>& __x, const typename basic_vec<_Tp,
_Ap>::mask_type& __mask,
+ _BinaryOperation __binary_op = {}, type_identity_t<_Tp>
__identity_element
+ = __default_identity_element<_Tp, _BinaryOperation>())
+ template <totally_ordered _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_min(const basic_vec<_Tp, _Ap>& __x) noexcept
+ {
+ template <totally_ordered _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_min(const basic_vec<_Tp, _Ap>& __x,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask)
noexcept
+ {
+ return reduce(__select_impl(__mask, __x, numeric_limits<_Tp>::max()),
+ []<typename _UV>(const _UV& __a, const _UV& __b) {
+ template <totally_ordered _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_max(const basic_vec<_Tp, _Ap>& __x) noexcept
+ {
+ template <totally_ordered _Tp, typename _Ap>
+ [[__gnu__::__always_inline__]]
+ reduce_max(const basic_vec<_Tp, _Ap>& __x,
+ const typename basic_vec<_Tp, _Ap>::mask_type& __mask)
noexcept
+ {
+ return reduce(__select_impl(__mask, __x,
numeric_limits<_Tp>::lowest()),
+ []<typename _UV>(const _UV& __a, const _UV& __b) {
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_REDUCTIONS_H
diff --git a/libstdc++-v3/include/bits/simd_vec.h b/libstdc++-v3/include/bits/
simd_vec.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_vec.h
@@ -0,0 +1,2091 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_mask.h"
+#include "simd_flags.h"
+
+#include <bits/stl_function.h>
+#include <cmath>
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ // disabled basic_vec
+ template <typename _Tp, typename _Ap>
+ class basic_vec
+ {
+ public:
+ using value_type = _Tp;
+
+ using abi_type = _Ap;
+
+ using mask_type = basic_mask<0, void>; // disabled
+
+ "of template arguments to basic_vec.")
+
+ basic_vec() = _GLIBCXX_DELETE_SIMD;
+
+ ~basic_vec() = _GLIBCXX_DELETE_SIMD;
+
+ basic_vec(const basic_vec&) = _GLIBCXX_DELETE_SIMD;
+
+ basic_vec& operator=(const basic_vec&) = _GLIBCXX_DELETE_SIMD;
+
+#undef _GLIBCXX_DELETE_SIMD
+ };
+ using _Vp = basic_vec<_Tp, _Ap>;
+
+ operator+(const _Vp& __x, const _Vp& __y) noexcept
+ {
+ operator-(const _Vp& __x, const _Vp& __y) noexcept
+ {
+ operator*(const _Vp& __x, const _Vp& __y) noexcept
+ {
+ operator/(const _Vp& __x, const _Vp& __y) noexcept
+ {
+ operator%(const _Vp& __x, const _Vp& __y) noexcept
+ requires requires (_Tp __a) { __a % __a; }
+ operator&(const _Vp& __x, const _Vp& __y) noexcept
+ requires requires (_Tp __a) { __a & __a; }
+ operator|(const _Vp& __x, const _Vp& __y) noexcept
+ requires requires (_Tp __a) { __a | __a; }
+ operator^(const _Vp& __x, const _Vp& __y) noexcept
+ requires requires (_Tp __a) { __a ^ __a; }
+ class basic_vec<_Tp, _Ap>
+ : _BinaryOps<_Tp, _Ap>
+ friend class basic_vec;
+
+ template <size_t, typename>
+ friend class basic_mask;
+
+ static constexpr bool _S_is_scalar = __scalar_abi_tag<_Ap>;
+
+ static_assert(!_S_is_scalar || _S_size == 1);
+
+ static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask;
+
+ static constexpr bool _S_is_partial = sizeof(_M_data) > sizeof(_Tp) *
_S_size;
+
+ public:
+ using value_type = _Tp;
+
+ using abi_type = _Ap;
+
+ using mask_type = basic_mask<sizeof(_Tp), abi_type>;
+
+ using iterator = __iterator<basic_vec>;
+
+ using const_iterator = __iterator<const basic_vec>;
+
+ begin() noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ begin() const noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ cbegin() const noexcept
+ { return {*this, 0}; }
+ constexpr default_sentinel_t
+ end() const noexcept
+ { return {}; }
+ constexpr default_sentinel_t
+ cend() const noexcept
+ { return {}; }
+ static constexpr basic_vec
+ _S_init(_DataType __x)
+ basic_vec __r;
+ __r._M_data = __x;
+ friend constexpr bool
+ __is_const_known(const basic_vec& __x)
+ { return __builtin_constant_p(__x._M_data); }
+ constexpr auto
+ _M_concat_data() const
+ return __vec_builtin_type<value_type, 1>{_M_data};
+ else
+ static constexpr basic_vec
+ _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&&
__idxmap)
+ {
+ using _Xp = basic_vec<value_type, _A0>;
+ basic_vec __r;
+ if constexpr (_S_is_scalar)
+ constexpr __simd_size_type __j = [&] consteval {
+ if constexpr (__index_permutation_function_nosize<_Fp>)
+ if constexpr (__j == simd::zero_element || __j ==
simd::uninit_element)
+ return basic_vec();
+ else
+ static_assert(__j >= 0 && __j < _Xp::_S_size);
+ __r._M_data = __x[__j];
+ auto __idxmap2 = [=](auto __i) consteval {
+ if constexpr (int(__i + _Offset) >= _Size) // _S_full_size >
_Size
+ return __simd_size_constant<simd::uninit_element>;
+ else if constexpr (__index_permutation_function_nosize<_Fp>)
+ if constexpr (__j == simd::zero_element)
+ return
__simd_size_constant<__bit_ceil(unsigned(_Xp::_S_size))>;
+ else if constexpr (__j == simd::uninit_element)
+ return __simd_size_constant<-1>;
+ static_assert(__j >= 0 && __j < _Xp::_S_size);
+ return __simd_size_constant<__j>;
+ constexpr bool __needs_zero_element = [&] {
+ constexpr auto [...__is] = _IotaArray<_S_size>;
+ return ((__idxmap2(__simd_size_constant<__is>).value ==
simd::zero_element) || ...);
+ }();
+ constexpr auto
+ _M_chunk() const noexcept
+ {
+ using _Rest = resize_t<__rem, _Vp>;
+ _Rest __rest;
+ _M_assign_from(auto _Offset, const basic_vec<value_type, _A0>& __x0,
+ const basic_vec<value_type, _As>&... __xs)
+ {
+ static constexpr basic_vec
+ _S_concat(const basic_vec<value_type, _A0>& __x0) noexcept
+ { return static_cast<const basic_vec&>(__x0); }
+
+ static constexpr basic_vec
+ _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
+ {
+ && ((!basic_vec<value_type, _As>::_S_is_partial
+ && _As::_S_size * sizeof...(_As) ==
_S_size) && ...))
+ return
basic_vec::_S_init(__vec_concat(__xs._M_concat_data()...));
+
+ constexpr bool __simple_inserts
+ = sizeof...(_As) == 2 && _A1::_S_size <= 2
+ && is_same_v<_DataType, typename basic_vec<value_type,
_A0>::_DataType>;
+ if (!__builtin_is_constant_evaluated() && __simple_inserts)
+ basic_vec __r;
+ __r._M_data = __x0._M_data;
+ return
basic_vec::_S_init(__vec_concat_sized<_As::_S_size...>(
+ __xs._M_concat_data()...));
+ constexpr auto
+ _M_reduce_1(auto __binary_op) const
+ static_assert(__has_single_bit(unsigned(_S_size)));
+ auto [__a, __b] = chunk<_S_size / 2>(*this);
+ constexpr value_type
+ _M_reduce_tail(const auto& __rest, auto __binary_op) const
+ auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
+ return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
+ constexpr basic_vec
+ _M_elements_shifted_down() const
+ static_assert(_Shift < _S_size && _Shift > 0);
+#ifdef __SSE2__
+ if (!__is_const_known(*this))
+ return reinterpret_cast<_DataType>(
+ __builtin_ia32_psrldqi128(__vec_bit_cast<long
long>(_M_data),
+ _Shift *
sizeof(value_type) * 8));
+ else
+ const auto __x =
reinterpret_cast<__vec_builtin_type_bytes<long long, 16>>(
+ __vec_zero_pad_to_16(_M_data));
+ __x, _Shift * sizeof(value_type) *
8);
+ return
_VecOps<_DataType>::_S_extract(__vec_bit_cast<value_type>(__shifted));
+ }
+#endif
+ return _S_static_permute(*this, [](int __i) {
+ return __i + _Shift >= _S_size ? zero_element : __i +
_Shift;
+ });
+ constexpr value_type
+ _M_reduce(_BinaryOp __binary_op) const
+ else if constexpr (_Traits.template _M_eval_as_f32<value_type>()
+ && (is_same_v<_BinaryOp, plus<>>
+ || is_same_v<_BinaryOp, multiplies<>>))
+ return value_type(rebind_t<float,
basic_vec>(*this)._M_reduce(__binary_op));
+#ifdef __SSE2__
+ else if constexpr (is_integral_v<value_type> && sizeof(value_type) ==
1
+ && is_same_v<decltype(__binary_op),
multiplies<>>)
+ {
+ // convert to unsigned short because of missing 8-bit mul
instruction
+ // we don't need to preserve the order of elements
+ using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short,
basic_vec>>;
+ auto __a = __builtin_bit_cast(_V16, *this);
+#endif
+ else if constexpr (__has_single_bit(unsigned(_S_size)))
+ static_assert(_S_size <= 16);
+ auto __x = *this;
+#ifdef __SSE2__
+ if constexpr (sizeof(_M_data) <= 16 &&
is_integral_v<value_type>)
+ {
+#endif
+ if constexpr (_S_size > 8)
+#ifdef __SSE2__
+ // avoid pshufb by "promoting" to int
+ if constexpr (is_integral_v<value_type> && sizeof(value_type)
<= 1)
+ return resize_t<4, rebind_t<int, basic_vec>>(chunk<4>(__x)
[0])
+ ._M_reduce(__binary_op);
+#endif
+ if constexpr (_S_size > 2)
+ if constexpr (is_integral_v<value_type> && sizeof(value_type)
== 2)
+ return __binary_op(__x, _S_static_permute(__x,
_SwapNeighbors<1>()))[0];
+ return __binary_op(vec<value_type, 1>(__x[0]),
vec<value_type, 1>(__x[1]))[0];
+ }
+ // e.g. _S_size = 16 + 16 + 15 (vec<char, 47>)
+ // -> 8 + 8 + 7 -> 4 + 4 + 3 -> 2 + 2 + 1 -> 1
+ if constexpr (tuple_size_v<_Cp> == 4)
+ {
+ else if constexpr (tuple_size_v<_Cp> == 3)
+ {
+ static_assert(false);
+ }
+ // ISO/IEC 60559 on the classification operations (5.7.2 General
Operations):
+ // "They are never exceptional, even for signaling NaNs."
+ //
+ constexpr mask_type
+ _M_isnan() const requires is_floating_point_v<value_type>
+ {
+ return mask_type(false);
+ else if constexpr (_S_is_scalar)
+ return mask_type(std::isnan(_M_data));
+ else if constexpr (_S_use_bitmask)
+ return mask_type([&](int __i) { return std::isnan(_M_data[__i]);
});
+ else
+ // 60559: NaN is represented as Inf + non-zero mantissa bits
+ using _Ip = __integer_from<sizeof(value_type)>;
+ return __builtin_bit_cast(_Ip,
numeric_limits<value_type>::infinity())
+ < __builtin_bit_cast(rebind_t<_Ip, basic_vec>,
_M_fabs());
+ }
+ constexpr mask_type
+ _M_isinf() const requires is_floating_point_v<value_type>
+ {
+ return mask_type(false);
+ else if constexpr (_S_is_scalar)
+ return mask_type(std::isinf(_M_data));
+ else if (__is_const_known(_M_data))
+ return mask_type([&](int __i) { return std::isinf(_M_data[__i]);
});
+#ifdef _GLIBCXX_X86
+ else if constexpr (_S_use_bitmask)
+ return mask_type::_S_init(__x86_bitmask_isinf(_M_data));
+ else if constexpr (_Traits._M_have_avx512dq())
+ return __x86_bit_to_vecmask<typename mask_type::_DataType>(
+ __x86_bitmask_isinf(_M_data));
+#endif
+ else
+ using _Ip = __integer_from<sizeof(value_type)>;
+ return __vec_bit_cast<_Ip>(_M_fabs()._M_data)
+ == __builtin_bit_cast(_Ip,
numeric_limits<value_type>::infinity());
+ }
+ constexpr basic_vec
+ _M_abs() const
+ if constexpr (is_floating_point_v<value_type>)
+ return _M_fabs();
+ constexpr basic_vec
+ _M_fabs() const
+ static_assert(is_floating_point_v<value_type>);
+ return __vec_andnot(_S_signmask<_DataType>, _M_data);
+ constexpr mask_type
+ _M_isunordered(basic_vec __y) const requires
is_floating_point_v<value_type>
+ {
+ return mask_type(false);
+ else if constexpr (_S_is_scalar)
+ return mask_type(std::isunordered(_M_data, __y._M_data));
+#ifdef _GLIBCXX_X86
+ else if constexpr (_S_use_bitmask)
+#endif
+ else
+ return mask_type([&](int __i) {
+ return std::isunordered(_M_data[__i], __y._M_data[__i]);
+ });
+ * Implementation of @ref partial_load.
+ *
+ * @param __mem A pointer to an array of @p __n values. Can be complex
or real.
+ * @param __n Read no more than @p __n values from memory. However,
depending on @p __mem
+ * alignment, out of bounds reads are benign.
+ static inline basic_vec
+ _S_partial_load(const _Up* __mem, size_t __n)
+ {
+ return __n == 0 ? basic_vec() :
basic_vec(static_cast<value_type>(*__mem));
+ else if constexpr (__converts_trivially<_Up, value_type>)
+ {
+ const auto __k = __n < _S_size ?
mask_type::_S_partial_mask_of_n(int(__n))
+ : mask_type(true);
+ return _S_masked_load(__mem,
mask_type::_S_partial_mask_of_n(int(__n)));
+ }
+#endif
+ if (__n >= size_t(_S_size)) [[unlikely]]
+ return basic_vec(_LoadCtorTag(), __mem);
+#if _GLIBCXX_X86 // TODO: where else is this "safe"?
+ // allow out-of-bounds read when it cannot lead to a #GP
+ return
__select_impl(mask_type::_S_partial_mask_of_n(int(__n)),
+ basic_vec(_LoadCtorTag(), __mem),
basic_vec());
+#endif
+ else if constexpr (_S_size > 4)
+ alignas(_DataType) byte __dst[sizeof(_DataType)] = {};
+ const byte* __src = reinterpret_cast<const byte*>(__mem);
+ __memcpy_chunks<sizeof(_Up), sizeof(_DataType)>(__dst,
__src, __n);
+ return basic_vec();
+ else if constexpr (_S_size == 2)
+ return _DataType {static_cast<value_type>(__mem[0]), 0};
+ else
+ static_cast<value_type>(__mem[0]),
+ static_cast<value_type>(__is + 1 < __n ? __mem[__is + 1]
: 0)...
+ };
+ return static_cast<basic_vec>(rebind_t<_Up,
basic_vec>::_S_partial_load(__mem, __n));
+ }
+ * Implementation of @ref masked_load.
+ */
+ static inline basic_vec
+ _S_masked_load(const _Up* __mem, mask_type __k)
+ {
+ if constexpr (__converts_trivially<_Up, value_type>)
+ return __x86_masked_load<_DataType>(__mem, __k._M_data);
+ using _UV = rebind_t<_Up, basic_vec>;
+ return basic_vec(_UV::_S_masked_load(__mem, typename
_UV::mask_type(__k)));
+ }
+#endif
+ if (__k._M_none_of()) [[unlikely]]
+ return basic_vec();
+ else if constexpr (_S_is_scalar)
+ return basic_vec(static_cast<value_type>(*__mem));
+ else
+ if constexpr (__converts_trivially<_Up, value_type>)
+ {
+ using _UV = rebind_t<_Up, basic_vec>;
+ alignas(_UV) _Up __tmp[sizeof(_UV) / sizeof(_Up)] = {};
+ __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]]
(int __i) {
+ return basic_vec(__builtin_bit_cast(_UV, __tmp));
+ }
+ inline void
+ _M_store(_Up* __mem) const
+ if constexpr (__converts_trivially<value_type, _Up>)
+ __builtin_memcpy(__mem, &_M_data, sizeof(_Up) * _S_size);
+ rebind_t<_Up, basic_vec>(*this)._M_store(__mem);
+ }
+ * Implementation of @ref partial_store.
+ *
+ * is not inlined.
+ *
+ _S_partial_store(const basic_vec __v, _Up* __mem, size_t __n)
+ {
+ const auto __k = __n < _S_size ?
mask_type::_S_partial_mask_of_n(int(__n))
+ : mask_type(true);
+ return _S_masked_store(__v, __mem, __k);
+#endif
+ if (__n >= _S_size) [[unlikely]]
+ else if constexpr (__converts_trivially<value_type, _Up>)
+ {
+ byte* __dst = reinterpret_cast<byte*>(__mem);
+ const byte* __src = reinterpret_cast<const
byte*>(&__v._M_data);
+ __memcpy_chunks<sizeof(_Up), sizeof(_M_data)>(__dst, __src,
__n);
+ using _UV = rebind_t<_Up, basic_vec>;
+ _UV::_S_partial_store(_UV(__v), __mem, __n);
+ _S_masked_store(const basic_vec __v, _Up* __mem, const mask_type __k)
+ {
+ if constexpr (__converts_trivially<value_type, _Up>)
+ __x86_masked_store(__v._M_data, __mem, __k._M_data);
+ using _UV = rebind_t<_Up, basic_vec>;
+ _UV::_S_masked_store(_UV(__v), __mem, typename
_UV::mask_type(__k));
+ }
+#endif
+ if (__k._M_none_of()) [[unlikely]]
+ if constexpr (__converts_trivially<value_type, _Up>)
+ {
+ const rebind_t<_Up, basic_vec> __cvted(__v);
+ __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]]
(int __i) {
+ basic_vec() = default;
+
+ // [simd.overview] impl-def conversions
---------------------------------
+ constexpr
+ basic_vec(_DataType __x) requires (!_S_is_scalar)
+ : _M_data(__x)
+ requires (!_S_is_scalar)
+ { return _M_data; }
+ requires same_as<__x86_intel_intrin_value_type<value_type>,
__vec_value_type<_IV>>
+ && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
+ && !is_same_v<_IV, _DataType>)
+ constexpr
+ basic_vec(_IV __x)
+ : _M_data(reinterpret_cast<_DataType>(__x))
+ {}
+ requires same_as<__x86_intel_intrin_value_type<value_type>,
__vec_value_type<_IV>>
+ && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
+ && !is_same_v<_IV, _DataType>)
+ constexpr
+ { return reinterpret_cast<_IV>(_M_data); }
+#endif
+
+ // [simd.ctor] broadcast constructor
------------------------------------
+ template <__simd_vec_bcast<value_type> _Up>
+ [[__gnu__::__always_inline__]]
+ constexpr explicit(!__broadcast_constructible<_Up, value_type>)
+ basic_vec(_Up&& __x) noexcept
+ : _M_data(_DataType() == _DataType() ? static_cast<value_type>(__x)
: value_type())
+ {}
+#ifdef _GLIBCXX_SIMD_CONSTEVAL_BROADCAST
+ template <__simd_vec_bcast_consteval<value_type> _Up>
+ consteval
+ basic_vec(_Up&& __x)
+ : _M_data(_DataType() == _DataType()
+ ? __value_preserving_cast<value_type>(__x) :
value_type())
+ {
+ // TODO: I would prefer the convertible_to check to be a constraint
on this constructor.
+ // However, that would change the order in overload resolution,
which
+ static_assert(convertible_to<_Up, value_type>);
+ static_assert(is_arithmetic_v<remove_cvref_t<_Up>>);
+ }
+#endif
+
+ // [simd.ctor] conversion constructor
-----------------------------------
+ template <typename _Up, typename _UAbi>
+ // FIXME(file LWG issue): missing constraint
`constructible_from<value_type, _Up>`
+ [[__gnu__::__always_inline__]]
+ explicit(!__value_preserving_convertible_to<_Up, value_type>
+ || __higher_rank_than<_Up, value_type>)
+ basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
+ : _M_data([&] [[__gnu__::__always_inline__]]() {
+ return static_cast<value_type>(__x[0]);
+ else
+ // [simd.ctor] generator constructor
------------------------------------
+ template <__simd_generator_invokable<value_type, _S_size> _Fp>
+ [[__gnu__::__always_inline__]]
+ basic_vec(_Fp&& __gen)
+ : _M_data([&] [[__gnu__::__always_inline__]] {
+ return
_DataType{static_cast<value_type>(__gen(__simd_size_constant<__is>))...};
+ }())
+ // [simd.ctor] load constructor
-----------------------------------------
+ template <typename _Up>
+ basic_vec(_LoadCtorTag, const _Up* __ptr)
+ : _M_data()
+ _M_data = static_cast<value_type>(__ptr[0]);
+ else if (__builtin_is_constant_evaluated())
+ _M_data = _DataType{static_cast<value_type>(__ptr[__is])...};
+ }
+ else if constexpr (__converts_trivially<_Up, value_type>)
+ {
+ // This assumes std::floatN_t to be bitwise equal to float/
double
+ __builtin_memcpy(&_M_data, __ptr, sizeof(value_type) *
_S_size);
+ }
+ // FIXME(file LWG issue):
+ // 1. missing constraint on `constructible_from<value_type,
range_value_t<_Rg>>`
+ // 2. Mandates should be Constraints to fix answers of convertible_to
and constructible_from
+ //
+ // Consider `convertible_to<array<complex<float>, 4>, vec<float,
4>>`. It should say false
+ // but currently would be true.
+ // Also, with `flag_convert` the current Mandates doesn't catch the
complex<float> -> float
+ // issue and fails with horrible diagnostics somewhere in the
instantiation.
+ basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
+ : basic_vec(_LoadCtorTag(), __flags.template
_S_adjust_pointer<basic_vec>(
+ std::ranges::data(__range)))
+
static_assert(__loadstore_convertible_to<std::ranges::range_value_t<_Rg>,
value_type,
+ _Flags...>);
+ // [simd.subscr]
--------------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr value_type
+ operator[](__simd_size_type __i) const
+ // increment and decrement are implemented in terms of operator+=/-=
which avoids UB on
+ // padding elements while not breaking UBsan
+ [[__gnu__::__always_inline__]]
+ constexpr basic_vec&
+ operator++() noexcept requires requires(value_type __a) { ++__a; }
+ { return *this += value_type(1); }
+
+ constexpr basic_vec
+ operator++(int) noexcept requires requires(value_type __a) { __a++; }
+ {
+ basic_vec __r = *this;
+ *this += value_type(1);
+ return __r;
+ constexpr basic_vec&
+ operator--() noexcept requires requires(value_type __a) { --__a; }
+ { return *this -= value_type(1); }
+
+ constexpr basic_vec
+ operator--(int) noexcept requires requires(value_type __a) { __a--; }
+ {
+ basic_vec __r = *this;
+ *this -= value_type(1);
+ return __r;
+ constexpr mask_type
+ operator!() const noexcept requires requires(value_type __a) { !__a; }
+ { return mask_type::_S_init(!_M_data); }
+
+ constexpr basic_vec
+ operator+() const noexcept requires requires(value_type __a) { +__a; }
+ { return *this; }
+ constexpr basic_vec
+ operator-() const noexcept requires requires(value_type __a) { -__a; }
+ { return _S_init(-_M_data); }
+ constexpr basic_vec
+ operator~() const noexcept requires requires(value_type __a) { ~__a; }
+ { return _S_init(~_M_data); }
+ // [simd.cassign] binary operators
+#define _GLIBCXX_SIMD_DEFINE_OP(sym) \
+ friend constexpr basic_vec& \
+ operator sym##=(basic_vec& __x, const basic_vec& __y) noexcept \
+ requires requires(value_type __a) { __a sym __a; } \
+ { \
+#undef _GLIBCXX_SIMD_DEFINE_OP
+
+ friend constexpr basic_vec&
+ operator+=(basic_vec& __x, const basic_vec& __y) noexcept
+ requires requires(value_type __a) { __a + __a; }
+ {
+ if constexpr (_S_is_partial && is_integral_v<value_type> &&
is_signed_v<value_type>)
+ { // avoid spurious UB on signed integer overflow of the padding
element(s). But don't
+ // remove UB of the active elements (so that UBsan can still do
its job).
+ using _UV = typename _Ap::template
_DataType<make_unsigned_t<value_type>>;
+ const _DataType __result
+ =
reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
+ +
reinterpret_cast<_UV>(__y._M_data));
+ const auto __positive = __y > value_type();
+ const auto __overflow = __positive != (__result > __x);
+ __builtin_unreachable(); // trigger UBsan
+ __x._M_data = __result;
+ else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
+ __x = basic_vec(rebind_t<float, basic_vec>(__x) + __y);
+ else
+ friend constexpr basic_vec&
+ operator-=(basic_vec& __x, const basic_vec& __y) noexcept
+ requires requires(value_type __a) { __a - __a; }
+ {
+ if constexpr (_S_is_partial && is_integral_v<value_type> &&
is_signed_v<value_type>)
+ { // avoid spurious UB on signed integer overflow of the padding
element(s). But don't
+ // remove UB of the active elements (so that UBsan can still do
its job).
+ using _UV = typename _Ap::template
_DataType<make_unsigned_t<value_type>>;
+ const _DataType __result
+ =
reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
+ -
reinterpret_cast<_UV>(__y._M_data));
+ const auto __positive = __y > value_type();
+ const auto __overflow = __positive != (__result < __x);
+ __builtin_unreachable(); // trigger UBsan
+ __x._M_data = __result;
+ else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
+ __x = basic_vec(rebind_t<float, basic_vec>(__x) - __y);
+ else
+ friend constexpr basic_vec&
+ operator*=(basic_vec& __x, const basic_vec& __y) noexcept
+ requires requires(value_type __a) { __a * __a; }
+ {
+ if constexpr (_S_is_partial && is_integral_v<value_type> &&
is_signed_v<value_type>)
+ { // avoid spurious UB on signed integer overflow of the padding
element(s). But don't
+ // remove UB of the active elements (so that UBsan can still do
its job).
+ for (int __i = 0; __i < _S_size; ++__i)
+ if (__builtin_mul_overflow_p(__x._M_data[__i],
__y._M_data[__i], value_type()))
+ __builtin_unreachable();
+ using _UV = typename _Ap::template
_DataType<make_unsigned_t<value_type>>;
+ __x._M_data =
reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
+ *
reinterpret_cast<_UV>(__y._M_data));
+ }
+ // 'uint16 * uint16' promotes to int and can therefore lead to UB.
The standard does not
+ // require to avoid the undefined behavior. It's unnecessary and easy
to avoid. It's also
+ // unexpected because there's no UB on the vector types (which don't
promote).
+ else if constexpr (_S_is_scalar && is_unsigned_v<value_type>
+ && is_signed_v<decltype(value_type() *
value_type())>)
+ __x._M_data = unsigned(__x._M_data) * unsigned(__y._M_data);
+ else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
+ __x = basic_vec(rebind_t<float, basic_vec>(__x) * __y);
+
+ friend constexpr basic_vec&
+ operator/=(basic_vec& __x, const basic_vec& __y) noexcept
+ requires requires(value_type __a) { __a / __a; }
+ {
+ const basic_vec __result([&](int __i) -> value_type { return
__x[__i] / __y[__i]; });
+ if (__is_const_known(__result))
+#ifdef __SSE2__
+ // x86 doesn't have integral SIMD division instructions
+ // While division is faster, the required conversions are still a
problem:
+ // see PR121274, PR121284, and PR121296 for missed optimizations
wrt. conversions
+ //
+ if constexpr (is_integral_v<value_type> && _S_size > 2
+ && __value_preserving_convertible_to<value_type,
double>)
+ {
+ // With AVX512FP16 use vdivph for 8-bit integers
+ if constexpr (_Traits._M_have_avx512fp16()
+ &&
__value_preserving_convertible_to<value_type, _Float16>)
+ return __x = basic_vec(rebind_t<_Float16, basic_vec>(__x)
/ __y);
+ else if constexpr
(__value_preserving_convertible_to<value_type, float>)
+ return __x = basic_vec(rebind_t<float, basic_vec>(__x) /
__y);
+ else
+ return __x = basic_vec(rebind_t<double, basic_vec>(__x) /
__y);
+ }
+#endif
+ if constexpr (_Traits._M_eval_as_f32<value_type>())
+ return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
+
+ basic_vec __y1 = __y;
+ if constexpr (_S_is_partial)
+ if constexpr (is_integral_v<value_type>)
+ {
+ __y1 =
__select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
+ __y, basic_vec(value_type(1)));
+ }
+ friend constexpr basic_vec&
+ operator%=(basic_vec& __x, const basic_vec& __y) noexcept
+ requires requires(value_type __a) { __a % __a; }
+ {
+ static_assert(is_integral_v<value_type>);
+ if constexpr (_S_is_partial)
+ const basic_vec __y1 =
__select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
+ __y,
basic_vec(value_type(1)));
+ if (__is_const_known(__y1))
+ friend constexpr basic_vec&
+ operator<<=(basic_vec& __x, const basic_vec& __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a) { __a << __a; }
+ {
+ __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y
>= value_type()),
+ "negative shift is undefined behavior");
+ __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
+ "too large shift invokes undefined
behavior");
+ friend constexpr basic_vec&
+ operator>>=(basic_vec& __x, const basic_vec& __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a) { __a >> __a; }
+ {
+ __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y
>= value_type()),
+ "negative shift is undefined behavior");
+ __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
+ "too large shift invokes undefined
behavior");
+ friend constexpr basic_vec&
+ operator<<=(basic_vec& __x, __simd_size_type __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
+ {
+ __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
+ "too large shift invokes undefined
behavior");
+ friend constexpr basic_vec&
+ operator>>=(basic_vec& __x, __simd_size_type __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
+ {
+ __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
+ "too large shift invokes undefined
behavior");
+ constexpr mask_type
+ _M_bitmask_cmp(_DataType __y) const
+ static_assert(_S_use_bitmask);
+ if (__is_const_known(_M_data, __y))
+ (value_type __a, value_type __b) {
+ if constexpr (_Cmp == _X86Cmp::_Eq)
+ return std::isunordered(__a, __b);
+ else if constexpr (_Cmp == _X86Cmp::_Neq)
+ static_assert(false);
+ };
+ return mask_type::_S_init(((__cmp_op(__vec_get(_M_data, __is),
__vec_get(__y, __is))
+ ? (1ULL << __is) : 0) | ...));
+ return mask_type::_S_init(__x86_bitmask_cmp<_Cmp>(_M_data, __y));
+ }
+#endif
+
+ friend constexpr mask_type
+ operator==(const basic_vec& __x, const basic_vec& __y) noexcept
+ {
+#endif
+ return mask_type::_S_init(__x._M_data == __y._M_data);
+ }
+ friend constexpr mask_type
+ operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
+ {
+#endif
+ return mask_type::_S_init(__x._M_data != __y._M_data);
+ }
+ friend constexpr mask_type
+ operator<(const basic_vec& __x, const basic_vec& __y) noexcept
+ {
+#endif
+ return mask_type::_S_init(__x._M_data < __y._M_data);
+ }
+ friend constexpr mask_type
+ operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
+ {
+#endif
+ return mask_type::_S_init(__x._M_data <= __y._M_data);
+ }
+ friend constexpr mask_type
+ operator>(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return __y < __x; }
+ friend constexpr mask_type
+ operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return __y <= __x; }
+ // [simd.cond]
---------------------------------------------------------
+ template <_TargetTraits _Traits = {}>
+ friend constexpr basic_vec
+ __select_impl(const mask_type& __k, const basic_vec& __t, const
basic_vec& __f) noexcept
+ {
+ return basic_vec([&](int __i) { return __k[__i] ? __t[__i] :
__f[__i]; });
+ else
+ static_assert(false, "TODO");
+#endif
+ }
+ if (is_integral_v<value_type> && sizeof(_M_data) >= 8
+ && _VO::_S_is_const_known_equal_to(__t._M_data, 1))
+ return basic_vec((-__k)._M_abs());
+ /* else if (is_integral_v<value_type> &&
sizeof(_M_data) >= 8
+ && _VO::_S_is_const_known_equal_to(__t._M_data,
value_type(-1)))
+ return basic_vec(-__k);*/
+ else
+ return
__vec_and(reinterpret_cast<_DataType>(__k._M_data), __t._M_data);
+ }
+ if (is_integral_v<value_type> && sizeof(_M_data) >= 8
+ && _VO::_S_is_const_known_equal_to(__f._M_data, 1))
+ return value_type(1) + basic_vec(-__k);
+ else
+ return
__vec_andnot(reinterpret_cast<_DataType>(__k._M_data), __f._M_data);
+ }
+ // This pattern, is recognized to match the x86 blend
instructions, which only consider
+ // the sign bit of the mask register. Also, without SSE4,
if the compiler knows that __k
+ // is a vector-mask, then the '< 0' is elided.
+#endif
+ return __k._M_data ? __t._M_data : __f._M_data;
+ class basic_vec<_Tp, _Ap>
+ : _BinaryOps<_Tp, _Ap>
+ friend class basic_vec;
+
+ static_assert(_DataType0::abi_type::_S_nreg +
_DataType1::abi_type::_S_nreg == _Ap::_S_nreg);
+
+ static constexpr bool _S_is_scalar = _DataType0::_S_is_scalar;
+
+ public:
+ using value_type = _Tp;
+
+ using abi_type = _Ap;
+
+ using mask_type = basic_mask<sizeof(_Tp), abi_type>;
+
+ using iterator = __iterator<basic_vec>;
+
+ using const_iterator = __iterator<const basic_vec>;
+
+ begin() noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ begin() const noexcept
+ { return {*this, 0}; }
+ constexpr const_iterator
+ cbegin() const noexcept
+ { return {*this, 0}; }
+ constexpr default_sentinel_t
+ end() const noexcept
+ { return {}; }
+ constexpr default_sentinel_t
+ cend() const noexcept
+ { return {}; }
+ static constexpr basic_vec
+ _S_init(const _DataType0& __x, const _DataType1& __y)
+ basic_vec __r;
+ __r._M_data0 = __x;
+ friend constexpr bool
+ __is_const_known(const basic_vec& __x)
+ { return __is_const_known(__x._M_data0) &&
__is_const_known(__x._M_data1); }
+ constexpr auto
+ _M_concat_data() const
+ static constexpr basic_vec
+ _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&&
__idxmap)
+ {
+ constexpr auto
+ _M_chunk() const noexcept
+ {
+ return tuple<_Vp, resize_t<__rem, _Vp>> {_M_data0, _M_data1};
+ }
+ static_assert(!_Vp::_S_is_partial);
+ return __builtin_bit_cast(_Rp, *this);
+ using _Rest = resize_t<__rem, _Vp>;
+ // can't bit-cast because the member order of tuple is reversed
+ _M_assign_from(auto _Offset, const basic_vec<value_type, _A0>& __x0,
+ const basic_vec<value_type, _As>&... __xs)
+ {
+ static constexpr basic_vec
+ _S_concat(const basic_vec<value_type, _A0>& __x0) noexcept
+ { return basic_vec(__x0); }
+
+ static constexpr basic_vec
+ _S_concat(const basic_vec<value_type, _A0>& __x0,
+ const basic_vec<value_type, _As>&... __xs) noexcept
+ {
+ basic_vec __r;
+ __r._M_data0.template _M_assign_from(integral_constant<int,
0>(), __x0, __xs...);
+ basic_vec __r = {};
+ byte* __dst = reinterpret_cast<byte*>(&__r);
+ constexpr size_t __nbytes0 = sizeof(value_type) * _A0::_S_size;
+ __builtin_memcpy(__dst, &__x0, _A0::_S_nreg == 1 ? sizeof(__x0)
: __nbytes0);
+ __dst += sizeof(value_type) * _A0::_S_size;
+ template for (const auto& __x : {__xs...})
+ constexpr size_t __nbytes = sizeof(value_type) *
__x.size.value;
+ __builtin_memcpy(__dst, &__x, __nbytes);
+ constexpr auto
+ _M_reduce_1(auto __binary_op) const
+ static_assert(_N0 == _N1);
+ return __binary_op(_M_data0, _M_data1);
+ constexpr value_type
+ _M_reduce_tail(const auto& __rest, auto __binary_op) const
+ auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
+ return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
+ constexpr value_type
+ _M_reduce(_BinaryOp __binary_op) const
+ if constexpr (_Traits.template _M_eval_as_f32<value_type>()
+ && (is_same_v<_BinaryOp, plus<>>
+ || is_same_v<_BinaryOp, multiplies<>>))
+ return value_type(rebind_t<float,
basic_vec>(*this)._M_reduce(__binary_op));
+#ifdef __SSE2__
+ else if constexpr (is_integral_v<value_type> && sizeof(value_type)
== 1
+ && is_same_v<decltype(__binary_op),
multiplies<>>)
+ {
+ // convert to unsigned short because of missing 8-bit mul
instruction
+ // we don't need to preserve the order of elements
+ using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short,
basic_vec>>;
+ auto __a = __builtin_bit_cast(_V16, *this);
+ using _V16 = rebind_t<unsigned short, basic_vec>;
+ return _V16(*this)._M_reduce(__binary_op);
+#endif
+ }
+#endif
+ else if constexpr (_N0 == _N1)
+ && requires {
+ __default_identity_element<value_type,
decltype(__binary_op)>();
+ })
+ = __default_identity_element<value_type,
decltype(__binary_op)>();
+ constexpr auto __k =
_DataType0::mask_type::_S_partial_mask_of_n(_N1);
+ __v1 = __select_impl(__k, __v1, __id);
+#endif
+ else
+ constexpr mask_type
+ _M_isnan() const requires is_floating_point_v<value_type>
+ { return mask_type::_S_init(_M_data0._M_isnan(), _M_data1._M_isnan());
}
+
+ constexpr mask_type
+ _M_isinf() const requires is_floating_point_v<value_type>
+ { return mask_type::_S_init(_M_data0._M_isinf(), _M_data1._M_isinf());
}
+
+ constexpr mask_type
+ _M_isunordered(basic_vec __y) const requires
is_floating_point_v<value_type>
+ {
+ return mask_type::_S_init(_M_data0._M_isunordered(__y._M_data0),
+ _M_data1._M_isunordered(__y._M_data1));
+ constexpr basic_vec
+ _M_abs() const
+ constexpr basic_vec
+ _M_fabs() const
+ static inline basic_vec
+ _S_partial_load(const _Up* __mem, size_t __n)
+ {
+ static inline basic_vec
+ _S_masked_load(const _Up* __mem, mask_type __k)
+ {
+ inline void
+ _M_store(_Up* __mem) const
+ _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
+ {
+ _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type&
__k)
+ {
+ basic_vec() = default;
+
+ // [simd.overview] impl-def conversions
---------------------------------
+ using _NativeVecType
+ return __vec_builtin_type<value_type,
__bit_ceil(unsigned(_S_size))>();
+ }());
+ basic_vec(const _NativeVecType& __x) requires (!_S_is_scalar)
+ : _M_data0(_VecOps<__vec_builtin_type<value_type,
_N0>>::_S_extract(__x)),
+ _M_data1(_VecOps<__vec_builtin_type<value_type,
__bit_ceil(unsigned(_N1))>>
+ ::_S_extract(__x, integral_constant<int, _N0>()))
+ // [simd.ctor] broadcast constructor
------------------------------------
+ template <__simd_vec_bcast<value_type> _Up>
+ [[__gnu__::__always_inline__]]
+ constexpr explicit(!__broadcast_constructible<_Up, value_type>)
+ basic_vec(_Up&& __x) noexcept
+ : _M_data0(static_cast<value_type>(__x)),
_M_data1(static_cast<value_type>(__x))
+ {}
+#ifdef _GLIBCXX_SIMD_CONSTEVAL_BROADCAST
+ template <__simd_vec_bcast_consteval<value_type> _Up>
+ consteval
+ basic_vec(_Up&& __x)
+ : _M_data0(__value_preserving_cast<value_type>(__x)),
+ _M_data1(__value_preserving_cast<value_type>(__x))
+ {
+ static_assert(convertible_to<_Up, value_type>);
+ static_assert(is_arithmetic_v<remove_cvref_t<_Up>>);
+ }
+#endif
+
+ // [simd.ctor] conversion constructor
-----------------------------------
+ template <typename _Up, typename _UAbi>
+ // FIXME(file LWG issue): missing constraint
`constructible_from<value_type, _Up>`
+ [[__gnu__::__always_inline__]]
+ explicit(!__value_preserving_convertible_to<_Up, value_type>
+ || __higher_rank_than<_Up, value_type>)
+ basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
+ : _M_data0(get<0>(chunk<_N0>(__x))),
+ // [simd.ctor] generator constructor
------------------------------------
+ template <__simd_generator_invokable<value_type, _S_size> _Fp>
+ [[__gnu__::__always_inline__]]
+ basic_vec(_Fp&& __gen)
+ : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]]
(auto __i) {
+ // [simd.ctor] load constructor
-----------------------------------------
+ template <typename _Up>
+ basic_vec(_LoadCtorTag, const _Up* __ptr)
+ : _M_data0(_LoadCtorTag(), __ptr),
+ // FIXME: see load ctor(s) above
+ constexpr
+ basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
+ : basic_vec(_LoadCtorTag(),
+ __flags.template
_S_adjust_pointer<basic_vec>(std::ranges::data(__range)))
+ {
+
static_assert(__loadstore_convertible_to<std::ranges::range_value_t<_Rg>,
value_type,
+ _Flags...>);
+ // [simd.subscr]
--------------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ constexpr value_type
+ operator[](__simd_size_type __i) const
+ { alignas(basic_vec) const value_type _M_values[_S_size]; };
+ return __builtin_bit_cast(_Tmp, *this)._M_values[__i];
+ constexpr basic_vec&
+ operator++() noexcept requires requires(value_type __a) { ++__a; }
+ {
+ constexpr basic_vec
+ operator++(int) noexcept requires requires(value_type __a) { __a++; }
+ {
+ basic_vec __r = *this;
+ ++_M_data0;
+ constexpr basic_vec&
+ operator--() noexcept requires requires(value_type __a) { --__a; }
+ {
+ constexpr basic_vec
+ operator--(int) noexcept requires requires(value_type __a) { __a--; }
+ {
+ basic_vec __r = *this;
+ --_M_data0;
+ constexpr mask_type
+ operator!() const noexcept requires requires(value_type __a) { !__a; }
+ { return mask_type::_S_init(!_M_data0, !_M_data1); }
+
+ constexpr basic_vec
+ operator+() const noexcept requires requires(value_type __a) { +__a; }
+ { return *this; }
+ constexpr basic_vec
+ operator-() const noexcept requires requires(value_type __a) { -__a; }
+ { return _S_init(-_M_data0, -_M_data1); }
+ constexpr basic_vec
+ operator~() const noexcept requires requires(value_type __a) { ~__a; }
+ { return _S_init(~_M_data0, ~_M_data1); }
+ // [simd.cassign]
-------------------------------------------------------
+#define _GLIBCXX_SIMD_DEFINE_OP(sym) \
+ friend constexpr basic_vec& \
+ operator sym##=(basic_vec& __x, const basic_vec& __y)
_GLIBCXX_SIMD_NOEXCEPT \
+ { \
+#undef _GLIBCXX_SIMD_DEFINE_OP
+
+ friend constexpr basic_vec&
+ operator<<=(basic_vec& __x, __simd_size_type __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
+ {
+ friend constexpr basic_vec&
+ operator>>=(basic_vec& __x, __simd_size_type __y)
_GLIBCXX_SIMD_NOEXCEPT
+ requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
+ {
+ friend constexpr mask_type
+ operator==(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 == __y._M_data0, __x._M_data1
== __y._M_data1); }
+
+ friend constexpr mask_type
+ operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 != __y._M_data0, __x._M_data1
!= __y._M_data1); }
+
+ friend constexpr mask_type
+ operator<(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 < __y._M_data0, __x._M_data1 <
__y._M_data1); }
+
+ friend constexpr mask_type
+ operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 <= __y._M_data0, __x._M_data1
<= __y._M_data1); }
+
+ friend constexpr mask_type
+ operator>(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 > __y._M_data0, __x._M_data1 >
__y._M_data1); }
+
+ friend constexpr mask_type
+ operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
+ { return mask_type::_S_init(__x._M_data0 >= __y._M_data0, __x._M_data1
>= __y._M_data1); }
+
+ // [simd.cond]
---------------------------------------------------------
+ [[__gnu__::__always_inline__]]
+ friend constexpr basic_vec
+ __select_impl(const mask_type& __k, const basic_vec& __t, const
basic_vec& __f) noexcept
+ {
+ basic_vec(_Rg&& __r, _Ts...)
+ -> basic_vec<ranges::range_value_t<_Rg>,
+ __deduce_abi_t<ranges::range_value_t<_Rg>,
+#if 0 // PR117849
+ ranges::size(__r)>>;
+#endif
+
+ // FIXME: file new LWG issue about this missing deduction guide
+ template <size_t _Bytes, typename _Ap>
+ basic_vec(basic_mask<_Bytes, _Ap>)
+ -> basic_vec<__integer_from<_Bytes>,
+ decltype(__abi_rebind<__integer_from<_Bytes>,
basic_mask<_Bytes, _Ap>::size.value,
+ _Ap>())>;
+#endif
+
+ // [P3319R5]
----------------------------------------------------------------
+ template <__vectorizable _Tp>
+ requires is_arithmetic_v<_Tp>
+ inline constexpr _Tp
+ inline constexpr basic_vec<_Tp, _Ap>
+ __iota<basic_vec<_Tp, _Ap>> = basic_vec<_Tp, _Ap>([](_Tp __i) -> _Tp {
+ static_assert(__simd_size_v<_Tp, _Ap> - 1 <=
numeric_limits<_Tp>::max(),
+ "iota object would overflow");
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_VEC_H
diff --git a/libstdc++-v3/include/bits/simd_x86.h b/libstdc++-v3/include/bits/
simd_x86.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/simd_x86.h
@@ -0,0 +1,1338 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "vec_ops.h"
+
+#endif
+
+#pragma GCC push_options
+// ensure GCC knows about the __builtin_ia32_* calls
+#pragma GCC target("avx2,bmi,bmi2,avx512vl,avx512bw,avx512dq,avx10.2")
+#pragma GCC pop_options
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ static constexpr size_t __x86_max_general_register_size
+#ifdef __x86_64__
+ = 8;
+#endif
+
+ * 2. The return type internally is 'int', but that fails on conversion to
uint64 if the MSB of a
+ * YMM 1/2-Byte vector-mask is set (sign extension). Therefore these helper
functions return
+ * unsigned instead.
+ * 3. ZMM inputs are not supported.
+ */
+#endif
+ return _Bitmask<8>(__x86_movmsk(__vec_zero_pad_to_16(__x)));
+ static_assert(__width_of<_TV> > 1);
+ if constexpr (sizeof(__x) == 32)
+#endif
+ return __x86_movmsk(__vec_zero_pad_to_16(__x));
+#endif
+ return __x86_movmsk(__vec_zero_pad_to_16(__x));
+#endif
+ return ((__bits >> 7) & 1) | ((__bits & 0x8000) >> 14);
+ static_assert(false);
+ }
+ inline bool
+ __x86_vec_is_zero(_TV __a)
+ static_assert(is_integral_v<_Tp>);
+ if constexpr (sizeof(_TV) <= __x86_max_general_register_size)
+ static_assert(false);
+ }
+ static_assert(false);
+ }
+ static_assert(sizeof(_TV) == 16 || sizeof(_TV) == 32);
+ static_assert(_Traits._M_have_sse4_1());
+ if constexpr (sizeof(_TV) == 32)
+ static_assert(sizeof(_TV) == 16 || sizeof(_TV) == 32);
+ static_assert(_Traits._M_have_sse4_1());
+ if constexpr (sizeof(_TV) == 32)
+ inline bool
+ __x86_vecmask_all(_TV __k)
+ static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
+ constexpr int __width = __width_of<_TV>;
+ static_assert(sizeof(__k) <= 32);
+ if constexpr (_Np == __width)
+ inline bool
+ __x86_vecmask_any(_TV __k)
+ static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
+ constexpr int __width = __width_of<_TV>;
+ static_assert(sizeof(__k) <= 32);
+ if constexpr (_Np == __width)
+ inline bool
+ __x86_vecmask_none(_TV __k)
+ static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
+ constexpr int __width = __width_of<_TV>;
+ static_assert(sizeof(__k) <= 32);
+ if constexpr (_Np == __width)
+ enum class _X86Cmp
+ {
+ requires is_floating_point_v<__vec_value_type<_TV>>
+ [[__gnu__::__always_inline__]]
+ inline auto
+ __x86_bitmask_cmp(_TV __x, _TV __y)
+ else if constexpr (is_same_v<_Tp, _Float16>)
+ {
+ // without AVX512_FP16, float16_t size needs to match float32_t
size
+ // (cf. __native_abi())
+ static_assert(sizeof(_TV) <= 32);
+ return __x86_bitmask_cmp<_Cmp>(__vec_cast<float>(__x),
__vec_cast<float>(__y));
+ static_assert(false);
+ }
+ if constexpr (is_integral_v<_Tp> || sizeof(_Tp) <= 2)
+ return __x86_intrin_int<_Tp>();
+ if constexpr (is_integral_v<_Tp>)
+ return 0ll;
+ return 0.f16;
+ }());
+ requires is_integral_v<__vec_value_type<_TV>>
+ [[__gnu__::__always_inline__]]
+ inline auto
+ __x86_bitmask_cmp(_TV __x, _TV __y)
+ else if constexpr (is_signed_v<_Tp>)
+ {
+ static_assert(false);
+ }
+ static_assert(false);
+ }
+ inline auto
+ __x86_bitmask_isinf(_TV __x)
+ static_assert(_Traits._M_have_avx512dq());
+ using _Tp = __vec_value_type<_TV>;
+ static_assert(is_floating_point_v<_Tp>);
+ if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
+ static_assert(false);
+ }
+ static_assert((sizeof(__bits) * __CHAR_BIT__ == __width_of<_KV>)
+ || (sizeof(__bits) == 1 && __CHAR_BIT__ >
__width_of<_KV>));
+ static_assert(false);
+ }
+ template <unsigned_integral _Kp, __vec_builtin _TV, _ArchTraits _Traits =
{}>
+ requires is_integral_v<__vec_value_type<_TV>>
+ [[__gnu__::__always_inline__]]
+ if constexpr (!is_same_v<_Ip, _Tp>)
+ return reinterpret_cast<_TV>(__x86_bitmask_blend(__k,
__vec_bit_cast<_Ip>(__t),
+
__vec_bit_cast<_Ip>(__f)));
+ static_assert(false);
+ }
+ template <unsigned_integral _Kp, __vec_builtin _TV, _ArchTraits _Traits =
{}>
+ requires is_floating_point_v<__vec_value_type<_TV>>
+ [[__gnu__::__always_inline__]]
+ else if constexpr (is_same_v<_Tp, _Float16>)
+ {
+ static_assert(false);
+ }
+ static_assert(_OutputBits <= 4);
+ constexpr _UInt<1> __mask = 0x55u >> ((4 - _OutputBits) * 2);
+#endif
+ __x &= __mask;
+ static_assert(_OutputBits <= 8);
+ constexpr _UInt<2> __mask = 0x5555u >> ((8 - _OutputBits) * 2);
+#endif
+ __x &= __mask;
+ static_assert(_OutputBits <= 16);
+ constexpr _UInt<4> __mask = 0x5555'5555u >> ((16 - _OutputBits) *
2);
+#endif
+ __x &= __mask;
+ static_assert(_OutputBits <= 32);
+ constexpr _UInt<8> __mask = 0x5555'5555'5555'5555ull >> ((32 -
_OutputBits) * 2);
+ return __builtin_ia32_pext_si(__x,
static_cast<unsigned>(__mask))
+ | (__builtin_ia32_pext_si(__x >> 32, __mask >> 32) <<
16);
+#endif
+ }
+#endif
+ __x &= __mask;
+ __x &= 0x0000'FFFF'0000'FFFFull;
+ __x |= __x >> 16;
+ constexpr auto
+ __duplicate_each_bit(unsigned_integral auto __x)
+ {
+ static_assert(__input_bits >= 1);
+ static_assert(sizeof(__x) * __CHAR_BIT__ >= __input_bits);
+ if constexpr (__input_bits <= 8)
+#endif
+ else if constexpr (__input_bits == 2) // 0000'00BA
+ return _UInt<1>(((__x + 0b0010u) & 0b0101u) * 3u); // 0B?A ->
0B0A -> BBAA
+ else if constexpr (__input_bits <= 4) // 0000'DCBA
+ {
+ return _UInt<1>(((__x + 0b0010'0010u) & __mask) * 3u); //
-> DDCC'BBAA
+ }
+ { // HGFE'DCBA
+ _UInt<2> __y = ((__x << 4) | __x) & 0x0F0Fu; // HGFE'0000'DCBA
+ __y |= __y << 2; // 00HG'??FE'00DC'??BA
+ return _UInt<2>((__y & __mask) * 3u); // HHGG'FFEE'DDCC'BBAA
+ }
+#endif
+ _UInt<4> __y = ((__x << 8) | __x) & 0x00FF00FFu;
+ return (3u * __builtin_ia32_pdep_si(__x,
static_cast<unsigned>(__mask))) | __hi << 32;
+#endif
+ }
+#endif
+ _UInt<8> __y = ((__x & 0xFFFF'0000ull) << 16) | (__x &
0x0000'FFFFu);
+ __y |= __y << 8;
+ constexpr auto
+ __duplicate_each_bit(const __trivial_pair<_U0, _U1>& __x)
+ static_assert(_InputBits != -1 || is_unsigned_v<_U1>);
+ constexpr int __input_bits = _InputBits == -1 ? (sizeof(_U0) +
sizeof(_U1)) * __CHAR_BIT__
+ static_assert(_Traits._M_have_fma());
+ static_assert(is_floating_point_v<_Tp>);
+
+ static_assert(false);
+ }
+ // FIXME: Work around PR121688
+ template <__vec_builtin _UV, __vec_builtin _TV>
+ constexpr bool __from_f16 = is_same_v<__vec_value_type<_TV>, _Float16>;
+ constexpr bool __to_f16 = !__from_f16;
+ if constexpr (__to_f16 && !is_same_v<__vec_value_type<_TV>, float>)
+ return __x86_cvt_f16c<_UV>(__vec_cast<float>(__v));
+ else if constexpr (__from_f16 && !is_same_v<__vec_value_type<_UV>,
float>)
+ return __vec_cast<_UV>(__x86_cvt_f16c<__vec_builtin_type<float,
__width_of<_TV>>>(__v));
+ static_assert(false);
+ }
+ return reinterpret_cast<_UV>(
+ __vec_split_lo(__vec_split_lo(__builtin_ia32_vcvtps2ph(
+ return
reinterpret_cast<_UV>(__vec_split_lo(__builtin_ia32_vcvtps2ph(__v, 4)));
+ else if constexpr (sizeof(_TV) == 32)
+ return reinterpret_cast<_UV>(__builtin_ia32_vcvtps2ph256(__v, 4));
+ else if constexpr (sizeof(_TV) == 64)
+ return reinterpret_cast<_UV>(__builtin_ia32_vcvtps2ph512_mask(
+ __v, 4, __vec_builtin_type<short,
16>(), -1));
+ static_assert(false);
+ }
+ * AVX instructions typically work per 128-bit chunk. Horizontal operations
thus produce vectors
+ * where the two 128-bit chunks in the center are swapped. This function
works as a fix-up step.
+ static_assert(sizeof(_TV) == 32);
+ using _UV = __vec_builtin_type_bytes<long long, 32>;
+ return
reinterpret_cast<_TV>(__builtin_shufflevector(reinterpret_cast<_UV>(__x),
_UV(),
+ 0, 2, 1, 3));
+ else if constexpr (sizeof(_T0) == 2 && sizeof(_T1) == 1) // -> packsswb
+ {
+ return
reinterpret_cast<_To>(__vec_split_lo(__builtin_ia32_packsswb128(__k, __k)));
+ else if constexpr (sizeof(__k) == 32)
+ return reinterpret_cast<_To>(
+ __vec_split_lo(__x86_swizzle4x64_acbd(
+ static_assert(false);
+ }
+ static_assert(false, "TODO");
+ }
+ static_assert(sizeof(_From) >= 16);
+ if constexpr (sizeof(_T0) == 2 && sizeof(_T1) == 1) // -> packsswb
+ {
+ return reinterpret_cast<_To>(__builtin_ia32_packsswb128(__k0,
__k1));
+ else if constexpr (sizeof(__k0) == 32)
+ return reinterpret_cast<_To>(__x86_swizzle4x64_acbd(
+ __builtin_ia32_packsswb256(__k0,
__k1)));
+ static_assert(false);
+ }
+ static_assert(false, "TODO");
+ }
+ * AVX512 masked (converting) loads
+ *
+ * @note AVX512VL and AVX512BW is required
+ */
+ __x86_masked_load(const _Up* __mem, unsigned_integral auto __k)
+ {
+ static_assert(_Traits._M_have_avx512vl() &&
_Traits._M_have_avx512bw());
+ using _Tp = __vec_value_type<_TV>;
+ const auto* __src = reinterpret_cast<const
__x86_intrin_type<_Up>*>(__mem);
+ const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 64> __z =
{};
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ return __builtin_ia32_loadups512_mask(__src, __z, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ return __builtin_ia32_loadupd512_mask(__src, __z, __k);
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquqi512_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 2)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquhi512_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 4)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqusi512_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 8)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqudi512_mask(__src, __z, __k));
+ else
+ static_assert(false);
+ }
+ const auto* __src = reinterpret_cast<const
__x86_intrin_type<_Up>*>(__mem);
+ const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 32> __z =
{};
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ return __builtin_ia32_loadups256_mask(__src, __z, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ return __builtin_ia32_loadupd256_mask(__src, __z, __k);
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquqi256_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 2)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquhi256_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 4)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqusi256_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 8)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqudi256_mask(__src, __z, __k));
+ else
+ static_assert(false);
+ }
+ const auto* __src = reinterpret_cast<const
__x86_intrin_type<_Up>*>(__mem);
+ const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 16> __z =
{};
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ return __builtin_ia32_loadups128_mask(__src, __z, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
+ return __builtin_ia32_loadupd128_mask(__src, __z, __k);
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquqi128_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 2)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddquhi128_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 4)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqusi128_mask(__src, __z, __k));
+ else if constexpr (sizeof(_Tp) == 8)
+ return
reinterpret_cast<_TV>(__builtin_ia32_loaddqudi128_mask(__src, __z, __k));
+ else
+ static_assert(false);
+ }
+ static_assert(false);
+ }
+ * AVX(2) masked loads (only trivial conversions)
+ */
+ static_assert(_Traits._M_have_avx() && __converts_trivially<_Up, _Tp>
&& sizeof(_Up) >= 4);
+ constexpr int __n = __width_of<_TV>;
+ const auto __vk = reinterpret_cast<_IV>(__k);
+ if constexpr (sizeof(_TV) < 16)
+ else if constexpr (_Traits._M_have_avx2() && is_integral_v<_Up>)
+ {
+ = reinterpret_cast<const
__vec_builtin_type<__x86_intrin_int<_Up>, __n>*>(__mem);
+ if constexpr (sizeof(_Up) == 4 && sizeof(_TV) == 32)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadd256(__src,
__vk));
+ else if constexpr (sizeof(_Up) == 4 && sizeof(_TV) == 16)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadd(__src,
__vk));
+ else if constexpr (sizeof(_Up) == 8 && sizeof(_TV) == 32)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadq256(__src,
__vk));
+ else if constexpr (sizeof(_Up) == 8 && sizeof(_TV) == 16)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadq(__src,
__vk));
+ else
+ static_assert(false);
+ }
+ const auto* __src = reinterpret_cast<const
__vec_builtin_type<float, __n>*>(__mem);
+ if constexpr (sizeof(_TV) == 32)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadps256(__src,
__vk));
+ else if constexpr (sizeof(_TV) == 16)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadps(__src,
__vk));
+ else
+ static_assert(false);
+ }
+ const auto* __src = reinterpret_cast<const
__vec_builtin_type<double, __n>*>(__mem);
+ if constexpr (sizeof(_TV) == 32)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadpd256(__src,
__vk));
+ else if constexpr (sizeof(_TV) == 16)
+ return reinterpret_cast<_TV>(__builtin_ia32_maskloadpd(__src,
__vk));
+ else
+ static_assert(false);
+ }
+ * AVX512 masked stores
+ *
+ * @note AVX512VL is required
+ */
+ inline void
+ __x86_masked_store(const _TV __v, _Up* __mem, unsigned_integral auto __k)
+ {
+ [[maybe_unused]] const auto __w =
__vec_bit_cast<__x86_intrin_type<_Tp>>(__v);
+ if constexpr (sizeof(_TV) == 64)
+ if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> &&
is_integral_v<_Up>)
+ {
+ auto* __dst = reinterpret_cast<
+ __vec_builtin_type<__x86_intrin_int<_Up>,
__n>*>(__mem);
+ static_assert(false);
+ }
+ auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ __builtin_ia32_storeups512_mask(__dst, __w, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) ==
8)
+ __builtin_ia32_storeupd512_mask(__dst, __w, __k);
+ static_assert(false);
+ }
+ if constexpr (is_floating_point_v<_Tp> && is_integral_v<_Up>
+ && sizeof(_Tp) > sizeof(_Up))
+ if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> &&
is_integral_v<_Up>)
+ {
+ auto* __dst = reinterpret_cast<
+ __vec_builtin_type<__x86_intrin_int<_Up>,
__n>*>(__mem);
+ static_assert(false);
+ }
+ auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ __builtin_ia32_storeups256_mask(__dst, __w, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) ==
8)
+ __builtin_ia32_storeupd256_mask(__dst, __w, __k);
+ static_assert(false);
+ }
+ if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> &&
is_integral_v<_Up>)
+ {
+ auto* __dst = reinterpret_cast<
+ __vec_builtin_type<__x86_intrin_int<_Up>,
__n>*>(__mem);
+ __builtin_ia32_pmovqd128mem_mask(reinterpret_cast<unsigned
long long*>(__mem),
+ __w, __k);
+ static_assert(false);
+ }
+ auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
+ if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
+ __builtin_ia32_storeups128_mask(__dst, __w, __k);
+ else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) ==
8)
+ __builtin_ia32_storeupd128_mask(__dst, __w, __k);
+ static_assert(false);
+ }
+ * AVX(2) masked stores
+ */
+ inline void
+ __x86_masked_store(const _TV __v, _Up* __mem, const _KV __k)
+ static_assert(sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
+ auto* __dst = reinterpret_cast<
+ __vec_builtin_type<__x86_intrin_type<_Up>,
__n>*>(__mem);
+ [[maybe_unused]] const auto __w =
__vec_bit_cast<__x86_intrin_type<_Tp>>(__v);
+ if constexpr (sizeof(_TV) < 16)
+ else if constexpr (_Traits._M_have_avx2() && is_integral_v<_Tp>)
+ {
+ static_assert(false);
+ }
+ static_assert(false);
+ }
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_SIMD_X86_H
diff --git a/libstdc++-v3/include/bits/vec_ops.h b/libstdc++-v3/include/bits/
vec_ops.h
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/bits/vec_ops.h
@@ -0,0 +1,612 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later WITH GCC-exception-3.1 */
+/* Copyright © 2025 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
+ * Matthias Kretz <m.kretz@gsi.de>
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#include "simd_details.h"
+
+// psabi warnings are bogus because the ABI of the internal types never leaks
into user code
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpsabi"
+
+ template <std::signed_integral _Tp>
+ constexpr bool
+ __signed_has_single_bit(_Tp __x)
+ { return __has_single_bit(make_unsigned_t<_Tp>(__x)); }
+
+ template <__vectorizable _Tp, size_t _Bytes>
+ requires (__has_single_bit(_Bytes))
+ = !is_arithmetic_v<_Tp> && __vectorizable<_ValueType>
+ && _Width >= 1 && sizeof(_Tp) / sizeof(_ValueType) == _Width
+ && same_as<__vec_builtin_type_bytes<_ValueType, sizeof(_Tp)>, _Tp>
+ && requires(_Tp& __v, _ValueType __x) { __v[0] = __x; };
+ = !is_class_v<_Tp> && requires(const _Tp& __x) {
+ requires __vec_builtin_of<_Tp, remove_cvref_t<decltype(__x[0])>>;
+ };
+ using __vec_value_type = remove_cvref_t<decltype(declval<const _Tp>()
[0])>;
+
+ * The width (number of value_type elements) of the given vector builtin or
arithmetic type.
+ */
+ { return reinterpret_cast<__vec_builtin_type_bytes<_Up,
sizeof(_TV)>>(__v); }
+
+ requires signed_integral<__vec_value_type<_TV>>
+ static constexpr _TV _S_vec_implicit_mask = []<int... _Is>
(integer_sequence<int, _Is...>) {
+ return _TV{ (_Is < _Np ? -1 : 0)... };
+ } (make_integer_sequence<int, __width_of<_TV>>());
+
+#ifdef _GLIBCXX_CLANG
+ if (__builtin_is_constant_evaluated())
+#endif
+ return __v[__i];
+#ifdef _GLIBCXX_CLANG
+ auto __arr = __builtin_bit_cast(array<__vec_value_type<_TV>,
__width_of<_TV>>, __v);
+#endif
+ }
+#ifdef _GLIBCXX_CLANG
+ using _RV = __vec_builtin_type<__vec_value_type<_TV>, _N0 * 2>;
+ static_assert(false);
+#elif __has_builtin(__integer_pack)
+ return __builtin_shufflevector(__a, __b, __integer_pack(2 * _N0)...);
+#endif
+ }
+ constexpr auto
+ __vec_zero_pad_to_16(_TV __x)
+ static_assert(sizeof(_TV) < 16);
+ using _Up = _UInt<sizeof(_TV)>;
+ template <size_t _Bytes, __vec_builtin _TV>
+ [[__gnu__::__always_inline__]]
+ constexpr auto
+ __vec_zero_pad_to(_TV __x)
+ static_assert(sizeof(_TV) <= _Bytes);
+ if constexpr (sizeof(_TV) == _Bytes)
+ constexpr bool
+ __is_const_known(const _Tp& __x)
+ return __is_const_known(__x.real()) && __is_const_known(__x.imag());
+ else
+ constexpr bool
+ __is_const_known(const auto&... __xs) requires(sizeof...(__xs) >= 2)
+ if consteval
+ {
+ constexpr bool
+ __is_const_known_equal_to(const auto& __x, const auto& __expect)
+#endif
+
+ * Simple wrapper around __builtin_convertvector to provide static_cast-
like syntax.
+ *
+ * Works around GCC failing to use the F16C/AVX512F cvtps2ph/cvtph2ps
instructions.
+ */
+ static_assert(__width_of<_UV> == __width_of<_TV>);
+#if _GLIBCXX_X86
+ constexpr bool __to_f16 = is_same_v<__vec_value_type<_UV>, _Float16>;
+ constexpr bool __from_f16 = is_same_v<__vec_value_type<_TV>, _Float16>;
+ constexpr bool __needs_f16c = _Traits._M_have_f16c() && !
_Traits._M_have_avx512fp16()
+ && (__to_f16 || __from_f16);
+ { // Work around PR121688
+ if constexpr (__needs_f16c)
+ if constexpr (is_floating_point_v<__vec_value_type<_TV>>
+ && is_integral_v<__vec_value_type<_UV>> && sizeof(_UV)
< sizeof(_TV)
+ && sizeof(__vec_value_type<_UV>) < sizeof(int))
+#endif
+ return __builtin_convertvector(__v, _UV);
+ static_assert(signed_integral<__vec_value_type<_UV>>);
+ static_assert(signed_integral<__vec_value_type<_TV>>);
+ // TODO: __builtin_convertvector cannot be optimal because it doesn't
consider input and
+ // output can only be 0 or -1.
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ if constexpr (is_floating_point_v<_Tp>)
+ {
+ if constexpr (is_floating_point_v<__vec_value_type<_TV>>)
+ return __builtin_bit_cast(_TV, ~__builtin_bit_cast(_UV, __a));
+ requires std::floating_point<__vec_value_type<_V>>
+ constexpr _V _S_signmask = __vec_xor(_V() + 1, _V() - 1);
+ typename = make_integer_sequence<int, _Np>>
+ struct _VecOps;
+ struct _VecOps<_TV, _Np, integer_sequence<int, _Is...>>
+ {
+ static_assert(_Np <= __width_of<_TV>);
+
+ using _HV = __half_vec_builtin_t<conditional_t<_Np >= 2, _TV,
__double_vec_builtin_t<_TV>>>;
+
+ static constexpr bool
+ _S_all_of(_TV __k) noexcept
+ { return (... && (__k[_Is] != 0)); }
+ static constexpr bool
+ _S_any_of(_TV __k) noexcept
+ { return (... || (__k[_Is] != 0)); }
+ static constexpr bool
+ _S_none_of(_TV __k) noexcept
+ { return (... && (__k[_Is] == 0)); }
+ static_assert(is_same_v<__vec_value_type<_TV>,
__vec_value_type<decltype(__x)>>);
+ return __builtin_shufflevector(__x, decltype(__x)(), (_Is +
_Offset::value)...);
+#ifdef _GLIBCXX_CLANG
+ __vec_concat(__y, __y),
+#endif
+ ((_Is & 1) == 0 ? __n + _Is / 2 :
_Is)...);
+#ifdef _GLIBCXX_CLANG
+ __vec_concat(__y, __y),
+#endif
+ ((_Is & 1) == 1 ? __n + _Is / 2 :
_Is)...);
+ // GCC recognizes this pattern as addsub
+ return __builtin_shufflevector(__x - __y, __x + __y,
+#endif
+ }
+ static constexpr bool
+ _S_is_const_known_equal_to(_TV __x, _Tp __ref)
+ // True iff all elements at even indexes are zero. This includes signed
zeros only when
+ // -fno-signed-zeros is in effect.
+ template <_OptTraits _Traits = {}>
+ static constexpr bool
+ _S_complex_real_is_const_known_zero(_TV __x)
+ // True iff all elements at odd indexes are zero. This includes signed
zeros only when
+ // -fno-signed-zeros is in effect.
+ template <_OptTraits _Traits = {}>
+ static constexpr bool
+ _S_complex_imag_is_const_known_zero(_TV __x)
+#pragma GCC diagnostic pop
+#endif // C++26
+#endif // _GLIBCXX_VEC_OPS_H
diff --git a/libstdc++-v3/include/bits/version.def b/libstdc++-v3/include/
bits/version.def
index 1bf98f74d45..4f71e4dd278 100644
--- a/libstdc++-v3/include/bits/version.def
+++ b/libstdc++-v3/include/bits/version.def
@@ -2191,6 +2191,17 @@ ftms = {
};
+ftms = {
+ name = simd;
+ no_stdname = true; // TODO: change once complete
+ v = 202506;
+ cxxmin = 26;
+ extra_cond = "__cpp_structured_bindings >= 202411L";
+ hosted = yes;
stds[97] = ">= 199711L";
stds[03] = ">= 199711L";
diff --git a/libstdc++-v3/include/bits/version.h b/libstdc++-v3/include/bits/
version.h
index 66de8b487e3..362b4283777 100644
--- a/libstdc++-v3/include/bits/version.h
+++ b/libstdc++-v3/include/bits/version.h
@@ -2455,4 +2455,13 @@
#endif /* !defined(__cpp_lib_philox_engine) */
#undef __glibcxx_want_philox_engine
+# endif
+# endif
+#endif /* !defined(__cpp_lib_simd) */
+#undef __glibcxx_want_simd
+
#undef __glibcxx_want_all
diff --git a/libstdc++-v3/include/std/simd b/libstdc++-v3/include/std/simd
new file mode 100644
--- /dev/null
+++ b/libstdc++-v3/include/std/simd
@@ -0,0 +1,48 @@
+// Copyright The GNU Toolchain Authors.
+//
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
+// <http://www.gnu.org/licenses/>.
+#ifndef _GLIBCXX_SIMD
+#define _GLIBCXX_SIMD 1
+#ifdef _GLIBCXX_SYSHDR
+#pragma GCC system_header
+#endif
+
+#ifdef __glibcxx_simd
+
+#include "bits/simd_vec.h"
+#include "bits/simd_loadstore.h"
+#include "bits/simd_mask_reductions.h"
+#include "bits/simd_reductions.h"
+#include "bits/simd_alg.h"
+
+#endif
+#endif
--
GSI Helmholtz Center for Heavy Ion Research https://gsi.de
std::simd
More information about the Libstdc++
mailing list