[RFC PATCH 3/3] RISC-V: Implement CPU yielding for busy loops with Zihintpause/Zawrs
Palmer Dabbelt
palmer@dabbelt.com
Thu Apr 18 17:17:38 GMT 2024
On Thu, 18 Apr 2024 02:46:35 PDT (-0700), christoph.muellner@vrull.eu wrote:
> The macro atomic_spin_nop can be used to implement arch-specific
> CPU yielding that is used in busy loops (e.g. in pthread_spin_lock).
> This patch introduces an ifunc-based implementation for RISC-V,
> that uses Zihintpause's PAUSE instruction for that matter (as PAUSE
> is a HINT instruction there is not dependency to Zihintpause at
> runtime). Further, we test for Zawrs via hwprobe() and if found
> we use WRS.STO instead of PAUSE.
>
> Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
> ---
> sysdeps/riscv/multiarch/cpu-relax_generic.S | 31 +++++++++++++++
> sysdeps/riscv/multiarch/cpu-relax_zawrs.S | 28 +++++++++++++
> .../unix/sysv/linux/riscv/atomic-machine.h | 3 ++
> .../unix/sysv/linux/riscv/multiarch/Makefile | 8 ++++
> .../sysv/linux/riscv/multiarch/cpu-relax.c | 39 +++++++++++++++++++
> .../linux/riscv/multiarch/ifunc-impl-list.c | 32 +++++++++++++--
> 6 files changed, 137 insertions(+), 4 deletions(-)
> create mode 100644 sysdeps/riscv/multiarch/cpu-relax_generic.S
> create mode 100644 sysdeps/riscv/multiarch/cpu-relax_zawrs.S
> create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c
>
> diff --git a/sysdeps/riscv/multiarch/cpu-relax_generic.S b/sysdeps/riscv/multiarch/cpu-relax_generic.S
> new file mode 100644
> index 0000000000..d3ccfdce84
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/cpu-relax_generic.S
> @@ -0,0 +1,31 @@
> +/* CPU strand yielding for busy loops. RISC-V version.
> + Copyright (C) 2024 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library. If not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +#include <sys/asm.h>
> +
> +.option push
> +.option arch, +zihintpause
> +ENTRY (__cpu_relax_generic)
> + /* While we can use the `pause` instruction without
> + the need of Zihintpause (because it is a HINT instruction),
> + we still have to enable Zihintpause for the assembler. */
> + pause
> + ret
> +END (__cpu_relax_generic)
> +.option pop
> diff --git a/sysdeps/riscv/multiarch/cpu-relax_zawrs.S b/sysdeps/riscv/multiarch/cpu-relax_zawrs.S
> new file mode 100644
> index 0000000000..6d27b354df
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/cpu-relax_zawrs.S
> @@ -0,0 +1,28 @@
> +/* CPU strand yielding for busy loops. RISC-V version with Zawrs.
> + Copyright (C) 2024 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library. If not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +#include <sys/asm.h>
> +
> +.option push
> +.option arch, +zawrs
> +ENTRY (__cpu_relax_zawrs)
> + wrs.sto
This has the same forward progress/eventual success violation as the
code you sent for GCC and Linux does. It doesn't really matter if the
user of the reservation is in a builtin, an asm block, or a function.
The compiler just doesn't know about those reservation rules and isn't
going to generate code that follows them.
That said, as far as I can tell the same grey area in the WRS spec that
I pointed out during one of the Linux reviews still exists. So if
there's some hardware that has a more generous set of LR->WRS rules than
the ISA-required LR->SC rules that's fine, we just need to know what the
hardware actually is so we can write down the rules and then stick to
them -- it's just a performance thing for WRS, so having weaker rules
seems entirely plausible.
I don't know of any hardware with WRS, but sorry if I missed one. Do
you have something publicly availiable that behaves this way?
> + ret
> +END (__cpu_relax_zawrs)
> +.option pop
> diff --git a/sysdeps/unix/sysv/linux/riscv/atomic-machine.h b/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
> index c1c9d949a0..02b9b7a421 100644
> --- a/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
> +++ b/sysdeps/unix/sysv/linux/riscv/atomic-machine.h
> @@ -178,4 +178,7 @@
> # error "ISAs that do not subsume the A extension are not supported"
> #endif /* !__riscv_atomic */
>
> +extern void __cpu_relax (void);
> +#define atomic_spin_nop() __cpu_relax()
> +
> #endif /* bits/atomic.h */
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> index fcef5659d4..0cdf37a38b 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> @@ -1,3 +1,11 @@
> +# nscd uses atomic_spin_nop which in turn requires cpu_relax
> +ifeq ($(subdir),nscd)
> +sysdep_routines += \
> + cpu-relax \
> + cpu-relax_generic \
> + cpu-relax_zawrs
> +endif
> +
> ifeq ($(subdir),string)
> sysdep_routines += \
> memcpy \
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c b/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c
> new file mode 100644
> index 0000000000..5aeb120e21
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/cpu-relax.c
> @@ -0,0 +1,39 @@
> +/* Multiple versions of cpu-relax.
> + All versions must be listed in ifunc-impl-list.c.
> + Copyright (C) 2024 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +# include <ifunc-init.h>
> +# include <riscv-ifunc.h>
> +# include <sys/hwprobe.h>
> +
> +void __cpu_relax (void);
> +extern __typeof (__cpu_relax) __cpu_relax_generic attribute_hidden;
> +extern __typeof (__cpu_relax) __cpu_relax_zawrs attribute_hidden;
> +
> +static inline __typeof (__cpu_relax) *
> +select_cpu_relax_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
> +{
> + unsigned long long int v;
> + if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
> + && (v & RISCV_HWPROBE_EXT_ZAWRS))
> + return __cpu_relax_zawrs;
> +
> + return __cpu_relax_generic;
> +}
> +
> +riscv_libc_ifunc (__cpu_relax, select_cpu_relax_ifunc);
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> index 9f806d7a9e..9c7a8c2e1f 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> @@ -20,24 +20,48 @@
> #include <string.h>
> #include <sys/hwprobe.h>
>
> +#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
> +
> +void __cpu_relax (void);
> +
> size_t
> __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> size_t max)
> {
> size_t i = max;
> + struct riscv_hwprobe pairs[] = {
> + { .key = RISCV_HWPROBE_KEY_IMA_EXT_0 },
> + { .key = RISCV_HWPROBE_KEY_CPUPERF_0 },
> + };
>
> bool fast_unaligned = false;
> + bool has_zawrs = false;
> +
> + if (__riscv_hwprobe (pairs, ARRAY_SIZE (pairs), 0, NULL, 0) == 0)
> + {
> + struct riscv_hwprobe *pair;
>
> - struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 };
> - if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0
> - && (pair.value & RISCV_HWPROBE_MISALIGNED_MASK)
> + /* RISCV_HWPROBE_KEY_IMA_EXT_0 */
> + pair = &pairs[0];
> + if (pair->value & RISCV_HWPROBE_EXT_ZAWRS)
> + has_zawrs = true;
> +
> + /* RISCV_HWPROBE_KEY_CPUPERF_0 */
> + pair = &pairs[1];
> + if ((pair->value & RISCV_HWPROBE_MISALIGNED_MASK)
> == RISCV_HWPROBE_MISALIGNED_FAST)
> - fast_unaligned = true;
> + fast_unaligned = true;
> + }
>
> IFUNC_IMPL (i, name, memcpy,
> IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
> __memcpy_noalignment)
> IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic))
>
> + IFUNC_IMPL (i, name, __cpu_relax,
> + IFUNC_IMPL_ADD (array, i, __cpu_relax, has_zawrs,
> + __cpu_relax_zawrs)
> + IFUNC_IMPL_ADD (array, i, __cpu_relax, 1, __cpu_relax_generic))
> +
> return 0;
> }
More information about the Libc-alpha
mailing list