Re: [PATCH] aarch64: Add tunable glibc.memset.dc_zva_threshold

[Siddhesh Poyarekar <siddhesh at gotplt dot org>]

On 26/07/19 5:28 PM, Feng Xue OS wrote:
> This patch is composed to add a tunable 'glibc.memset.dc_zva_threshold'
> to control using DC ZVA in memset or not. Only when memset size exceeds
> this threshold, DC ZVA will be used.
> 
> The background is that DC ZVA does not always outperform normal
> memory-store zeroing, especially when there are multiple processes/threads
> contending for memory/cache.
> 
> Feng
> ----
> 
>     * manual/tunables.texi: Document glibc.memset.dc_zva_threshold.
>     * sysdeps/aarch64/dl-tunables.list (glibc):
>     Add memset.dc_zva_threshold.
>     * sysdeps/aarch64/multiarch/init-arch.h [HAVE_TUNABLES]: Include
>     dl-tunables.h
>     (INIT_ZVA_THRESHOLD): New macro, with new local variable
>     zva_threshold.
>     (INIT_ARCH): Add INIT_ZVA_THRESHOLD.
>     * sysdeps/aarch64/multiarch/memset.c (__memset_dc_zva_threshold):
>     New variable.
>     (init_memset): New macro.
>     * sysdeps/aarch64/memset_base64.S (__memset_base64) [HAVE_TUNABLES]:
>     Add conditional compare over __memset_dc_zva_threshold.
>     * sysdeps/aarch64/memset_emag.S (DC_ZVA_THRESHOLD): Changed to a
>     new value.
> ---
>  ChangeLog                                 | 18 ++++++++++++++++++
>  manual/tunables.texi                      |  9 +++++++++
>  sysdeps/aarch64/dl-tunables.list          |  6 ++++++
>  sysdeps/aarch64/multiarch/init-arch.h     | 11 +++++++++++
>  sysdeps/aarch64/multiarch/memset.c        | 22 +++++++++++++++++++---
>  sysdeps/aarch64/multiarch/memset_base64.S |  7 +++++++
>  sysdeps/aarch64/multiarch/memset_emag.S   |  4 ++--
>  7 files changed, 72 insertions(+), 5 deletions(-)
> 
> diff --git a/ChangeLog b/ChangeLog
> index dbdb85d..1921e2a 100644
> --- a/ChangeLog
> +++ b/ChangeLog
> @@ -1,3 +1,21 @@
> +2019-07-26  Feng Xue  <fxue@os.amperecomputing.com>
> +
> +	* manual/tunables.texi: Document glibc.memset.dc_zva_threshold.
> +	* sysdeps/aarch64/dl-tunables.list (glibc):
> +	Add memset.dc_zva_threshold.
> +	* sysdeps/aarch64/multiarch/init-arch.h [HAVE_TUNABLES]: Include
> +	dl-tunables.h
> +	(INIT_ZVA_THRESHOLD): New macro, with new local variable
> +	zva_threshold.
> +	(INIT_ARCH): Add INIT_ZVA_THRESHOLD.
> +	* sysdeps/aarch64/multiarch/memset.c (__memset_dc_zva_threshold):
> +	New variable.
> +	(init_memset): New macro.
> +	* sysdeps/aarch64/memset_base64.S (__memset_base64) [HAVE_TUNABLES]:
> +	Add conditional compare	over __memset_dc_zva_threshold.
> +	* sysdeps/aarch64/memset_emag.S (DC_ZVA_THRESHOLD): Changed to a
> +	new value.
> +
>  2019-07-25  Florian Weimer  <fweimer@redhat.com>
>  
>  	[BZ #24677]
> diff --git a/manual/tunables.texi b/manual/tunables.texi
> index ee0fdf2..c7c13cc 100644
> --- a/manual/tunables.texi
> +++ b/manual/tunables.texi
> @@ -411,3 +411,12 @@ instead.
>  
>  This tunable is specific to i386 and x86-64.
>  @end deftp
> +
> +@deftp Tunable glibc.memset.dc_zva_threshold
> +The @code{glibc.memset.dc_zva_threshold} tunable allows the user to set
> +threshold to trigger DC ZVA in memset. When memset size is less than this
> +threshold, normal memory store instruction will be used, otherwise DC ZVA
> +instruction will be used. Value of zero means default threshold.
> +
> +This tunable is specific to aarch64.
> +@end deftp
> diff --git a/sysdeps/aarch64/dl-tunables.list b/sysdeps/aarch64/dl-tunables.list
> index 5fac533..0f5b5e1 100644
> --- a/sysdeps/aarch64/dl-tunables.list
> +++ b/sysdeps/aarch64/dl-tunables.list
> @@ -22,4 +22,10 @@ glibc {
>        type: STRING
>      }
>    }
> +  memset {
> +    dc_zva_threshold {
> +      type: SIZE_T
> +      default: 0 
> +    }
> +  }
>  }

This should be called cache.aarch64_dc_zva_threshold or
cache.aarch64_dczva_threshold.

> diff --git a/sysdeps/aarch64/multiarch/init-arch.h b/sysdeps/aarch64/multiarch/init-arch.h
> index b9020ae..93133a2 100644
> --- a/sysdeps/aarch64/multiarch/init-arch.h
> +++ b/sysdeps/aarch64/multiarch/init-arch.h
> @@ -18,7 +18,18 @@
>  
>  #include <ldsodefs.h>
>  
> +#if HAVE_TUNABLES
> +# include <elf/dl-tunables.h>
> +
> +# define INIT_ZVA_THRESHOLD()						      \
> +  uint64_t __attribute__((unused)) zva_threshold =			      \
> +    TUNABLE_GET(glibc, memset, dc_zva_threshold, size_t, NULL);
> +#else
> +# define INIT_ZVA_THRESHOLD() 
> +#endif
> +
>  #define INIT_ARCH()							      \
> +  INIT_ZVA_THRESHOLD()							      \
>    uint64_t __attribute__((unused)) midr =				      \
>      GLRO(dl_aarch64_cpu_features).midr_el1;				      \
>    unsigned __attribute__((unused)) zva_size =				      \
> diff --git a/sysdeps/aarch64/multiarch/memset.c b/sysdeps/aarch64/multiarch/memset.c
> index 4817587..2015bce 100644
> --- a/sysdeps/aarch64/multiarch/memset.c
> +++ b/sysdeps/aarch64/multiarch/memset.c
> @@ -32,12 +32,28 @@ extern __typeof (__redirect_memset) __memset_falkor attribute_hidden;
>  extern __typeof (__redirect_memset) __memset_emag attribute_hidden;
>  extern __typeof (__redirect_memset) __memset_generic attribute_hidden;
>  
> +# if HAVE_TUNABLES
> +uint64_t __memset_dc_zva_threshold = 512;
> +
> +#  define init_memset(fn, default_zva_threshold)		\
> +({								\
> +  if (zva_threshold)						\
> +    __memset_dc_zva_threshold = zva_threshold;			\
> +  else if (default_zva_threshold)				\
> +    __memset_dc_zva_threshold = default_zva_threshold;		\
> +  fn;								\
> +})
> +# else
> +#  define init_memset(fn, default_zva_threshold)  (fn)
> +# endif
> +
>  libc_ifunc (__libc_memset,
> +
>  	    ((IS_FALKOR (midr) || IS_PHECDA (midr)) && zva_size == 64
> -	     ? __memset_falkor
> +	     ? init_memset (__memset_falkor, 0)
>  	     : (IS_EMAG (midr) && zva_size == 64
> -	       ? __memset_emag
> -	       : __memset_generic)));
> +	       ? init_memset (__memset_emag, 8*1024*1024)
> +	       : init_memset (__memset_generic, 0))));

The default threshold initialization needs to take place in the emag
file and not here, this code is already getting complicated and it won't
be long until it starts looking like a character soup.  That should also
take care of the unexplained magic number (8M).

>  
>  # undef memset
>  strong_alias (__libc_memset, memset);
> diff --git a/sysdeps/aarch64/multiarch/memset_base64.S b/sysdeps/aarch64/multiarch/memset_base64.S
> index 9a62325..6350a6d 100644
> --- a/sysdeps/aarch64/multiarch/memset_base64.S
> +++ b/sysdeps/aarch64/multiarch/memset_base64.S
> @@ -91,7 +91,14 @@ L(set96):
>  	.p2align 4
>  L(set_long):
>  	stp	val, val, [dstin]
> +#if HAVE_TUNABLES
> +	adrp	tmp1, __memset_dc_zva_threshold
> +	add	tmp1, tmp1, :lo12:__memset_dc_zva_threshold
> +	ldr	tmp2, [tmp1]	/* Load DC ZVA tunable threshold value. */
> +	cmp	count, tmp2
> +#else
>  	cmp	count, DC_ZVA_THRESHOLD
> +#endif
>  	ccmp	val, 0, 0, cs
>  	bic	dst, dstin, 15
>  	b.eq	L(zva_64)
> diff --git a/sysdeps/aarch64/multiarch/memset_emag.S b/sysdeps/aarch64/multiarch/memset_emag.S
> index 1c1fabc..78a2a14 100644
> --- a/sysdeps/aarch64/multiarch/memset_emag.S
> +++ b/sysdeps/aarch64/multiarch/memset_emag.S
> @@ -23,10 +23,10 @@
>  /*
>   * Using dc zva to zero memory does not produce better performance if
>   * memory size is not very large, especially when there are multiple
> - * processes/threads contending memory/cache. Here we use a somewhat
> + * processes/threads contending memory/cache. Here we use a very 
>   * large threshold to trigger usage of dc zva.
>  */
> -# define DC_ZVA_THRESHOLD 1024
> +# define DC_ZVA_THRESHOLD (8*1024*1024)
>  
>  # include "./memset_base64.S"
>  #endif
>