This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: [PATCH V3] benchtests:Enable BENCHSET to run subset of tests
- From: Siddhesh Poyarekar <siddhesh at gotplt dot org>
- To: Victor Rodriguez <victor dot rodriguez dot bahena at intel dot com>, libc-alpha at sourceware dot org
- Cc: Icarus Sparry <icarus dot w dot sparry at intel dot com>
- Date: Thu, 23 Nov 2017 23:10:35 +0530
- Subject: Re: [PATCH V3] benchtests:Enable BENCHSET to run subset of tests
- Authentication-results: sourceware.org; auth=none
- References: <20171123172112.5276-1-victor.rodriguez.bahena@intel.com>
On Thursday 23 November 2017 10:51 PM, Victor Rodriguez wrote:
> This patch adds BENCHSET variable to benchtests/Makefile in order to
> provide the capability to run a list of subsets of benchmark tests, ie;
>
> make bench BENCHSET="bench-pthread bench-math malloc-thread"
>
> This helps users to benchmark specific glibc area
>
> ChangeLog:
>
> * benchtests/Makefile:Add BENCHSET to allow subsets of
> benchmarks to be run.
> * benchtests/README: Add documentation for: Running subsets of
> benchmarks.
>
> Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
> Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com>
OK.
Reviewed-By: Siddhesh Poyarekar <siddhesh@sourceware.org>
> ---
> ChangeLog | 8 +++++++
> benchtests/Makefile | 69 +++++++++++++++++++++++++++++++++++++----------------
> benchtests/README | 19 +++++++++++++++
> 3 files changed, 76 insertions(+), 20 deletions(-)
>
> diff --git a/ChangeLog b/ChangeLog
> index 284bc87963..7c4a98acb6 100644
> --- a/ChangeLog
> +++ b/ChangeLog
> @@ -1,3 +1,11 @@
> +2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com>
> + Icarus Sparry <icarus.w.sparry@intel.com>
> +
> + * benchtests/Makefile:Add BENCHSET to allow subsets of
> + benchmarks to be run.
> + * benchtests/README: Add documentation for: Running subsets of
> + benchmarks.
> +
> 2017-11-15 Victor Rodriguez <victor.rodriguez.bahena@intel.com>
>
> * benchtests/scripts/benchout.schema.json: Add throughput as accepted
> diff --git a/benchtests/Makefile b/benchtests/Makefile
> index 4157f8ce22..74b3821ccf 100644
> --- a/benchtests/Makefile
> +++ b/benchtests/Makefile
> @@ -30,7 +30,11 @@ bench-pthread := pthread_once thread_create
>
> bench-string := ffs ffsll
>
> +ifeq (${BENCHSET},)
> bench := $(bench-math) $(bench-pthread) $(bench-string)
> +else
> +bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}})
> +endif
>
> # String function benchmarks.
> string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
> @@ -68,8 +72,12 @@ stdio-common-benchset := sprintf
>
> math-benchset := math-inlines
>
> +ifeq (${BENCHSET},)
> benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \
> $(math-benchset)
> +else
> +benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}})
> +endif
>
> CFLAGS-bench-ffs.c += -fno-builtin
> CFLAGS-bench-ffsll.c += -fno-builtin
> @@ -81,7 +89,11 @@ CFLAGS-bench-fmaxf.c += -fno-builtin
> CFLAGS-bench-trunc.c += -fno-builtin
> CFLAGS-bench-truncf.c += -fno-builtin
>
> +ifeq (${BENCHSET},)
> bench-malloc := malloc-thread
> +else
> +bench-malloc := $(filter malloc-%,${BENCHSET})
> +endif
>
> $(addprefix $(objpfx)bench-,$(bench-math)): $(libm)
> $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm)
> @@ -149,6 +161,19 @@ bench-clean:
> rm -f $(timing-type) $(addsuffix .o,$(timing-type))
> rm -f $(addprefix $(objpfx),$(bench-extra-objs))
>
> +# Validate the passed in BENCHSET
> +ifneq ($(strip ${BENCHSET}),)
> +VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \
> + wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \
> + malloc-thread
> +INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET})
> +ifneq (${INVALIDBENCHSETNAMES},)
> +$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES})
> +$(info The valid ones are: ${VALIDBENCHSETNAMES})
> +$(error Invalid BENCHSET value)
> +endif
> +endif
> +
> # Define the bench target only if the target has a usable python installation.
> ifdef PYTHON
> bench: bench-build bench-set bench-func bench-malloc
> @@ -175,10 +200,11 @@ bench-set: $(binaries-benchset)
> done
>
> bench-malloc: $(binaries-bench-malloc)
> - run=$(objpfx)bench-malloc-thread; \
> - for thr in 1 8 16 32; do \
> - echo "Running $${run} $${thr}"; \
> + for run in $^; do \
> + for thr in 1 8 16 32; do \
> + echo "Running $${run} $${thr}"; \
> $(run-bench) $${thr} > $${run}-$${thr}.out; \
> + done;\
> done
>
> # Build and execute the benchmark functions. This target generates JSON
> @@ -186,25 +212,28 @@ bench-malloc: $(binaries-bench-malloc)
> # so one could even execute them individually and process it using any JSON
> # capable language or tool.
> bench-func: $(binaries-bench)
> + if [ -n '$^' ] ; then \
> { timing_type=$$($(timing-type)); \
> - echo "{\"timing_type\": \"$${timing_type}\","; \
> - echo " \"functions\": {"; \
> - for run in $^; do \
> - if ! [ "x$${run}" = "x$<" ]; then \
> - echo ","; \
> + echo "{\"timing_type\": \"$${timing_type}\","; \
> + echo " \"functions\": {"; \
> + for run in $^; do \
> + if ! [ "x$${run}" = "x$<" ]; then \
> + echo ","; \
> + fi; \
> + echo "Running $${run}" >&2; \
> + $(run-bench) $(DETAILED_OPT); \
> + done; \
> + echo; \
> + echo " }"; \
> + echo "}"; \
> + } > $(objpfx)bench.out-tmp; \
> + if [ -f $(objpfx)bench.out ]; then \
> + mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> fi; \
> - echo "Running $${run}" >&2; \
> - $(run-bench) $(DETAILED_OPT); \
> - done; \
> - echo; \
> - echo " }"; \
> - echo "}"; } > $(objpfx)bench.out-tmp; \
> - if [ -f $(objpfx)bench.out ]; then \
> - mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
> - fi; \
> - mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
> - $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> - scripts/benchout.schema.json
> + mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \
> + $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
> + scripts/benchout.schema.json; \
> + fi
>
> $(timing-type) $(binaries-bench) $(binaries-benchset) \
> $(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \
> diff --git a/benchtests/README b/benchtests/README
> index 9aa750a519..4ddff794d1 100644
> --- a/benchtests/README
> +++ b/benchtests/README
> @@ -53,6 +53,25 @@ otherwise the above command may try to build the benchmark again. Benchmarks
> that require generated code to be executed during the build are skipped when
> cross-building.
>
> +Running subsets of benchmarks:
> +==============================
> +
> +To run only a subset of benchmarks, one may invoke make as follows
> +
> + $ make bench BENCHSET="bench-pthread bench-math malloc-thread"
> +
> +where BENCHSET may be a space-separated list of the following values:
> +
> + bench-math
> + bench-pthread
> + bench-string
> + string-benchset
> + wcsmbs-benchset
> + stdlib-benchset
> + stdio-common-benchset
> + math-benchset
> + malloc-thread
> +
> Adding a function to benchtests:
> ===============================
>
>