[RFC] Process files in parallel
Tom de Vries
tdevries@suse.de
Tue Mar 23 20:21:45 GMT 2021
Hi,
Add an option -j <n> / --jobs <n> that allows multiple files to be processed
in parallel. Note that this does not yet parallelize when multifile is used.
Consider the experiment do.sh:
...
ns=$(seq 1 10)
for n in $ns; do
cp debug/cc1 $n
done
time.sh ./dwz $@ -lnone $ns
...
On a 4 smt-thread, dual core system we get:
...
$ for n in $(seq 1 4); do echo "N: $n"; ./do.sh -j $n; done
N: 1
maxmem: 1261744
real: 56.53
user: 53.24
system: 3.17
N: 2
maxmem: 1260216
real: 31.74
user: 58.84
system: 4.39
N: 3
maxmem: 1261868
real: 28.26
user: 75.65
system: 5.00
N: 4
maxmem: 1262036
real: 26.80
user: 87.31
system: 5.69
...
The sweet spot of real time reduction vs. extra user/system time seems to be
around 2, so we set the default -j to processors / 2.
Any comments?
Thanks,
- Tom
Process files in parallel
2021-03-23 Tom de Vries <tdevries@suse.de>
PR dwz/25951
* args.c (max_forks): New var.
(dwz_options, dwz_multi_file_options_help, usage): Add entries for -j <n>.
(parse_args): Handle -j <n>. Assign default value for max_forks.
* args.h (max_forks): Declare.
* dwz.1: Add entries for -j <n> / --jobs <n> entry.
* dwz.c (dwz_files_1): Handle max_forks.
---
args.c | 27 +++++++++++++--
args.h | 1 +
dwz.1 | 4 +++
dwz.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
4 files changed, 137 insertions(+), 18 deletions(-)
diff --git a/args.c b/args.c
index cc8c717..11946b8 100644
--- a/args.c
+++ b/args.c
@@ -25,6 +25,7 @@
#include <string.h>
#include <stdlib.h>
#include <error.h>
+#include <sys/sysinfo.h>
#include "args.h"
@@ -52,6 +53,7 @@ int progress_p;
int progress_mem_p;
int import_opt_p = 1;
int force_p;
+int max_forks = -1;
enum deduplication_mode deduplication_mode = dm_inter_cu;
@@ -161,6 +163,7 @@ static struct option dwz_options[] =
{ "odr", no_argument, &odr, 1 },
{ "no-odr", no_argument, &odr, 0 },
{ "odr-mode", required_argument, &odr_mode_parsed, 1 },
+ { "jobs", required_argument, 0, 'j' },
{ NULL, no_argument, 0, 0 }
};
@@ -219,7 +222,9 @@ static struct option_help dwz_multi_file_options_help[] =
" to multifile." },
{ "5", "dwarf-5", NULL, NULL,
"Emit DWARF 5 standardized supplementary object files instead of"
- " GNU extension .debug_altlink." }
+ " GNU extension .debug_altlink." },
+ { "j", "jobs", "<n>", "number of processors / 2",
+ "Process <n> files in parallel" }
};
/* Describe misc command line options. */
@@ -363,7 +368,8 @@ usage (const char *progname, int failing)
fprintf (stream,
("Usage:\n"
- " %s [common options] [-h] [-m COMMONFILE] [-M NAME | -r] [FILES]\n"
+ " %s [common options] [-h] [-m COMMONFILE] [-M NAME | -r] [-j N]"
+ " [FILES]\n"
" %s [common options] -o OUTFILE FILE\n"
" %s [ -v | -? ]\n"),
progname, progname, progname);
@@ -487,7 +493,7 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
while (1)
{
int option_index = -1;
- int c = getopt_long (argc, argv, "m:o:qhl:L:M:r?v5", dwz_options,
+ int c = getopt_long (argc, argv, "m:o:j:qhl:L:M:r?v5", dwz_options,
&option_index);
if (c == -1)
break;
@@ -619,6 +625,13 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
case 'v':
version ();
break;
+
+ case 'j':
+ l = strtoul (optarg, &end, 0);
+ if (*end != '\0' || optarg == end || (unsigned int) l != l)
+ error (1, 0, "invalid argument -j %s", optarg);
+ max_forks = l;
+ break;
}
}
@@ -634,4 +647,12 @@ parse_args (int argc, char *argv[], bool *hardlink, const char **outfile)
if (multifile_relative && multifile_name)
error (1, 0, "-M and -r options can't be specified together");
+
+ if (max_forks == -1)
+ {
+ long nprocs = get_nprocs ();
+ /* Be conservative on max forks: 4 procs may be actually be 4 SMT threads
+ with only 2 cores. */
+ max_forks = nprocs / 2;
+ }
}
diff --git a/args.h b/args.h
index c899003..ca71194 100644
--- a/args.h
+++ b/args.h
@@ -54,6 +54,7 @@ extern int progress_p;
extern int progress_mem_p;
extern int import_opt_p;
extern int force_p;
+extern int max_forks;
enum deduplication_mode
{
diff --git a/dwz.1 b/dwz.1
index 19dc814..23fe878 100644
--- a/dwz.1
+++ b/dwz.1
@@ -111,6 +111,10 @@ Emit standard DWARF 5 Supplementary Object Files with \fI.debug_sup\fR and
corresponding forms, instead of the GNU extension \fI.gnu_debugaltlink\fR
and corresponding forms.
.TP
+.B \-j <N> \-\-jobs <N>
+Process \fIN\fR files in parallel. The default is processors / 2. Disabled
+when multifile is used.
+.TP
.B \-\-odr / \-\-no-odr
.B Experimental.
Enable/disable One-Definition-Rule optimization for C++ compilation units.
diff --git a/dwz.c b/dwz.c
index 245e540..07e1da1 100644
--- a/dwz.c
+++ b/dwz.c
@@ -34,6 +34,7 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/times.h>
+#include <sys/wait.h>
#include <obstack.h>
@@ -16392,22 +16393,114 @@ dwz_files_1 (int nr_files, char *files[], bool hardlink,
if (hardlink)
hardlink = detect_hardlinks (nr_files, files, resa);
- for (i = 0; i < nr_files; i++)
+ int nr_forks = 0;
+ if (max_forks > 1 && multifile == NULL)
{
- int thisret;
- file = files[i];
- struct file_result *res = &resa[i];
- if (res->res == -2)
- /* Skip hard links. */
- continue;
- if (stats_p)
- init_stats (file);
- bool low_mem_p;
- thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
- if (thisret == 1)
- ret = 1;
- else if (!low_mem_p && resa[i].res >= 0)
- successcount++;
+ pid_t pids[nr_files];
+ for (i = 0; i < nr_files; i++)
+ {
+ int thisret;
+ file = files[i];
+ struct file_result *res = &resa[i];
+ if (res->res == -2)
+ /* Skip hard links. */
+ continue;
+
+ if (nr_forks == max_forks)
+ {
+ int state;
+ pid_t got_pid = waitpid (-1, &state, 0);
+ if (!WIFEXITED (state))
+ error (1, 0, "Child dwz process got killed");
+ thisret = WEXITSTATUS (state) & 0x3;
+ bool low_mem_p = false;
+ if (thisret == 2)
+ {
+ thisret = 0;
+ low_mem_p = true;
+ }
+ (void)low_mem_p;
+ res->res = (int)((WEXITSTATUS (state) & ~0x3) >> 2) - 3;
+ if (thisret == 1)
+ ret = 1;
+ nr_forks--;
+ int j;
+ for (j = 0; j < i; ++j)
+ if (pids[j] == got_pid)
+ {
+ pids[j] = 0;
+ break;
+ }
+ assert (j < i);
+ }
+
+ pid_t fork_res = fork ();
+ assert (fork_res != -1);
+ if (fork_res == 0)
+ {
+ bool low_mem_p;
+ thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
+ /* Encode thisret, low_mem_p and res->res into return status. */
+ if (thisret == 0 && low_mem_p)
+ thisret = 2;
+ assert (thisret >= 0 && thisret <= 2);
+ assert (res->res >= -3);
+ thisret = thisret + ((res->res + 3) << 2);
+ return thisret;
+ }
+ else
+ {
+ pids[i] = fork_res;
+ nr_forks++;
+ }
+ }
+ for (i = 0; i < nr_files; i++)
+ {
+ int thisret;
+ file = files[i];
+ struct file_result *res = &resa[i];
+ if (res->res == -2)
+ /* Skip hard links. */
+ continue;
+ if (pids[i] == 0)
+ continue;
+ int state;
+ pid_t got_pid = waitpid (pids[i], &state, 0);
+ assert (got_pid == pids[i]);
+ if (!WIFEXITED (state))
+ error (1, 0, "Child dwz process got killed");
+ thisret = WEXITSTATUS (state) & 0x3;
+ bool low_mem_p = false;
+ if (thisret == 2)
+ {
+ thisret = 0;
+ low_mem_p = true;
+ }
+ (void)low_mem_p;
+ res->res = (int)((WEXITSTATUS (state) & ~0x3) >> 2) - 3;
+ if (thisret == 1)
+ ret = 1;
+ }
+ }
+ else
+ {
+ for (i = 0; i < nr_files; i++)
+ {
+ int thisret;
+ file = files[i];
+ struct file_result *res = &resa[i];
+ if (res->res == -2)
+ /* Skip hard links. */
+ continue;
+ if (stats_p)
+ init_stats (file);
+ bool low_mem_p;
+ thisret = dwz_with_low_mem (file, NULL, res, &low_mem_p);
+ if (thisret == 1)
+ ret = 1;
+ else if (!low_mem_p && resa[i].res >= 0)
+ successcount++;
+ }
}
if (hardlink)
More information about the Dwz
mailing list