This is the mail archive of the glibc-cvs@sourceware.org mailing list for the glibc project.
Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
---|---|---|
Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |
Other format: | [Raw text] |
This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "GNU C Library master sources". The branch, zack/build-many-improvements has been created at 45666f9bdab2ba3fa7edb34ec4ee9559c525beb3 (commit) - Log ----------------------------------------------------------------- http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=45666f9bdab2ba3fa7edb34ec4ee9559c525beb3 commit 45666f9bdab2ba3fa7edb34ec4ee9559c525beb3 Author: Zack Weinberg <zackw@panix.com> Date: Fri Jul 7 10:45:37 2017 -0400 build-many-glibcs: Impose a memory limit on build processes. There are sometimes bugs in the compiler (e.g. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78460) that cause it to consume all available memory. To limit the impact of this on automated test robots, impose memory limits on all subprocesses in build-many-glibcs.py. When the bug hits, the compiler will still run out of memory and crash, but that should not affect any other simultaneous task. The limit can be configured with the --memory-limit command line switch. The default is 1.5 gigabytes or physical RAM divided by the number of jobs to run in parallel, whichever is larger. (Empirically, 1.5 gigs per process is enough for everything but the files affected by GCC bug 78640, but 1 gig per process is insufficient for some of the math tests and also for the "genautomata" step when building compilers for powerpc64.) Rather than continue to lengthen the argument list of the Context constructor, it now takes the entire 'opts' object as its sole argument. * scripts/build-many-glibcs.py (total_ram): New function. (Context.set_memory_limits): New function. (Context.run_builds): Call set_memory_limits immediately before do_build. (get_parser): Add --memory-limit command-line switch. (Context.__init__): Take 'opts' object as sole argument. Add 'memory_limit' attribute to self. Make topdir absolute here. (main): Update to match. diff --git a/scripts/build-many-glibcs.py b/scripts/build-many-glibcs.py index 7b267d1..a4d72f2 100755 --- a/scripts/build-many-glibcs.py +++ b/scripts/build-many-glibcs.py @@ -86,33 +86,56 @@ except: subprocess.run = _run +def total_ram(): + """Retrieve the total amount of physical RAM available on this computer.""" + + # This can't be done cross-platform using the Python standard library. + # If the add-on 'psutil' module is available, use it. + try: + import psutil + # Despite the name, virtual_memory() counts only physical RAM, not swap. + return psutil.virtual_memory().total + + except ImportError: + pass + + # This works on Linux, but (reportedly) not on all other Unixes. + try: + return \ + os.sysconf('SC_PAGESIZE') * os.sysconf('SC_PHYS_PAGES') + + except: + pass + + # We don't know. Return a very large number. + return sys.maxsize class Context(object): """The global state associated with builds in a given directory.""" - def __init__(self, topdir, parallelism, keep, replace_sources, strip, - action): + def __init__(self, opts): """Initialize the context.""" - self.topdir = topdir - self.parallelism = parallelism - self.keep = keep - self.replace_sources = replace_sources - self.strip = strip - self.srcdir = os.path.join(topdir, 'src') + self.topdir = os.path.abspath(opts.topdir) + self.parallelism = opts.parallelism + self.memory_limit = opts.memory_limit + self.keep = opts.keep + self.replace_sources = opts.replace_sources + self.strip = opts.strip + self.srcdir = os.path.join(self.topdir, 'src') self.versions_json = os.path.join(self.srcdir, 'versions.json') - self.build_state_json = os.path.join(topdir, 'build-state.json') - self.bot_config_json = os.path.join(topdir, 'bot-config.json') - self.installdir = os.path.join(topdir, 'install') + self.build_state_json = os.path.join(self.topdir, 'build-state.json') + self.bot_config_json = os.path.join(self.topdir, 'bot-config.json') + self.installdir = os.path.join(self.topdir, 'install') self.host_libraries_installdir = os.path.join(self.installdir, 'host-libraries') - self.builddir = os.path.join(topdir, 'build') - self.logsdir = os.path.join(topdir, 'logs') - self.logsdir_old = os.path.join(topdir, 'logs-old') + self.builddir = os.path.join(self.topdir, 'build') + self.logsdir = os.path.join(self.topdir, 'logs') + self.logsdir_old = os.path.join(self.topdir, 'logs-old') self.makefile = os.path.join(self.builddir, 'Makefile') self.wrapper = os.path.join(self.builddir, 'wrapper') self.save_logs = os.path.join(self.builddir, 'save-logs') self.script_text = self.get_script_text() - if action != 'checkout': + if opts.action != 'checkout': self.build_triplet = self.get_build_triplet() self.glibc_version = self.get_glibc_version() self.configs = {} @@ -134,6 +157,50 @@ class Context(object): sys.stdout.flush() os.execv(sys.executable, [sys.executable] + sys.argv) + def set_memory_limits(self): + """Impose a memory-consumption limit on this process, and therefore + all of the subprocesses it creates. The limit can be set + on the command line; the default is either physical RAM + divided by the number of jobs to be run in parallel, or 1.5 + gigabytes, whichever is larger. (1GB is too small for + genautomata on MIPS and for the compilation of several + large math test cases.) + """ + if self.memory_limit == 0: + return + try: + import resource + except ImportError as e: + print('warning: cannot set memory limit:', e) + return + + if self.memory_limit is None: + physical_ram = total_ram() + memory_limit = int(max(physical_ram / self.parallelism, + 1.5 * 1024 * 1024 * 1024)) + else: + if memory_limit < 1.5: + print('warning: memory limit %.5g GB known to be too small' + % memory_limit) + memory_limit = int(memory_limit * 1024 * 1024 * 1024) + + set_a_limit = False + for mem_rsrc_name in ['RLIMIT_DATA', 'RLIMIT_STACK', 'RLIMIT_RSS', + 'RLIMIT_VMEM', 'RLIMIT_AS']: + mem_rsrc = getattr(resource, mem_rsrc_name, None) + if mem_rsrc is not None: + soft, hard = resource.getrlimit(mem_rsrc) + if hard == resource.RLIM_INFINITY or hard > memory_limit: + hard = memory_limit + if soft == resource.RLIM_INFINITY or soft > hard: + soft = hard + resource.setrlimit(mem_rsrc, (soft, hard)) + set_a_limit = True + + if set_a_limit: + print("Per-process memory limit set to %.5g GB." % + (memory_limit / (1024 * 1024 * 1024))) + def get_build_triplet(self): """Determine the build triplet with config.guess.""" config_guess = os.path.join(self.component_srcdir('gcc'), @@ -465,6 +532,7 @@ class Context(object): old_versions = self.build_state['compilers']['build-versions'] self.build_glibcs(configs) self.write_files() + self.set_memory_limits() self.do_build() if configs: # Partial build, do not update stored state. @@ -1589,6 +1657,9 @@ def get_parser(): parser.add_argument('-j', dest='parallelism', help='Run this number of jobs in parallel', type=int, default=os.cpu_count()) + parser.add_argument('--memory-limit', + help='Per-process memory limit in gigabytes (0 for unlimited)', + type=float, default=None) parser.add_argument('--keep', dest='keep', help='Whether to keep all build directories, ' 'none or only those from failed builds', @@ -1614,9 +1685,7 @@ def main(argv): """The main entry point.""" parser = get_parser() opts = parser.parse_args(argv) - topdir = os.path.abspath(opts.topdir) - ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources, - opts.strip, opts.action) + ctx = Context(opts) ctx.run_builds(opts.action, opts.configs) http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=04be1dfbe7485cf3949f3e4cde8ade4901cc662f commit 04be1dfbe7485cf3949f3e4cde8ade4901cc662f Author: Zack Weinberg <zackw@panix.com> Date: Fri Jul 7 09:58:23 2017 -0400 build-many-glibcs: Install kernel headers separately. When building glibc, ideally we want the compiler's headers and the kernel's headers to be visible, but not (a previous version of) our own headers, or any third-party headers. Our own build process already supports this, with the configure option --with-headers, but build-many-glibcs doesn't use that option, and when prepping compilers it installs kernel headers in $(sysroot)/usr/include, the same place that the preliminary build of glibc will install its own headers. This patch changes build-many-glibcs to install the kernel headers in $(sysroot)/usr/share/linux/include (only Linux targets are currently supported in build-many-glibcs) and then symlink them into $(sysroot)/usr/include. A symlink farm is necessary for the "full" build of GCC to work correctly. The way it is created is a little kludgey and exposed a bug in the shell quotation logic, but it works. Note that $(sysroot)/usr/share/linux/include is made read-only to all after it is created, in order to catch cases where glibc's "make install" installs headers into a subdirectory belonging to the kernel. One such directory already exists, /usr/include/scsi, and it's special-cased in here, but we should fix that. * scripts/build-many-glibcs.py (Config.install_linux_headers): Install the headers in $sysroot/usr/share/linux/include and then make them read-only and create a symlink farm in $sysroot/usr/include pointing to them. (Glibc.build_glibc): If the OS is Linux, pass configure the --with-headers=$sysroot/usr/share/linux/include option. (CommandList.link_dir_contents): New function. (Command.__init__, Command.shell_make_quote_string): Use str.replace instead of str.translate. (Context.write_files): Process all nested quotes, not just the first. diff --git a/scripts/build-many-glibcs.py b/scripts/build-many-glibcs.py index 5fbb564..7b267d1 100755 --- a/scripts/build-many-glibcs.py +++ b/scripts/build-many-glibcs.py @@ -525,7 +525,7 @@ class Context(object): ' printf " %s" "$word"\n' ' else\n' ' printf " \'"\n' - ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n' + ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/g"\n' ' printf "\'"\n' ' fi\n' 'done >> "$this_log"\n' @@ -1213,15 +1213,37 @@ class Config(object): assert linux_arch is not None srcdir = self.ctx.component_srcdir('linux') builddir = self.component_builddir('linux') - headers_dir = os.path.join(self.sysroot, 'usr') + kernel_prefix = os.path.join(self.sysroot, 'usr/share/linux') + kernel_headers_dir = os.path.join(kernel_prefix, 'include') + all_headers_dir = os.path.join(self.sysroot, 'usr/include') cmdlist.push_subdesc('linux') cmdlist.create_use_dir(builddir) + # Only this operation has any business writing files below the + # kernel_prefix; in particular we want to trap any case where + # glibc's "make install" installs a header in a subdirectory of + # /usr/include that belongs to the kernel. + cmdlist.add_command('install-mkdir', + ['mkdir', '-p', kernel_prefix]) + cmdlist.add_command('install-make-writable', + ['chmod', '-R', 'u+w', kernel_prefix]) cmdlist.add_command('install-headers', ['make', '-C', srcdir, 'O=%s' % builddir, 'ARCH=%s' % linux_arch, - 'INSTALL_HDR_PATH=%s' % headers_dir, + 'INSTALL_HDR_PATH=%s' % kernel_prefix, 'headers_install']) cmdlist.cleanup_dir() + cmdlist.add_command('install-make-read-only', + ['chmod', '-R', 'a-w', kernel_prefix]) + cmdlist.link_dir_contents(kernel_headers_dir, all_headers_dir) + # Currently, some headers in /usr/include/scsi are provided by + # the kernel and some by glibc. This should be cleaned up, but + # for the time being we have to special-case this directory. + cmdlist.add_command('link-scsi-rm', + ['rm', '-f', os.path.join(all_headers_dir, 'scsi')]) + cmdlist.link_dir_contents( + os.path.join(kernel_headers_dir, 'scsi'), + os.path.join(all_headers_dir, 'scsi'), + 'scsi') cmdlist.pop_subdesc() def build_gcc(self, cmdlist, bootstrap): @@ -1357,6 +1379,11 @@ class Glibc(object): 'RANLIB=%s' % self.tool_name('ranlib'), 'READELF=%s' % self.tool_name('readelf'), 'STRIP=%s' % self.tool_name('strip')] + if self.os.startswith('linux'): + cfg_cmd.append('--with-headers=%s' + % os.path.join(self.compiler.sysroot, + 'usr/share/linux/include')) + cfg_cmd += self.cfg cmdlist.add_command('configure', cfg_cmd) cmdlist.add_command('build', ['make']) @@ -1389,8 +1416,7 @@ class Command(object): self.dir = dir self.path = path self.desc = desc - trans = str.maketrans({' ': '-'}) - self.logbase = '%03d-%s' % (num, desc.translate(trans)) + self.logbase = '%03d-%s' % (num, desc.replace(' ', '-')) self.command = command self.always_run = always_run @@ -1401,10 +1427,8 @@ class Command(object): assert '\n' not in s if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s): return s - strans = str.maketrans({"'": "'\\''"}) - s = "'%s'" % s.translate(strans) - mtrans = str.maketrans({'$': '$$'}) - return s.translate(mtrans) + s = "'%s'" % s.replace("'", "'\\''") + return s.replace('$', '$$') @staticmethod def shell_make_quote_list(l, translate_make): @@ -1470,6 +1494,32 @@ class CommandList(object): self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent]) self.add_command_dir('copy', None, ['cp', '-a', src, dest]) + def link_dir_contents(self, src, dest, tag=''): + """Create relative symbolic links in DEST pointing to each of the + files and directories in SRC. If DEST does not already exist, + it is created.""" + relpath = os.path.relpath(src, dest) + if tag: + mkdir_tag = 'link-%s-mkdir' % tag + link_tag = 'link-%s' % tag + else: + mkdir_tag = 'link-mkdir' + link_tag = 'link' + + self.add_command_dir(mkdir_tag, None, ['mkdir', '-p', dest]) + + # The exec() below works around a limitation of the Python + # grammar; a 'compound' statement (like 'for f in ...: ...') + # cannot be set off from a preceding statement with a + # semicolon, only a newline. Since shell_make_quote can't + # quote newlines, we have to turn the loop into a 'simple' + # statement somehow. + self.add_command_dir(link_tag, dest, [ + sys.executable, '-c', + "import glob, os, sys;" + "exec(\"for f in glob.glob(os.path.join(sys.argv[1], '*')):" + " os.symlink(f, os.path.basename(f))\")", relpath]) + def add_command_dir(self, desc, dir, command, always_run=False): """Add a command to run in a given directory.""" cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path, ----------------------------------------------------------------------- hooks/post-receive -- GNU C Library master sources
Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
---|---|---|
Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |