Abnormal memory usage with glibc 2.31 related to trimming strategy ?

[Xavier Roche]

Hi,

On Mon, Sep 21, 2020 at 1:18 PM Xavier Roche <xavier.roche@algolia.com> wrote:
> Just to add some small details that may be insightful: in the later
> scenario, with 12 heaps, the total overall rest size was 100376946145

To be correct, even with GLIBC_TUNABLES=glibc.malloc.arena_max=1, the
pool (which is the main sbrk() pool, I suppose) grows.
In the later example, I could count 6806 blocks up to 35326884838 Bytes.

The interesting part is that each pool does not seem to consume more
than the maximum memory allocated at some point in the process (see
green cuve: https://i.imgur.com/Mig7Vek.png).

Could it be a threshold that is pushed to an upper limit ?


<malloc version="1">
<heap nr="0">
<sizes>
  <size from="49" to="49" total="441" count="9"/>
  <size from="113" to="113" total="34613369" count="306313"/>
  <size from="129" to="129" total="174795" count="1355"/>
  <size from="193" to="193" total="193" count="1"/>
  <size from="209" to="209" total="209" count="1"/>
  <size from="241" to="241" total="6266" count="26"/>
  <size from="257" to="257" total="1542" count="6"/>
  <size from="273" to="273" total="2730" count="10"/>
  <size from="289" to="289" total="289" count="1"/>
  <size from="353" to="353" total="353" count="1"/>
  <size from="449" to="449" total="449" count="1"/>
  <size from="465" to="465" total="465" count="1"/>
  <size from="481" to="481" total="481" count="1"/>
  <size from="513" to="513" total="513" count="1"/>
  <size from="545" to="545" total="545" count="1"/>
  <size from="657" to="657" total="657" count="1"/>
  <size from="1009" to="1009" total="1009" count="1"/>
  <size from="1041" to="1041" total="27066" count="26"/>
  <size from="1169" to="1169" total="1169" count="1"/>
  <size from="1601" to="1649" total="576371" count="355"/>
  <size from="1665" to="1713" total="602773" count="357"/>
  <size from="1729" to="1777" total="599126" count="342"/>
  <size from="1793" to="1841" total="510185" count="281"/>
  <size from="1857" to="1905" total="590058" count="314"/>
  <size from="1921" to="1969" total="507509" count="261"/>
  <size from="1985" to="2033" total="527687" count="263"/>
  <size from="2049" to="2049" total="145479" count="71"/>
  <size from="2337" to="2353" total="162053" count="69"/>
  <size from="2369" to="2417" total="538753" count="225"/>
  <size from="2433" to="2481" total="608664" count="248"/>
  <size from="2497" to="2545" total="585288" count="232"/>
  <size from="2561" to="2609" total="566843" count="219"/>
  <size from="2625" to="2673" total="598210" count="226"/>
  <size from="2689" to="2737" total="637595" count="235"/>
  <size from="2753" to="2801" total="572606" count="206"/>
  <size from="2817" to="2865" total="590912" count="208"/>
  <size from="2881" to="2929" total="597982" count="206"/>
  <size from="2945" to="2993" total="445638" count="150"/>
  <size from="3009" to="3057" total="497748" count="164"/>
  <size from="3073" to="3121" total="548737" count="177"/>
  <size from="3137" to="3569" total="3185467" count="955"/>
  <size from="3585" to="4081" total="3172670" count="830"/>
  <size from="4097" to="4097" total="532610" count="130"/>
  <size from="11153" to="12273" total="6874748" count="588"/>
  <size from="12289" to="16369" total="27765895" count="1975"/>
  <size from="16385" to="20465" total="23269372" count="1260"/>
  <size from="20481" to="24561" total="24447050" count="1098"/>
  <size from="24577" to="28657" total="22167173" count="837"/>
  <size from="28673" to="32753" total="22671238" count="742"/>
  <size from="33985" to="36849" total="14771665" count="417"/>
  <size from="36865" to="40945" total="20814264" count="536"/>
  <size from="40961" to="65521" total="111657063" count="2135"/>
  <size from="65553" to="98241" total="149901396" count="1860"/>
  <size from="98305" to="131041" total="144180545" count="1265"/>
  <size from="131073" to="163825" total="146380250" count="1002"/>
  <size from="163857" to="262113" total="330656633" count="1609"/>
  <size from="262161" to="524177" total="753145332" count="2068"/>
  <size from="524913" to="1162873249" total="35326884838" count="6806"/>
  <unsorted from="7041" to="7041" total="7041" count="1"/>
</sizes>
<total type="fast" count="0" size="0"/>
<total type="rest" count="338681" size="37178462424"/>
<system type="current" size="37812953048"/>
<system type="max" size="38723010520"/>
<aspace type="total" size="37812953048"/>
<aspace type="mprotect" size="37812953048"/>
</heap>
<total type="fast" count="0" size="0"/>
<total type="rest" count="338681" size="37178462424"/>
<total type="mmap" count="0" size="0"/>
<system type="current" size="37812953048"/>
<system type="max" size="38723010520"/>
<aspace type="total" size="37812953048"/>
<aspace type="mprotect" size="37812953048"/>
</malloc>


Regards,

-- 
Xavier Roche -