PR: 25978

Noah Sanci nsanci@redhat.com
Fri Jul 2 18:24:14 GMT 2021


diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index 286c910a..06d03e72 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,9 @@
+2021-06-28 Noah Sanci <nsanci@redhat.com>
+
+ PR25978
+ * debuginfod.cxx: Added command line options
+ --fdcache-prefetch-fds/mbs and associated metrics/functionality.
+
 2021-06-03  Frank Ch. Eigler <fche@redhat.com>

  PR27863
diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx
index 543044c6..431605aa 100644
--- a/debuginfod/debuginfod.cxx
+++ b/debuginfod/debuginfod.cxx
@@ -368,7 +368,13 @@ static const struct argp_option options[] =
    { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number
of archive files to prefetch into fdcache.", 0 },
 #define ARGP_KEY_FDCACHE_MINTMP 0x1004
    { "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum
free space% on tmpdir.", 0 },
-   { NULL, 0, NULL, 0, NULL, 0 }
+#define ARGP_KEY_FDCACHE_PREFETCH_MBS 0x1005
+   { "fdcache-prefetch-mbs", ARGP_KEY_FDCACHE_PREFETCH_MBS, "MB",
0,"Megabytes allocated to the \
+      prefetch cache.", 0},
+#define ARGP_KEY_FDCACHE_PREFETCH_FDS 0x1006
+   { "fdcache-prefetch-fds", ARGP_KEY_FDCACHE_PREFETCH_FDS, "NUM",
0,"Number of files allocated to the \
+      prefetch cache.", 0},
+   { NULL, 0, NULL, 0, NULL, 0 },
   };

 /* Short description of program.  */
@@ -412,6 +418,8 @@ static long fdcache_fds;
 static long fdcache_mbs;
 static long fdcache_prefetch;
 static long fdcache_mintmp;
+static long fdcache_prefetch_mbs;
+static long fdcache_prefetch_fds;
 static string tmpdir;

 static void set_metric(const string& key, double value);
@@ -538,10 +546,22 @@ parse_opt (int key, char *arg,
       break;
     case ARGP_KEY_FDCACHE_MINTMP:
       fdcache_mintmp = atol (arg);
+      if( fdcache_mintmp > 100 || fdcache_mintmp < 0 )
+        argp_failure(state, 1, EINVAL, "fdcache mintmp percent");
       break;
     case ARGP_KEY_ARG:
       source_paths.insert(string(arg));
       break;
+    case ARGP_KEY_FDCACHE_PREFETCH_FDS:
+      fdcache_prefetch_fds = atol(arg);
+      if ( fdcache_prefetch_fds <= 0)
+        argp_failure(state, 1, EINVAL, "fdcache prefetch fds");
+      break;
+    case ARGP_KEY_FDCACHE_PREFETCH_MBS:
+      fdcache_prefetch_mbs = atol(arg);
+      if ( fdcache_prefetch_mbs <= 0)
+        argp_failure(state, 1, EINVAL, "fdcache prefetch mbs");
+      break;
       // case 'h': argp_state_help (state, stderr,
ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
     default: return ARGP_ERR_UNKNOWN;
     }
@@ -1199,16 +1219,24 @@ private:
   };
   deque<fdcache_entry> lru; // @head: most recently used
   long max_fds;
+  deque<fdcache_entry> prefetch; // prefetched
   long max_mbs;
+  long max_prefetch_mbs;
+  long max_prefetch_fds;

 public:
   void set_metrics()
   {
-    double total_mb = 0.0;
+    double fdcache_mb = 0.0;
+    double prefetch_mb = 0.0;
     for (auto i = lru.begin(); i < lru.end(); i++)
-      total_mb += i->fd_size_mb;
-    set_metric("fdcache_bytes", (int64_t)(total_mb*1024.0*1024.0));
-    set_metric("fdcache_count", lru.size());
+      fdcache_mb += i->fd_size_mb;
+    for (auto j = prefetch.begin(); j < prefetch.end(); j++)
+      prefetch_mb += j->fd_size_mb;
+    set_metric("fdcache_bytes", fdcache_mb*1024.0*1024.0);
+    set_metric("fdcache_count", fdcache_mb);
+    set_metric("fdcache_prefetch_bytes", prefetch_mb*1024.0*1024.0);
+    set_metric("fdcache_prefetch_count", prefetch_mb);
   }

   void intern(const string& a, const string& b, string fd, off_t sz,
bool front_p)
@@ -1221,7 +1249,17 @@ public:
             {
               unlink (i->fd.c_str());
               lru.erase(i);
-              inc_metric("fdcache_op_count","op","dequeue");
+              inc_metric("fdcache_op_count","op","lru_dequeue");
+              break; // must not continue iterating
+            }
+        }
+      for (auto i = prefetch.begin(); i < prefetch.end(); i++) //
nuke preexisting copy in prefetch
+        {
+          if (i->archive == a && i->entry == b)
+            {
+              unlink (i->fd.c_str());
+              prefetch.erase(i);
+              inc_metric("fdcache_op_count","op","prefetch_dequeue");
               break; // must not continue iterating
             }
         }
@@ -1229,13 +1267,13 @@ public:
       fdcache_entry n = { a, b, fd, mb };
       if (front_p)
         {
-          inc_metric("fdcache_op_count","op","enqueue_front");
+          inc_metric("fdcache_op_count","op","lru_enqueue");
           lru.push_front(n);
         }
       else
         {
-          inc_metric("fdcache_op_count","op","enqueue_back");
-          lru.push_back(n);
+          inc_metric("fdcache_op_count","op","prefetch_enqueue");
+          prefetch.push_front(n);
         }
       if (verbose > 3)
         obatched(clog) << "fdcache interned a=" << a << " b=" << b
@@ -1248,10 +1286,10 @@ public:
       {
         inc_metric("fdcache_op_count","op","emerg-flush");
         obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
-        this->limit(0, 0); // emergency flush
+        this->limit(0, 0, 0, 0); // emergency flush
       }
     else if (front_p)
-      this->limit(max_fds, max_mbs); // age cache if required
+      this->limit(max_fds, max_mbs, max_prefetch_fds,
max_prefetch_mbs); // age cache if required
   }

   int lookup(const string& a, const string& b)
@@ -1266,21 +1304,32 @@ public:
               fdcache_entry n = *i;
               lru.erase(i); // invalidates i, so no more iteration!
               lru.push_front(n);
-              inc_metric("fdcache_op_count","op","requeue_front");
+              inc_metric("fdcache_op_count","op","lru_requeue_front");
+              fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if
dup() fails; looks like cache miss
+              break;
+            }
+        }
+      for ( auto i = prefetch.begin(); fd == -1 && i < prefetch.end(); ++i)
+        {
+          if (i->archive == a && i->entry == b)
+            { // found it; take the entry from the prefetch deque to
the lru deque, since it has now been accessed.
+              fdcache_entry n = *i;
+              prefetch.erase(i);
+              lru.push_front(n);
+              inc_metric("fdcache_op_count","op","prefetch_access");
               fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if
dup() fails; looks like cache miss
               break;
             }
         }
     }
-
     if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
       {
         inc_metric("fdcache_op_count","op","emerg-flush");
         obatched(clog) << "fdcache emergency flush for filling tmpdir";
-        this->limit(0, 0); // emergency flush
+        this->limit(0, 0, 0, 0); // emergency flush
       }
     else if (fd >= 0)
-      this->limit(max_fds, max_mbs); // age cache if required
+      this->limit(max_fds, max_mbs, max_prefetch_fds,
max_prefetch_mbs); // age cache if required

     return fd;
   }
@@ -1292,7 +1341,15 @@ public:
       {
         if (i->archive == a && i->entry == b)
           {
-            inc_metric("fdcache_op_count","op","probe_hit");
+            inc_metric("fdcache_op_count","op","lru_probe_hit");
+            return true;
+          }
+      }
+    for (auto i = prefetch.begin(); i < prefetch.end(); i++)
+      {
+        if (i->archive == a && i->entry == b)
+          {
+            inc_metric("fdcache_op_count","op","prefetch_probe_hit");
             return true;
           }
       }
@@ -1306,19 +1363,30 @@ public:
     for (auto i = lru.begin(); i < lru.end(); i++)
       {
         if (i->archive == a && i->entry == b)
-          { // found it; move it to head of lru
+          { // found it; erase it from lru
             fdcache_entry n = *i;
             lru.erase(i); // invalidates i, so no more iteration!
-            inc_metric("fdcache_op_count","op","clear");
+            inc_metric("fdcache_op_count","op","lru_clear");
             unlink (n.fd.c_str());
             set_metrics();
             return;
           }
       }
+    for (auto i = prefetch.begin(); i < prefetch.end(); i++)
+      {
+        if (i->archive == a && i->entry == b)
+          { // found it; erase it from lru
+            fdcache_entry n = *i;
+            prefetch.erase(i); // invalidates i, so no more iteration!
+            inc_metric("fdcache_op_count","op","prefetch_clear");
+            unlink (n.fd.c_str());
+            set_metrics();
+            return;
+          }
+      }
   }

-
-  void limit(long maxfds, long maxmbs, bool metrics_p = true)
+  void limit(long maxfds, long maxmbs, long maxprefetchfds, long
maxprefetchmbs , bool metrics_p = true)
   {
     if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
       obatched(clog) << "fdcache limited to maxfds=" << maxfds << "
maxmbs=" << maxmbs << endl;
@@ -1326,7 +1394,8 @@ public:
     unique_lock<mutex> lock(fdcache_lock);
     this->max_fds = maxfds;
     this->max_mbs = maxmbs;
-
+    this->max_prefetch_fds = maxprefetchfds;
+    this->max_prefetch_mbs = maxprefetchmbs;
     long total_fd = 0;
     double total_mb = 0.0;
     for (auto i = lru.begin(); i < lru.end(); i++)
@@ -1334,7 +1403,7 @@ public:
         // accumulate totals from most recently used one going backward
         total_fd ++;
         total_mb += i->fd_size_mb;
-        if (total_fd > max_fds || total_mb > max_mbs)
+        if (total_fd > this->max_fds || total_mb > this->max_mbs)
           {
             // found the cut here point!

@@ -1344,7 +1413,7 @@ public:
                   obatched(clog) << "fdcache evicted a=" <<
j->archive << " b=" << j->entry
                                  << " fd=" << j->fd << " mb=" <<
j->fd_size_mb << endl;
                 if (metrics_p)
-                  inc_metric("fdcache_op_count","op","evict");
+                  inc_metric("fdcache_op_count","op","lru_evict");
                 unlink (j->fd.c_str());
               }

@@ -1352,6 +1421,29 @@ public:
             break;
           }
       }
+    total_fd = 0;
+    total_mb = 0.0;
+    for(auto i = prefetch.begin(); i < prefetch.end(); i++){
+      // accumulate totals from most recently used one going backward
+        total_fd ++;
+        total_mb += i->fd_size_mb;
+        if (total_fd > this->max_prefetch_fds || total_mb >
this->max_prefetch_mbs)
+          {
+            // found the cut here point!
+            for (auto j = i; j < prefetch.end(); j++) // close all
the fds from here on in
+              {
+                if (verbose > 3)
+                  obatched(clog) << "fdcache evicted from prefetch
a=" << j->archive << " b=" << j->entry
+                                 << " fd=" << j->fd << " mb=" <<
j->fd_size_mb << endl;
+                if (metrics_p)
+                  inc_metric("fdcache_op_count","op","prefetch_evict");
+                unlink (j->fd.c_str());
+              }
+
+            prefetch.erase(i, prefetch.end()); // erase the nodes generally
+            break;
+          }
+    }
     if (metrics_p) set_metrics();
   }

@@ -1360,7 +1452,7 @@ public:
   {
     // unlink any fdcache entries in $TMPDIR
     // don't update metrics; those globals may be already destroyed
-    limit(0, 0, false);
+    limit(0, 0, 0, 0, false);
   }
 };
 static libarchive_fdcache fdcache;
@@ -1547,7 +1639,7 @@ handle_buildid_r_match (bool internal_req_p,
           // responsible for unlinking it later.
           fdcache.intern(b_source0, fn,
                          tmppath, archive_entry_size(e),
-                         false); // prefetched ones go to back of lru
+                         false); // prefetched ones go to the prefetch cache
           prefetch_count --;
           close (fd); // we're not saving this fd to make a mhd-response from!
           continue;
@@ -3293,8 +3385,8 @@ void groom()
   sqlite3_db_release_memory(dbq); // ... for both connections
   debuginfod_pool_groom(); // and release any debuginfod_client
objects we've been holding onto

-  fdcache.limit(0,0); // release the fdcache contents
-  fdcache.limit(fdcache_fds,fdcache_mbs); // restore status quo parameters
+  fdcache.limit(0,0,0,0); // release the fdcache contents
+  fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds,
fdcache_prefetch_mbs); // restore status quo parameters

   clock_gettime (CLOCK_MONOTONIC, &ts_end);
   double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec
- ts_start.tv_nsec)/1.e9;
@@ -3456,7 +3548,7 @@ main (int argc, char *argv[])
   if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
     obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;

-  fdcache.limit(fdcache_fds, fdcache_mbs);
+  fdcache.limit(fdcache_fds, fdcache_mbs, fdcache_prefetch_fds,
fdcache_prefetch_mbs);

   (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it
incidentally, ignore
   (void) signal (SIGINT, signal_handler); // ^C
@@ -3600,6 +3692,9 @@ main (int argc, char *argv[])
   obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
   obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
   obatched(clog) << "groom time " << groom_s << endl;
+  obatched(clog) << "prefetch fds " << fdcache_prefetch_fds << endl;
+  obatched(clog) << "prefetch mbs " << fdcache_prefetch_mbs << endl;
+
   if (scan_archives.size()>0)
     {
       obatched ob(clog);
diff --git a/doc/debuginfod.8 b/doc/debuginfod.8
index 1ba42cf6..8945eb9b 100644
--- a/doc/debuginfod.8
+++ b/doc/debuginfod.8
@@ -212,6 +212,16 @@ $TMPDIR or \fB/tmp\fP filesystem.  This is
because that is where the
 most recently used extracted files are kept.  Grooming cleans this
 cache.

+.TP
+.B "\-\-fdcache\-\-prefetch\-fds=NUM"  "\-\-fdcache\-\-prefetch\-mbs=MB"
+Configure how many file descriptors (fds) and megabytes (mbs) are
+allocated to the prefetch portion of the fdcache. If unspecified,
+values of \fB\-\-prefetch\-fds\fP and \fB\-\-prefetch\-mbs\fP depend
+on concurrency of the system and on the available disk space on
+the $TMPDIR. Allocating more to the prefetch cache will improve
+performance in environments where different parts of several large
+archives are being accessed.
+
 .TP
 .B "\-\-fdcache\-mintmp=NUM"
 Configure a disk space threshold for emergency flushing of the cache.
diff --git a/tests/ChangeLog b/tests/ChangeLog
index d8fa97fa..94201959 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,10 @@
+2021-06-28  Noah Sanci <nsanci@redhat.com>
+
+ PR25978
+ * run-debuginfod-find.sh: Test to ensure options
+ fdcache-prefetch-fds/mbs are set. Check that inc_metric works for lru
+ and prefetch cache metrics.
+
 2021-06-16  Frank Ch. Eigler <fche@redhat.com>

  * run-debuginfod-find.sh: Fix intermittent groom/stale failure,
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index 456dc2f8..47e33424 100755
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -119,12 +119,15 @@ wait_ready()
   fi
 }

+FDCACHE_FDS=50
+FDCACHE_MBS=190
+PREFETCH_FDS=10
+PREFETCH_MBS=120
 # create a bogus .rpm file to evoke a metric-visible error
 # Use a cyclic symlink instead of chmod 000 to make sure even root
 # would see an error (running the testsuite under root is NOT encouraged).
 ln -s R/nothing.rpm R/nothing.rpm
-
-env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=
${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p
$PORT1 -t0 -g0 --fdcache-fds 1 --fdcache-mbs 2 --fdcache-mintmp 0 -Z
.tar.xz -Z .tar.bz2=bzcat -v R F Z L > vlog$PORT1 2>&1 &
+env LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS=
${abs_builddir}/../debuginfod/debuginfod $VERBOSE -F -R -d $DB -p
$PORT1 -t0 -g0 --fdcache-mbs=$FDCACHE_MBS --fdcache-fds=$FDCACHE_FDS
--fdcache-prefetch-mbs=$PREFETCH_MBS
--fdcache-prefetch-fds=$PREFETCH_FDS --fdcache-mintmp 0 -Z .tar.xz -Z
.tar.bz2=bzcat -v R F Z L > vlog$PORT1 2>&1 &
 PID1=$!
 tempfiles vlog$PORT1
 errfiles vlog$PORT1
@@ -470,6 +473,25 @@ archive_test
f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/h

 egrep '(libc.error.*rhel7)|(bc1febfd03ca)|(f0aa15b8aba)' vlog$PORT1

+########################################################################
+## PR25978
+# Ensure that the fdcache options are working.
+grep "prefetch fds" vlog$PORT1
+grep "prefetch mbs" vlog$PORT1
+grep "fdcache fds" vlog$PORT1
+grep "fdcache mbs" vlog$PORT1
+
+wait_ready $PORT1 'fdcache_op_count{op="lru_enqueue"}' 28
+wait_ready $PORT1 'fdcache_op_count{op="lru_evict"}' 24
+wait_ready $PORT1 'fdcache_op_count{op="lru_probe_hit"}' 6
+wait_ready $PORT1 'fdcache_op_count{op="lru_requeue_front"}' 11
+wait_ready $PORT1 'fdcache_op_count{op="lru_probe_miss"}' 0
+wait_ready $PORT1 'fdcache_prefetch_bytes' 0
+wait_ready $PORT1 'fdcache_prefetch_count' 0
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_access"}' 4
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_enqueue"}' 4
+wait_ready $PORT1 'fdcache_op_count{op="prefetch_evict"}' 0
+
 ########################################################################

 # Federation mode

fpaste: https://paste.centos.org/view/raw/c798096f



More information about the Elfutils-devel mailing list