cache.cxx

   1 // systemtap cache manager
   2 // Copyright (C) 2006-2009 Red Hat Inc.
   3 //
   4 // This file is part of systemtap, and is free software.  You can
   5 // redistribute it and/or modify it under the terms of the GNU General
   6 // Public License (GPL); either version 2, or (at your option) any
   7 // later version.
   8
   9 #include "config.h"
  10 #include "session.h"
  11 #include "cache.h"
  12 #include "util.h"
  13 #include "stap-probe.h"
  14 #include <cerrno>
  15 #include <string>
  16 #include <fstream>
  17 #include <cstring>
  18 #include <cassert>
  19 #include <sstream>
  20 #include <vector>
  21
  22 extern "C" {
  23 #include <sys/types.h>
  24 #include <sys/stat.h>
  25 #include <fcntl.h>
  26 #include <glob.h>
  27 #include <regex.h>
  28 #include <utime.h>
  29 #include <sys/time.h>
  30 #include <unistd.h>
  31 }
  32
  33 using namespace std;
  34
  35
  36 #define SYSTEMTAP_CACHE_MAX_FILENAME "cache_mb_limit"
  37 #define SYSTEMTAP_CACHE_DEFAULT_MB 256
  38 #define SYSTEMTAP_CACHE_CLEAN_INTERVAL_FILENAME "cache_clean_interval_s"
  39 #define SYSTEMTAP_CACHE_CLEAN_DEFAULT_INTERVAL_S 300
  40
  41 struct cache_ent_info {
  42   vector<string> paths;
  43   off_t size; // sum across all paths
  44   time_t mtime; // newest of all paths
  45
  46   cache_ent_info(const vector<string>& paths);
  47   bool operator<(const struct cache_ent_info& other) const;
  48   void unlink() const;
  49 };
  50
  51
  52 void
  53 add_stapconf_to_cache(systemtap_session& s)
  54 {
  55   bool verbose = s.verbose > 1;
  56
  57   string stapconf_src_path = s.tmpdir + "/" + s.stapconf_name;
  58   if (!copy_file(stapconf_src_path, s.stapconf_path, verbose))
  59     {
  60       // NB: this is not so severe as to prevent reuse of the .ko
  61       // already copied.
  62       //
  63       // s.use_script_cache = false;
  64       // return;
  65     }
  66 }
  67
  68
  69 void
  70 add_script_to_cache(systemtap_session& s)
  71 {
  72   bool verbose = s.verbose > 1;
  73
  74   // PR10543: clean the cache *before* we try putting something new into it.
  75   // We don't want to risk having the brand new contents being erased again.
  76   clean_cache(s);
  77
  78   string module_src_path = s.tmpdir + "/" + s.module_filename();
  79   PROBE2(stap, cache__add__module, module_src_path.c_str(), s.hash_path.c_str());
  80   if (!copy_file(module_src_path, s.hash_path, verbose))
  81     {
  82       s.use_script_cache = false;
  83       return;
  84     }
  85   // Copy the signature file, if any. It is not an error if this fails.
  86   if (file_exists (module_src_path + ".sgn"))
  87     copy_file(module_src_path + ".sgn", s.hash_path + ".sgn", verbose);
  88
  89   string c_dest_path = s.hash_path;
  90   if (endswith(c_dest_path, ".ko") || endswith(c_dest_path, ".so"))
  91     c_dest_path.resize(c_dest_path.size() - 3);
  92   c_dest_path += ".c";
  93
  94   PROBE2(stap, cache__add__source, s.translated_source.c_str(), c_dest_path.c_str());
  95   if (!copy_file(s.translated_source, c_dest_path, verbose))
  96     {
  97       // NB: this is not so severe as to prevent reuse of the .ko
  98       // already copied.
  99       //
 100       // s.use_script_cache = false;
 101     }
 102 }
 103
 104
 105 bool
 106 get_stapconf_from_cache(systemtap_session& s)
 107 {
 108   if (s.poison_cache)
 109     return false;
 110
 111   string stapconf_dest_path = s.tmpdir + "/" + s.stapconf_name;
 112   int fd_stapconf;
 113
 114   // See if stapconf exists
 115   fd_stapconf = open(s.stapconf_path.c_str(), O_RDONLY);
 116   if (fd_stapconf == -1)
 117     {
 118       // It isn't in cache.
 119       return false;
 120     }
 121
 122   // Copy the stapconf header file to the destination
 123   if (!get_file_size(fd_stapconf) ||
 124       !copy_file(s.stapconf_path, stapconf_dest_path))
 125     {
 126       close(fd_stapconf);
 127       return false;
 128     }
 129
 130   // We're done with this file handle.
 131   close(fd_stapconf);
 132
 133   if (s.verbose > 1)
 134     clog << _("Pass 4: using cached ") << s.stapconf_path << endl;
 135
 136   return true;
 137 }
 138
 139
 140 bool
 141 get_script_from_cache(systemtap_session& s)
 142 {
 143   if (s.poison_cache)
 144     return false;
 145
 146   string module_dest_path = s.tmpdir + "/" + s.module_filename();
 147   string c_src_path = s.hash_path;
 148   int fd_module, fd_c;
 149
 150   if (endswith(c_src_path, ".ko") || endswith(c_src_path, ".so"))
 151     c_src_path.resize(c_src_path.size() - 3);
 152   c_src_path += ".c";
 153
 154   // See if module exists
 155   fd_module = open(s.hash_path.c_str(), O_RDONLY);
 156   if (fd_module == -1)
 157     {
 158       // It isn't in cache.
 159       return false;
 160     }
 161
 162   // See if C file exists.
 163   fd_c = open(c_src_path.c_str(), O_RDONLY);
 164   if (fd_c == -1)
 165     {
 166       // The module is there, but the C file isn't.  Cleanup and
 167       // return.
 168       close(fd_module);
 169       unlink(s.hash_path.c_str());
 170       return false;
 171     }
 172
 173   // Check that the files aren't empty, and then
 174   // copy the cached C file to the destination
 175   if (!get_file_size(fd_module) || !get_file_size(fd_c) ||
 176       !copy_file(c_src_path, s.translated_source))
 177     {
 178       close(fd_module);
 179       close(fd_c);
 180       return false;
 181     }
 182
 183   // Copy the cached module to the destination (if needed)
 184   if (s.last_pass != 3)
 185     {
 186       if (!copy_file(s.hash_path, module_dest_path))
 187         {
 188           unlink(c_src_path.c_str());
 189           close(fd_module);
 190           close(fd_c);
 191           return false;
 192         }
 193       // Copy the module signature file, if any.
 194       // It is not an error if this fails.
 195       if (file_exists (s.hash_path + ".sgn"))
 196         copy_file(s.hash_path + ".sgn", module_dest_path + ".sgn");
 197     }
 198
 199   // We're done with these file handles.
 200   close(fd_module);
 201   close(fd_c);
 202
 203   // To preserve semantics (since this will happen if we're not
 204   // caching), display the C source if the last pass is 3.
 205   if (s.last_pass == 3)
 206     {
 207       ifstream i (s.translated_source.c_str());
 208       cout << i.rdbuf();
 209     }
 210   // And similarly, display probe module name for -p4.
 211   if (s.last_pass == 4)
 212     cout << s.hash_path << endl;
 213
 214   // If everything worked, tell the user.  We need to do this here,
 215   // since if copying the cached C file works, but copying the cached
 216   // module fails, we remove the cached C file and let the C file get
 217   // regenerated.
 218   // NB: don't use s.verbose here, since we're still in pass-2,
 219   // i.e., s.verbose = s.perpass_verbose[1].
 220   if (s.perpass_verbose[2])
 221     clog << _("Pass 3: using cached ") << c_src_path << endl;
 222   if (s.perpass_verbose[3] && s.last_pass != 3)
 223     clog << _("Pass 4: using cached ") << s.hash_path << endl;
 224
 225   PROBE2(stap, cache__get, c_src_path.c_str(), s.hash_path.c_str());
 226
 227   return true;
 228 }
 229
 230
 231 void
 232 clean_cache(systemtap_session& s)
 233 {
 234   if (s.cache_path != "")
 235     {
 236       /* Get cache size limit from file in the stap cache dir */
 237       string cache_max_filename = s.cache_path + "/";
 238       cache_max_filename += SYSTEMTAP_CACHE_MAX_FILENAME;
 239       ifstream cache_max_file(cache_max_filename.c_str(), ios::in);
 240       unsigned long cache_mb_max;
 241
 242       if (cache_max_file.is_open())
 243         {
 244           cache_max_file >> cache_mb_max;
 245           cache_max_file.close();
 246         }
 247       else
 248         {
 249           //file doesnt exist, create a default size
 250           ofstream default_cache_max(cache_max_filename.c_str(), ios::out);
 251           default_cache_max << SYSTEMTAP_CACHE_DEFAULT_MB << endl;
 252           cache_mb_max = SYSTEMTAP_CACHE_DEFAULT_MB;
 253
 254           if (s.verbose > 1)
 255             clog << _F("Cache limit file %s/%s missing, creating default.",
 256                        s.cache_path.c_str(), SYSTEMTAP_CACHE_MAX_FILENAME) << endl;
 257         }
 258
 259       /* Get cache clean interval from file in the stap cache dir */
 260       string cache_clean_interval_filename = s.cache_path + "/";
 261       cache_clean_interval_filename += SYSTEMTAP_CACHE_CLEAN_INTERVAL_FILENAME;
 262       ifstream cache_clean_interval_file(cache_clean_interval_filename.c_str(), ios::in);
 263       unsigned long cache_clean_interval;
 264
 265       if (cache_clean_interval_file.is_open())
 266         {
 267           cache_clean_interval_file >> cache_clean_interval;
 268           cache_clean_interval_file.close();
 269         }
 270       else
 271         {
 272           //file doesnt exist, create a default interval
 273           ofstream default_cache_clean_interval(cache_clean_interval_filename.c_str(), ios::out);
 274           default_cache_clean_interval << SYSTEMTAP_CACHE_CLEAN_DEFAULT_INTERVAL_S << endl;
 275           cache_clean_interval = SYSTEMTAP_CACHE_CLEAN_DEFAULT_INTERVAL_S;
 276
 277           if (s.verbose > 1)
 278             clog << _F("Cache clean interval file %s missing, creating default.",
 279                        cache_clean_interval_filename.c_str())<< endl;
 280         }
 281
 282       /* Check the cache cleaning interval */
 283       struct stat sb;
 284       if(stat(cache_clean_interval_filename.c_str(), &sb) < 0)
 285         {
 286           const char* e = strerror (errno);
 287           cerr << _F("clean_cache stat error: %s", e) << endl;
 288           return;
 289         }
 290
 291       struct timeval current_time;
 292       gettimeofday(&current_time, NULL);
 293       if(difftime(current_time.tv_sec, sb.st_mtime) < cache_clean_interval)
 294         {
 295           //interval not passed, don't continue
 296           if (s.verbose > 1)
 297             clog << _F("Cache cleaning skipped, interval not reached %lu s / %lu s.",
 298                        (current_time.tv_sec-sb.st_mtime), cache_clean_interval)  << endl;
 299           return;
 300         }
 301       else
 302         {
 303           //interval reached, continue
 304           if (s.verbose > 1)
 305             clog << _F("Cleaning cache, interval reached %lu s > %lu s.",
 306                        (current_time.tv_sec-sb.st_mtime), cache_clean_interval)  << endl;
 307         }
 308
 309       // glob for all files that look like hashes
 310       glob_t cache_glob;
 311       ostringstream glob_pattern;
 312       glob_pattern << s.cache_path << "/*/*";
 313       for (unsigned int i = 0; i < 32; i++)
 314         glob_pattern << "[[:xdigit:]]";
 315       glob_pattern << "*";
 316       int rc = glob(glob_pattern.str().c_str(), 0, NULL, &cache_glob);
 317       if (rc) {
 318         cerr << _F("clean_cache glob error rc=%d", rc) << endl;
 319         return;
 320       }
 321
 322       regex_t hash_len_re;
 323       rc = regcomp (&hash_len_re, "([[:xdigit:]]{32}_[[:digit:]]+)", REG_EXTENDED);
 324       if (rc) {
 325         cerr << _F("clean_cache regcomp error rc=%d", rc) << endl;
 326         globfree(&cache_glob);
 327         return;
 328       }
 329
 330       // group all files with the same HASH_LEN
 331       map<string, vector<string> > cache_groups;
 332       for (size_t i = 0; i < cache_glob.gl_pathc; i++)
 333         {
 334           const char* path = cache_glob.gl_pathv[i];
 335           regmatch_t hash_len;
 336           rc = regexec(&hash_len_re, path, 1, &hash_len, 0);
 337           if (rc || hash_len.rm_so == -1 || hash_len.rm_eo == -1)
 338             cache_groups[path].push_back(path); // ungrouped
 339           else
 340             cache_groups[string(path + hash_len.rm_so,
 341                                 hash_len.rm_eo - hash_len.rm_so)]
 342               .push_back(path);
 343         }
 344       regfree(&hash_len_re);
 345       globfree(&cache_glob);
 346
 347
 348       // create each cache entry and accumulate the sum
 349       off_t cache_size_b = 0;
 350       set<cache_ent_info> cache_contents;
 351       for (map<string, vector<string> >::const_iterator it = cache_groups.begin();
 352            it != cache_groups.end(); ++it)
 353         {
 354           cache_ent_info cur_info(it->second);
 355           if (cache_contents.insert(cur_info).second)
 356             cache_size_b += cur_info.size;
 357         }
 358
 359       unsigned long r_cache_size = cache_size_b;
 360       vector<const cache_ent_info*> removed;
 361
 362       //unlink .ko and .c until the cache size is under the limit
 363       for (set<cache_ent_info>::iterator i = cache_contents.begin();
 364            i != cache_contents.end(); ++i)
 365         {
 366           if (r_cache_size < cache_mb_max * 1024 * 1024) //convert cache_mb_max to bytes
 367             break;
 368
 369           //remove this (*i) cache_entry, add to removed list
 370           for (size_t j = 0; j < i->paths.size(); ++j)
 371             PROBE1(stap, cache__clean, i->paths[j].c_str());
 372           i->unlink();
 373           r_cache_size -= i->size;
 374           removed.push_back(&*i);
 375         }
 376
 377       if (s.verbose > 1 && !removed.empty())
 378         {
 379           clog << _("Cache cleaning successful, removed entries: ") << endl;
 380           for (size_t i = 0; i < removed.size(); ++i)
 381             for (size_t j = 0; j < removed[i]->paths.size(); ++j)
 382               clog << "  " << removed[i]->paths[j] << endl;
 383         }
 384
 385       if(utime(cache_clean_interval_filename.c_str(), NULL)<0)
 386         {
 387           const char* e = strerror (errno);
 388           cerr << _F("clean_cache utime error: %s", e) << endl;
 389           return;
 390         }
 391     }
 392   else
 393     {
 394       if (s.verbose > 1)
 395         clog << _("Cache cleaning skipped, no cache path.") << endl;
 396     }
 397 }
 398
 399
 400 cache_ent_info::cache_ent_info(const vector<string>& paths):
 401   paths(paths), size(0), mtime(0)
 402 {
 403   struct stat file_info;
 404   for (size_t i = 0; i < paths.size(); ++i)
 405     if (stat(paths[i].c_str(), &file_info) == 0)
 406       {
 407         size += file_info.st_size;
 408         if (file_info.st_mtime > mtime)
 409           mtime = file_info.st_mtime;
 410       }
 411 }
 412
 413
 414 // The ordering here determines the order that
 415 // files will be removed from the cache.
 416 bool
 417 cache_ent_info::operator<(const struct cache_ent_info& other) const
 418 {
 419   if (mtime != other.mtime)
 420     return mtime < other.mtime;
 421   if (size != other.size)
 422     return size < other.size;
 423   if (paths.size() != other.paths.size())
 424     return paths.size() < other.paths.size();
 425   for (size_t i = 0; i < paths.size(); ++i)
 426     if (paths[i] != other.paths[i])
 427       return paths[i] < other.paths[i];
 428   return false;
 429 }
 430
 431
 432 void
 433 cache_ent_info::unlink() const
 434 {
 435   for (size_t i = 0; i < paths.size(); ++i)
 436     ::unlink(paths[i].c_str());
 437 }
 438
 439
 440 /* vim: set sw=2 ts=8 cino=>4,n-2,{2,^-2,t0,(0,u0,w1,M1 : */