remote.cxx

   1 // systemtap remote execution
   2 // Copyright (C) 2010-2015 Red Hat Inc.
   3 //
   4 // This file is part of systemtap, and is free software.  You can
   5 // redistribute it and/or modify it under the terms of the GNU General
   6 // Public License (GPL); either version 2, or (at your option) any
   7 // later version.
   8
   9 #include "config.h"
  10
  11 extern "C" {
  12 #include <fcntl.h>
  13 #include <poll.h>
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <unistd.h>
  17 #include <sys/socket.h>
  18 #include <sys/un.h>
  19 }
  20
  21 #include <cstdio>
  22 #include <iomanip>
  23 #include <memory>
  24 #include <stdexcept>
  25 #include <sstream>
  26 #include <string>
  27 #include <vector>
  28
  29 #include "buildrun.h"
  30 #include "remote.h"
  31 #include "util.h"
  32
  33 using namespace std;
  34
  35 // Decode URIs as per RFC 3986, though not bothering to be strict
  36 class uri_decoder {
  37   public:
  38     const string uri;
  39     string scheme, authority, path, query, fragment;
  40     bool has_authority, has_query, has_fragment;
  41
  42     uri_decoder(const string& uri):
  43       uri(uri), has_authority(false), has_query(false), has_fragment(false)
  44     {
  45       const string re =
  46         "^([^:]+):(//[^/?#]*)?([^?#]*)(\\?[^#]*)?(#.*)?$";
  47
  48       vector<string> matches;
  49       if (regexp_match(uri, re, matches) != 0)
  50         throw runtime_error(_F("string doesn't appear to be a URI: %s", uri.c_str()));
  51
  52       scheme = matches[1];
  53
  54       if (!matches[2].empty())
  55         {
  56           has_authority = true;
  57           authority = matches[2].substr(2);
  58         }
  59
  60       path = matches[3];
  61
  62       if (!matches[4].empty())
  63         {
  64           has_query = true;
  65           query = matches[4].substr(1);
  66         }
  67
  68       if (!matches[5].empty())
  69         {
  70           has_fragment = true;
  71           fragment = matches[5].substr(1);
  72         }
  73     }
  74 };
  75
  76
  77 // loopback target for running locally
  78 class direct : public remote {
  79   private:
  80     pid_t child;
  81     vector<string> args;
  82     direct(systemtap_session& s): remote(s), child(0) {}
  83
  84     int start()
  85       {
  86         args = make_run_command(*s);
  87         if (! staprun_r_arg.empty()) // PR13354
  88           {
  89             if (s->runtime_mode == systemtap_session::dyninst_runtime)
  90               args.insert(args.end(), {"_stp_dyninst_remote=" + staprun_r_arg});
  91             else if (s->runtime_mode == systemtap_session::kernel_runtime)
  92               args.insert(args.end(), { "-r", staprun_r_arg });
  93             // leave args empty for bpf_runtime
  94           }
  95
  96         pid_t pid = stap_spawn (s->verbose, args);
  97         if (pid <= 0)
  98           return 1;
  99         child = pid;
 100         return 0;
 101       }
 102
 103     int finish()
 104       {
 105         if (child <= 0)
 106           return 1;
 107
 108         int ret = stap_waitpid(s->verbose, child);
 109         if (ret > 128)
 110           s->print_warning(_F("%s exited with signal: %d (%s)",
 111                               args.front().c_str(), ret - 128,
 112                               strsignal(ret - 128)));
 113         else if (ret > 0)
 114           s->print_warning(_F("%s exited with status: %d",
 115                               args.front().c_str(), ret));
 116         child = 0;
 117         return ret;
 118       }
 119
 120   public:
 121     friend class remote;
 122
 123     virtual ~direct() { finish(); }
 124 };
 125
 126
 127 class stapsh : public remote {
 128   private:
 129     int interrupts_sent;
 130     int fdin, fdout;
 131     FILE *IN, *OUT;
 132     string remote_version;
 133     size_t data_size;
 134     string target_stream;
 135
 136     enum {
 137       STAPSH_READY, // ready to receive next command
 138       STAPSH_DATA   // currently printing data from a 'data' command
 139     } stream_state;
 140
 141     virtual void prepare_poll(vector<pollfd>& fds)
 142       {
 143         if (fdout >= 0 && OUT)
 144           {
 145             pollfd p = { fdout, POLLIN, 0 };
 146             fds.push_back(p);
 147           }
 148
 149         // need to send a signal?
 150         if (fdin >= 0 && IN && interrupts_sent < pending_interrupts)
 151           {
 152             pollfd p = { fdin, POLLOUT, 0 };
 153             fds.push_back(p);
 154           }
 155       }
 156
 157     virtual void handle_poll(vector<pollfd>& fds)
 158       {
 159         for (unsigned i=0; i < fds.size(); ++i)
 160           if (fds[i].fd == fdin || fds[i].fd == fdout)
 161             {
 162               bool err = false;
 163
 164               // need to send a signal?
 165               if (fds[i].revents & POLLOUT && IN &&
 166                   interrupts_sent < pending_interrupts)
 167                 {
 168                   if (send_command("quit\n") == 0)
 169                     ++interrupts_sent;
 170                   else
 171                     err = true;
 172                 }
 173
 174               // have data to read?
 175               if (fds[i].revents & POLLIN && OUT)
 176                 {
 177                   // if stapsh doesn't support commands, then just dump everything
 178                   if (!vector_has(options, string("data")))
 179                     {
 180                       // NB: we could splice here, but we don't know how much
 181                       // data is available for reading. One way could be to
 182                       // splice in small chunks and poll() fdout to check if
 183                       // there's more.
 184                       char buf[4096];
 185                       size_t bytes_read;
 186                       while ((bytes_read = fread(buf, 1, sizeof(buf), OUT)) > 0)
 187                         printout(buf, bytes_read);
 188                       if (errno != EAGAIN)
 189                         err = true;
 190                     }
 191                   else // we expect commands (just data for now)
 192                     {
 193                       // When the 'data' option is turned on, all outputs from
 194                       // staprun are first prepended with a line formatted as
 195                       // "data <stream> <size>\n", where <stream> is either
 196                       // "stdout" or "stderr", and <size> is the size of the
 197                       // data. Also, the line "quit\n" is sent from stapsh right
 198                       // before it exits.
 199
 200                       if (stream_state == STAPSH_READY)
 201                         {
 202                           char cmdbuf[1024];
 203                           char *rc;
 204                           while ((rc = fgets(cmdbuf, sizeof(cmdbuf), OUT)) != NULL)
 205                             {
 206                               // check if it's a debug output from stapsh itself
 207                               string line = string(cmdbuf);
 208                               if (startswith(line, "stapsh:"))
 209                                 clog << line;
 210                               else if (startswith(line, "quit\n"))
 211                                   // && vector_has(options, string("data")))
 212                                   // uncomment above if another command becomes
 213                                   // possible
 214                                 {
 215                                   err = true; // close connection
 216                                   break;
 217                                 }
 218                               else if (startswith(line, "data"))
 219                                   // && vector_has(options, string("data")))
 220                                   // uncomment above if another command becomes
 221                                   // possible
 222                                 {
 223                                   vector<string> cmd;
 224                                   tokenize(line, cmd, " \t\r\n");
 225                                   if (!is_valid_data_cmd(cmd))
 226                                     {
 227                                       clog << _("invalid data command from stapsh") << endl;
 228                                       clog << _("received: ") << line;
 229                                       err = true;
 230                                     }
 231                                   else
 232                                     {
 233                                       target_stream = cmd[1];
 234                                       data_size = lex_cast<size_t>(cmd[2]);
 235                                       stream_state = STAPSH_DATA;
 236                                     }
 237                                   break;
 238                                 }
 239                             }
 240                           if (rc == NULL && errno != EAGAIN)
 241                             err = true;
 242                         }
 243
 244                       if (stream_state == STAPSH_DATA)
 245                         {
 246                           if (data_size != 0)
 247                             {
 248                               // keep reading from OUT until either dry or data_size bytes done
 249                               char buf[4096];
 250                               size_t max_read = min<size_t>(sizeof(buf), data_size);
 251                               size_t bytes_read = 0;
 252                               while (max_read > 0
 253                                   && (bytes_read = fread(buf, 1, max_read, OUT)) > 0)
 254                                 {
 255                                   printout(buf, bytes_read);
 256                                   data_size -= bytes_read;
 257                                   max_read = min<size_t>(sizeof(buf), data_size);
 258                                 }
 259                               if (bytes_read == 0 && errno != EAGAIN)
 260                                 err = true;
 261                             }
 262                           if (data_size == 0)
 263                             stream_state = STAPSH_READY;
 264                         }
 265                     }
 266                 }
 267
 268               // any errors?
 269               if (err || fds[i].revents & ~(POLLIN|POLLOUT))
 270                 close();
 271             }
 272       }
 273
 274     string get_reply()
 275       {
 276         // Some schemes like unix may have stdout and stderr mushed together.
 277         // There shouldn't be anything except dbug messages on stderr before we
 278         // actually start running, and there's no get_reply after that.  So
 279         // we'll just loop and skip those that start with "stapsh:".
 280         char reply[4096];
 281         while (fgets(reply, sizeof(reply), OUT))
 282           {
 283             if (!startswith(reply, "stapsh:"))
 284               return reply;
 285
 286             // if data is not prefixed, we will have no way to distinguish
 287             // between stdout and stderr during staprun runtime, so we might as
 288             // well print to stdout now as well to be more consistent
 289             if (vector_has(options, string("data")))
 290               clog << reply; // must be stderr since only replies go to stdout
 291             else
 292               cout << reply; // output to stdout to be more consistent with later
 293           }
 294
 295         // Reached EOF, nothing to reply...
 296         return "";
 297       }
 298
 299     int send_command(const string& cmd)
 300       {
 301         if (!IN)
 302           return 2;
 303         if (fputs(cmd.c_str(), IN) < 0 ||
 304             fflush(IN) != 0)
 305           return 1;
 306         return 0;
 307       }
 308
 309     int send_file(const string& filename, const string& dest)
 310       {
 311         int rc = 0;
 312         FILE* f = fopen(filename.c_str(), "r");
 313         if (!f)
 314           return 1;
 315
 316         struct stat fs;
 317         rc = fstat(fileno(f), &fs);
 318         if (!rc)
 319           {
 320             ostringstream cmd;
 321             cmd << "file " << fs.st_size << " " << dest << "\n";
 322             rc = send_command(cmd.str());
 323           }
 324
 325         off_t i = 0;
 326         while (!rc && i < fs.st_size)
 327           {
 328             char buf[4096];
 329             size_t r = sizeof(buf);
 330             if (fs.st_size - i < (off_t)r)
 331               r = fs.st_size - i;
 332             r = fread(buf, 1, r, f);
 333             if (r == 0)
 334               rc = 1;
 335             else
 336               {
 337                 size_t w = fwrite(buf, 1, r, IN);
 338                 if (w != r)
 339                   rc = 1;
 340                 else
 341                   i += w;
 342               }
 343           }
 344         if (!rc)
 345           rc = fflush(IN);
 346
 347         fclose(f);
 348
 349         if (!rc)
 350           {
 351             string reply = get_reply();
 352             if (reply != "OK\n")
 353               {
 354                 rc = 1;
 355                 if (s->verbose > 0)
 356                   {
 357                     if (reply.empty())
 358                       clog << _("stapsh file ERROR: no reply") << endl;
 359                     else
 360                       clog << _F("stapsh file replied %s", reply.c_str());
 361                   }
 362               }
 363           }
 364
 365         return rc;
 366       }
 367
 368     static string qpencode(const string& str)
 369       {
 370         ostringstream o;
 371         o << setfill('0') << hex;
 372         for (const char* s = str.c_str(); *s; ++s)
 373           if (*s >= 33 && *s <= 126 && *s != 61)
 374             o << *s;
 375           else
 376             o << '=' << setw(2) << (unsigned)(unsigned char) *s;
 377         return o.str();
 378       }
 379
 380     static bool is_valid_data_cmd(const vector<string>& cmd)
 381       {
 382         bool err = false;
 383
 384         err = err || (cmd[0] != "data");
 385         err = err || (cmd[1] != "stdout" && cmd[1] != "stderr");
 386
 387         if (!err) // try to cast as size_t
 388           try { lex_cast<size_t>(cmd[2]); }
 389           catch (...) { err = true; }
 390
 391         return !err;
 392       }
 393
 394     virtual void printout(const char *buf, size_t size)
 395       {
 396         static string last_prefix;
 397         static bool on_same_line;
 398
 399         if (size == 0)
 400           return;
 401
 402         // don't prefix for stderr to be more consistent with ssh
 403         if (!prefix.empty() && target_stream != "stderr")
 404           {
 405             vector<pair<const char*,int> > lines = split_lines(buf, size);
 406             for (vector<pair<const char*,int> >::iterator it = lines.begin();
 407                 it != lines.end(); ++it)
 408               {
 409                 if (last_prefix != prefix)
 410                   {
 411                     if (on_same_line)
 412                       cout << endl;
 413                     cout << prefix;
 414                   }
 415                 else if (!on_same_line)
 416                   cout << prefix;
 417                 cout.write(it->first, it->second);
 418               }
 419             cout.flush();
 420             const char *last_line = lines.back().first;
 421             const char last_char = last_line[lines.back().second-1];
 422             on_same_line = !lines.empty() && last_char != '\n';
 423             last_prefix = prefix;
 424           }
 425         else
 426           {
 427             // Otherwise dump the whole block
 428             // NB: The buf could contain binary data,
 429             // including \0, so write as a block instead of
 430             // the usual << string.
 431             if (target_stream == "stdout")
 432               {
 433                 cout.write(buf, size);
 434                 cout.flush();
 435               }
 436             else // stderr
 437               clog.write(buf, size);
 438           }
 439       }
 440
 441   protected:
 442     stapsh(systemtap_session& s)
 443       : remote(s), interrupts_sent(0),
 444         fdin(-1), fdout(-1), IN(0), OUT(0),
 445         data_size(0), target_stream("stdout"), // default to stdout for schemes
 446         stream_state(STAPSH_READY)         // that don't pipe stderr (e.g. ssh)
 447       {}
 448
 449     vector<string> options;
 450
 451     virtual int prepare()
 452       {
 453         int rc = 0;
 454
 455         string extension;
 456         if (s->runtime_mode == systemtap_session::dyninst_runtime)
 457           extension = ".so";
 458         else if (s->runtime_mode == systemtap_session::bpf_runtime)
 459           extension = ".bo";
 460         else
 461           extension = ".ko";
 462
 463         string localmodule = s->tmpdir + "/" + s->module_name + extension;
 464         string remotemodule = s->module_name + extension;
 465
 466         if ((rc = send_file(localmodule, remotemodule)))
 467           return rc;
 468
 469         if (file_exists(localmodule + ".sgn") &&
 470             (rc = send_file(localmodule + ".sgn", remotemodule + ".sgn")))
 471           return rc;
 472
 473         if (!s->uprobes_path.empty())
 474           {
 475             string remoteuprobes = basename(s->uprobes_path.c_str());
 476             if ((rc = send_file(s->uprobes_path, remoteuprobes)))
 477               return rc;
 478
 479             if (file_exists(s->uprobes_path + ".sgn") &&
 480                 (rc = send_file(s->uprobes_path + ".sgn", remoteuprobes + ".sgn")))
 481               return rc;
 482           }
 483
 484         return rc;
 485       }
 486
 487     virtual int start()
 488       {
 489         // Send the staprun args
 490         // NB: The remote is left to decide its own staprun path
 491         ostringstream run("run", ios::out | ios::ate);
 492         vector<string> cmd = make_run_command(*s, ".", remote_version);
 493
 494         // PR13354: identify our remote index/url
 495         if (strverscmp("1.7", remote_version.c_str()) <= 0 && // -r supported?
 496             ! staprun_r_arg.empty())
 497           {
 498              if (s->runtime_mode == systemtap_session::dyninst_runtime)
 499                cmd.insert(cmd.end(), { "_stp_dyninst_remote=" +  staprun_r_arg});
 500              else if (s->runtime_mode == systemtap_session::kernel_runtime)
 501                cmd.insert(cmd.end(), { "-r", staprun_r_arg });
 502              // leave args empty for bpf_runtime
 503           }
 504
 505         for (unsigned i = 1; i < cmd.size(); ++i)
 506           run << ' ' << qpencode(cmd[i]);
 507         run << '\n';
 508
 509         int rc = send_command(run.str());
 510
 511         if (!rc)
 512           {
 513             string reply = get_reply();
 514             if (reply != "OK\n")
 515               {
 516                 rc = 1;
 517                 if (s->verbose > 0)
 518                   {
 519                     if (reply.empty())
 520                       clog << _("stapsh run ERROR: no reply") << endl;
 521                     else
 522                       clog << _F("stapsh run replied %s", reply.c_str());
 523                   }
 524               }
 525           }
 526
 527         if (!rc)
 528           {
 529             long flags;
 530             if ((flags = fcntl(fdout, F_GETFL)) == -1
 531               || fcntl(fdout, F_SETFL, flags | O_NONBLOCK) == -1)
 532               {
 533                 clog << _("failed to change to non-blocking mode") << endl;
 534                 rc = 1;
 535               }
 536           }
 537
 538         if (rc)
 539           // If run failed for any reason, then this
 540           // connection is effectively dead to us.
 541           close();
 542
 543         return rc;
 544       }
 545
 546     void close()
 547       {
 548         if (IN) fclose(IN);
 549         if (OUT) fclose(OUT);
 550         IN = OUT = NULL;
 551         fdin = fdout = -1;
 552       }
 553
 554     virtual int finish()
 555       {
 556         close();
 557         return 0;
 558       }
 559
 560     void set_child_fds(int in, int out)
 561       {
 562         if (fdin >= 0 || fdout >= 0 || IN || OUT)
 563           throw runtime_error(_("stapsh file descriptors already set"));
 564
 565         fdin = in;
 566         fdout = out;
 567         IN = fdopen(fdin, "w");
 568         OUT = fdopen(fdout, "r");
 569         if (!IN || !OUT)
 570           throw runtime_error(_("invalid file descriptors for stapsh"));
 571
 572         if (send_command("stap " VERSION "\n"))
 573           throw runtime_error(_("error sending hello to stapsh"));
 574
 575         string reply = get_reply();
 576         if (reply.empty())
 577           throw runtime_error(_("error receiving hello from stapsh"));
 578
 579         // stapsh VERSION MACHINE RELEASE
 580         vector<string> uname;
 581         tokenize(reply, uname, " \t\r\n");
 582         if (uname.size() != 4 || uname[0] != "stapsh")
 583           throw runtime_error(_F("invalid hello from stapsh: %s", reply.c_str()));
 584
 585         // We assume that later versions will know how to talk to us.
 586         // Looking backward, we use this for make_run_command().
 587         this->remote_version = uname[1];
 588
 589         this->s = s->clone(uname[2], uname[3]);
 590
 591         // set any option requested
 592         if (!this->options.empty())
 593           {
 594             // check first if the option command is supported
 595             if (strverscmp("2.4", this->remote_version.c_str()) > 0)
 596               throw runtime_error(_F("stapsh %s does not support options",
 597                                             this->remote_version.c_str()));
 598
 599             for (vector<string>::iterator it = this->options.begin();
 600                 it != this->options.end(); ++it)
 601               {
 602                 int rc = send_command("option " + *it + "\n");
 603                 if (rc != 0)
 604                   throw runtime_error(_F("could not set option %s: "
 605                                          "send_command returned %d",
 606                                          it->c_str(), rc));
 607                 string reply = get_reply();
 608                 if (reply != "OK\n")
 609                   throw runtime_error(_F("could not set option %s: %s",
 610                                           it->c_str(), reply.c_str()));
 611               }
 612           }
 613       }
 614
 615   public:
 616     virtual ~stapsh() { close(); }
 617 };
 618
 619
 620 // direct_stapsh is meant only for testing, as a way to exercise the stapsh
 621 // mechanisms without requiring test machines to have actual remote access.
 622 class direct_stapsh : public stapsh {
 623   private:
 624     pid_t child;
 625
 626     direct_stapsh(systemtap_session& s)
 627       : stapsh(s), child(0)
 628       {
 629         int in, out;
 630         vector<string> cmd { BINDIR "/stapsh" };
 631         if (s.perpass_verbose[4] > 1)
 632           cmd.push_back("-v");
 633         if (s.perpass_verbose[4] > 2)
 634           cmd.push_back("-v");
 635
 636         // mask signals while we spawn, so we can simulate manual signals to
 637         // the "remote" target, as we must for the real ssh_remote case.
 638         {
 639           stap_sigmasker masked;
 640           child = stap_spawn_piped(s.verbose, cmd, &in, &out);
 641         }
 642
 643         if (child <= 0)
 644           throw runtime_error(_("error launching stapsh"));
 645
 646         try
 647           {
 648             set_child_fds(in, out);
 649           }
 650         catch (runtime_error&)
 651           {
 652             finish();
 653             throw;
 654           }
 655       }
 656
 657     virtual int finish()
 658       {
 659         int rc = stapsh::finish();
 660         if (child <= 0)
 661           return rc;
 662
 663         int rc2 = stap_waitpid(s->verbose, child);
 664         child = 0;
 665         return rc ?: rc2;
 666       }
 667
 668   public:
 669     friend class remote;
 670
 671     virtual ~direct_stapsh() { finish(); }
 672 };
 673
 674
 675 // Connect to an existing stapsh on a unix socket.
 676 class unix_stapsh : public stapsh {
 677   private:
 678
 679     unix_stapsh(systemtap_session& s, const uri_decoder& ud)
 680       : stapsh(s)
 681       {
 682         // Request that data be encapsulated since both stdout and stderr have
 683         // to go over the same line. Also makes stapsh tell us when it quits.
 684         this->options.push_back("data");
 685
 686         // set verbosity to the requested level
 687         for (unsigned i = 1; i < s.perpass_verbose[4]; i++)
 688           this->options.push_back("verbose");
 689
 690         sockaddr_un server;
 691         server.sun_family = AF_UNIX;
 692         if (ud.path.empty())
 693           throw runtime_error(_("unix target requires a /path"));
 694         if (ud.path.size() > sizeof(server.sun_path) - 1)
 695           throw runtime_error(_("unix target /path is too long"));
 696         strcpy(server.sun_path, ud.path.c_str());
 697
 698         if (ud.has_authority)
 699           throw runtime_error(_("unix target doesn't support a hostname"));
 700         if (ud.has_query)
 701           throw runtime_error(_("unix target URI doesn't support a ?query"));
 702         if (ud.has_fragment)
 703           throw runtime_error(_("unix target URI doesn't support a #fragment"));
 704
 705         int fd = socket(AF_UNIX, SOCK_STREAM, 0);
 706         if (fd <= 0)
 707           throw runtime_error(_("error opening a socket"));
 708
 709         if (connect(fd, (struct sockaddr *)&server, SUN_LEN(&server)) < 0)
 710           {
 711             const char *msg = strerror(errno);
 712             ::close(fd);
 713             throw runtime_error(_F("error connecting to socket %s: %s",
 714                                     server.sun_path, msg));
 715           }
 716
 717         // Try to dup it, so class stapsh can have truly separate fds for its
 718         // fdopen handles.  If it fails for some reason, it can still work with
 719         // just one though.
 720         int fd2 = dup(fd);
 721         if (fd2 < 0)
 722           fd2 = fd;
 723
 724         try
 725           {
 726             set_child_fds(fd, fd2);
 727           }
 728         catch (runtime_error&)
 729           {
 730             finish();
 731             ::close(fd);
 732             ::close(fd2);
 733             throw;
 734           }
 735       }
 736
 737   public:
 738     friend class remote;
 739
 740     virtual ~unix_stapsh() { finish(); }
 741 };
 742
 743 class libvirt_stapsh : public stapsh {
 744   private:
 745
 746     pid_t child;
 747
 748     libvirt_stapsh(systemtap_session& s, const uri_decoder& ud)
 749       : stapsh(s)
 750       {
 751         string domain;
 752         string libvirt_uri;
 753
 754         // Request that data be encapsulated since both stdout and stderr have
 755         // to go over the same line. Also makes stapsh tell us when it quits.
 756         this->options.push_back("data");
 757
 758         // set verbosity to the requested level
 759         for (unsigned i = 1; i < s.perpass_verbose[4]; i++)
 760           this->options.push_back("verbose");
 761
 762         // A valid libvirt URI has one of two forms:
 763         //  - libvirt://DOMAIN/LIBVIRT_URI?LIBVIRT_QUERY
 764         //  - libvirt:DOMAIN/LIBVIRT_URI?LIBVIRT_QUERY
 765         // We only advertise the first form, but to be nice, we also accept the
 766         // second form. In the first form, DOMAIN is authority, LIBVIRT_URI is
 767         // path, and LIBVIRT_QUERY is the query. In the second form, the DOMAIN
 768         // is part of the path.
 769
 770         if (ud.has_fragment)
 771           throw runtime_error(_("libvirt target URI doesn't support a #fragment"));
 772
 773         if (ud.has_authority) // first form
 774           {
 775             domain = ud.authority;
 776             if (!ud.path.empty())
 777               {
 778                 libvirt_uri = ud.path.substr(1);
 779                 if (ud.has_query)
 780                   libvirt_uri += "?" + ud.query;
 781               }
 782           }
 783         else // second form
 784           {
 785             if (ud.path.empty())
 786               throw runtime_error(_("libvirt target URI requires a domain name"));
 787
 788             size_t slash = ud.path.find_first_of('/');
 789             if (slash == string::npos)
 790               domain = ud.path;
 791             else
 792               {
 793                 domain = ud.path.substr(0, slash);
 794                 libvirt_uri = ud.path.substr(slash+1);
 795               }
 796           }
 797
 798         int in, out;
 799
 800         string stapvirt = BINDIR "/stapvirt";
 801         if (!file_exists(stapvirt))
 802           throw runtime_error(_("stapvirt missing"));
 803
 804         vector<string> cmd { stapvirt };
 805
 806         // carry verbosity into stapvirt
 807         if (s.perpass_verbose[4] > 0)
 808           cmd.insert(cmd.end(), s.perpass_verbose[4] - 1, "-v");
 809
 810         if (!libvirt_uri.empty())
 811           cmd.insert(cmd.end(), { "-c", libvirt_uri });
 812
 813         cmd.insert(cmd.end(), { "connect", domain });
 814
 815         // mask signals for stapvirt since it relies instead on stap or stapsh
 816         // closing its connection to know when to exit (otherwise, stapvirt
 817         // would receive SIGINT on Ctrl-C and exit nonzero)
 818
 819         // NB: There is an interesting issue to note here. Some libvirt URIs
 820         // will create child processes of stapvirt (e.g. qemu+ssh:// will
 821         // create an ssh process under stapvirt, also qemu+ext://, see
 822         // <libvirt.org/remote.html> for more info). These processes do not
 823         // keep the mask of stapvirt we are creating here, but are still part
 824         // of the same process group.
 825         //     This means that e.g. SIGINT will not be blocked. Thus, stapvirt
 826         // explicitly adds a handler for SIGCHLD and forcefully disconnects
 827         // upon receiving it (otherwise it would await indefinitely).
 828         {
 829           stap_sigmasker masked;
 830           child = stap_spawn_piped(s.verbose, cmd, &in, &out);
 831         }
 832
 833         if (child <= 0)
 834           throw runtime_error(_("error launching stapvirt"));
 835
 836         try
 837           {
 838             set_child_fds(in, out);
 839           }
 840         catch (runtime_error&)
 841           {
 842             finish();
 843             throw;
 844           }
 845       }
 846
 847     int finish()
 848       {
 849         int rc = stapsh::finish();
 850         if (child <= 0)
 851           return rc;
 852
 853         int rc2 = stap_waitpid(s->verbose, child);
 854         child = 0;
 855         return rc ?: rc2;
 856       }
 857
 858     public:
 859       friend class remote;
 860
 861       virtual ~libvirt_stapsh() { finish(); }
 862 };
 863
 864 // stapsh-based ssh_remote
 865 class ssh_remote : public stapsh {
 866   private:
 867     pid_t child;
 868
 869     ssh_remote(systemtap_session& s): stapsh(s), child(0) {}
 870
 871     int connect(const string& host, const string& port)
 872       {
 873         int rc = 0;
 874         int in, out;
 875         vector<string> cmd { "ssh", "-q", host };
 876         if (!port.empty())
 877           cmd.insert(cmd.end(), { "-p", port });
 878
 879         // This is crafted so that we get a silent failure with status 127 if
 880         // the command is not found.  The combination of -P and $cmd ensures
 881         // that we pull the command out of the PATH, not aliases or such.
 882         string stapsh_cmd = "cmd=`type -P stapsh || exit 127` && exec \"$cmd\"";
 883         if (s->perpass_verbose[4] > 1)
 884           stapsh_cmd.append(" -v");
 885         if (s->perpass_verbose[4] > 2)
 886           stapsh_cmd.append(" -v");
 887         // NB: We need to explicitly choose bash, as $SHELL may be weird...
 888         cmd.push_back("/bin/bash -c '" + stapsh_cmd + "'");
 889
 890         // mask signals while we spawn, so we can manually send even tty
 891         // signals *through* ssh rather than to ssh itself
 892         {
 893           stap_sigmasker masked;
 894           child = stap_spawn_piped(s->verbose, cmd, &in, &out);
 895         }
 896
 897         if (child <= 0)
 898           throw runtime_error(_("error launching stapsh"));
 899
 900         try
 901           {
 902             set_child_fds(in, out);
 903           }
 904         catch (runtime_error&)
 905           {
 906             rc = finish();
 907
 908             // ssh itself signals errors with 255
 909             if (rc == 255)
 910               throw runtime_error(_("error establishing ssh connection"));
 911
 912             // If rc == 127, that's command-not-found, so we let ::create()
 913             // try again in legacy mode.  But only do this if there's a single
 914             // remote, as the old code didn't handle ttys well with multiple
 915             // remotes.  Otherwise, throw up again. *barf*
 916             if (rc != 127 || s->remote_uris.size() > 1)
 917               throw;
 918           }
 919
 920         return rc;
 921       }
 922
 923     int finish()
 924       {
 925         int rc = stapsh::finish();
 926         if (child <= 0)
 927           return rc;
 928
 929         int rc2 = stap_waitpid(s->verbose, child);
 930         child = 0;
 931         return rc ?: rc2;
 932       }
 933
 934   public:
 935
 936     static remote* create(systemtap_session& s, const string& host);
 937     static remote* create(systemtap_session& s, const uri_decoder& ud);
 938
 939     virtual ~ssh_remote() { finish(); }
 940 };
 941
 942
 943 // ssh connection without stapsh, for legacy stap installations
 944 // NB: ssh commands use a tty (-t) so signals are passed along to the remote.
 945 // It does this by putting the local tty in raw mode, so it only works for tty
 946 // signals, and only for a single remote at a time.
 947 class ssh_legacy_remote : public remote {
 948   private:
 949     vector<string> ssh_args, scp_args;
 950     string ssh_control;
 951     string host, port, tmpdir;
 952     pid_t child;
 953
 954     ssh_legacy_remote(systemtap_session& s, const string& host, const string& port)
 955       : remote(s), host(host), port(port), child(0)
 956       {
 957         open_control_master();
 958         try
 959           {
 960             get_uname();
 961           }
 962         catch (runtime_error&)
 963           {
 964             close_control_master();
 965             throw;
 966           }
 967       }
 968
 969     vector<string> ssh_command(initializer_list<string> ilist)
 970       {
 971         vector<string> cmd = ssh_args;
 972         cmd.insert(cmd.end(), ilist);
 973         return cmd;
 974       }
 975
 976     int copy_to_remote(const string& local, const string& remote)
 977       {
 978         vector<string> cmd = scp_args;
 979         cmd.insert(cmd.end(), { local, host + ":" + remote });
 980         return stap_system(s->verbose, cmd);
 981       }
 982
 983     void open_control_master()
 984       {
 985         static unsigned index = 0;
 986
 987         if (s->tmpdir.empty()) // sanity check, shouldn't happen
 988           throw runtime_error(_("No tmpdir available for ssh control master"));
 989
 990         ssh_control = s->tmpdir + "/ssh_remote_control_" + lex_cast(++index);
 991
 992         scp_args = { "scp", "-q", "-o", "ControlPath=" + ssh_control };
 993         ssh_args = { "ssh", "-q", "-o", "ControlPath=" + ssh_control, host };
 994
 995         if (!port.empty())
 996           {
 997             scp_args.insert(scp_args.end(), { "-P", port });
 998             ssh_args.insert(ssh_args.end(), { "-p", port });
 999           }
1000
1001         // NB: ssh -f will stay in the foreground until authentication is
1002         // complete and the control socket is created, so we know it's ready to
1003         // go when stap_system returns.
1004         auto cmd = ssh_command({ "-f", "-N", "-M" });
1005         int rc = stap_system(s->verbose, cmd);
1006         if (rc != 0)
1007           throw runtime_error(_F("failed to create an ssh control master for %s : rc= %d",
1008                                  host.c_str(), rc));
1009
1010         if (s->verbose>1)
1011           clog << _F("Created ssh control master at %s",
1012                      lex_cast_qstring(ssh_control).c_str()) << endl;
1013       }
1014
1015     void close_control_master()
1016       {
1017         if (ssh_control.empty())
1018           return;
1019
1020         auto cmd = ssh_command({ "-O", "exit" });
1021         int rc = stap_system(s->verbose, cmd, true, true);
1022         if (rc != 0)
1023           cerr << _F("failed to stop the ssh control master for %s : rc=%d",
1024                      host.c_str(), rc) << endl;
1025
1026         ssh_control.clear();
1027         scp_args.clear();
1028         ssh_args.clear();
1029       }
1030
1031     void get_uname()
1032       {
1033         ostringstream out;
1034         vector<string> uname;
1035         auto cmd = ssh_command({ "-t", "uname -rm" });
1036         int rc = stap_system_read(s->verbose, cmd, out);
1037         if (rc == 0)
1038           tokenize(out.str(), uname, " \t\r\n");
1039         if (uname.size() != 2)
1040           throw runtime_error(_F("failed to get uname from %s : rc= %d", host.c_str(), rc));
1041         const string& release = uname[0];
1042         const string& arch = uname[1];
1043         // XXX need to deal with command-line vs. implied arch/release
1044         this->s = s->clone(arch, release);
1045       }
1046
1047     int start()
1048       {
1049         int rc;
1050         string localmodule = s->tmpdir + "/" + s->module_name + ".ko";
1051         string tmpmodule;
1052
1053         // Make a remote tempdir.
1054         {
1055           ostringstream out;
1056           vector<string> vout;
1057           auto cmd = ssh_command({ "-t", "mktemp -d -t stapXXXXXX" });
1058           rc = stap_system_read(s->verbose, cmd, out);
1059           if (rc == 0)
1060             tokenize(out.str(), vout, "\r\n");
1061           if (vout.size() != 1)
1062             {
1063               cerr << _F("failed to make a tempdir on %s : rc=%d",
1064                          host.c_str(), rc) << endl;
1065               return -1;
1066             }
1067           tmpdir = vout[0];
1068           tmpmodule = tmpdir + "/" + s->module_name + ".ko";
1069         }
1070
1071         // Transfer the module.
1072         if (rc == 0)
1073           {
1074             rc = copy_to_remote(localmodule, tmpmodule);
1075             if (rc != 0)
1076               cerr << _F("failed to copy the module to %s : rc=%d",
1077                          host.c_str(), rc) << endl;
1078           }
1079
1080         // Transfer the module signature.
1081         if (rc == 0 && file_exists(localmodule + ".sgn"))
1082           {
1083             rc = copy_to_remote(localmodule + ".sgn", tmpmodule + ".sgn");
1084             if (rc != 0)
1085               cerr << _F("failed to copy the module signature to %s : rc=%d",
1086                          host.c_str(), rc) << endl;
1087           }
1088
1089         // What about transfering uprobes.ko?  In this ssh "legacy" mode, we
1090         // don't the remote systemtap version, but -uPATH wasn't added until
1091         // 1.4.  Rather than risking a getopt error, we'll just assume that
1092         // this isn't supported.  The remote will just have to provide its own
1093         // uprobes.ko in SYSTEMTAP_RUNTIME or already loaded.
1094
1095         // Run the module on the remote.
1096         if (rc == 0) {
1097           // We don't know the actual version, but all <=1.3 are approx equal.
1098           vector<string> staprun_cmd = make_run_command(*s, tmpdir, "1.3");
1099           staprun_cmd[0] = "staprun"; // NB: The remote decides its own path
1100           // NB: PR13354: we assume legacy installations don't have
1101           // staprun -r support, so we ignore staprun_r_arg.
1102           auto cmd = ssh_command({ "-t", cmdstr_join(staprun_cmd) });
1103           pid_t pid = stap_spawn(s->verbose, cmd);
1104           if (pid > 0)
1105             child = pid;
1106           else
1107             {
1108               cerr << _F("failed to run the module on %s : ret=%d",
1109                          host.c_str(), pid) << endl;
1110               rc = -1;
1111             }
1112         }
1113
1114         return rc;
1115       }
1116
1117     int finish()
1118       {
1119         int rc = 0;
1120
1121         if (child > 0)
1122           {
1123             rc = stap_waitpid(s->verbose, child);
1124             child = 0;
1125           }
1126
1127         if (!tmpdir.empty())
1128           {
1129             // Remove the tempdir.
1130             // XXX need to make sure this runs even with e.g. CTRL-C exits
1131             auto cmd = ssh_command({ "-t", "rm -r " + cmdstr_quoted(tmpdir) });
1132             int rc2 = stap_system(s->verbose, cmd);
1133             if (rc2 != 0)
1134               cerr << _F("failed to delete the tempdir on %s : rc=%d",
1135                          host.c_str(), rc2) << endl;
1136             if (rc == 0)
1137               rc = rc2;
1138             tmpdir.clear();
1139           }
1140
1141         close_control_master();
1142
1143         return rc;
1144       }
1145
1146   public:
1147     friend class ssh_remote;
1148
1149     virtual ~ssh_legacy_remote()
1150       {
1151         close_control_master();
1152       }
1153 };
1154
1155
1156 // Try to establish a stapsh connection to the remote, but fallback
1157 // to the older mechanism if the command is not found.
1158 remote*
1159 ssh_remote::create(systemtap_session& s, const string& target)
1160 {
1161   string port, host = target;
1162   size_t i = host.find(':');
1163   if (i != string::npos)
1164     {
1165       port = host.substr(i + 1);
1166       host.erase(i);
1167     }
1168
1169   unique_ptr<ssh_remote> p (new ssh_remote(s));
1170   int rc = p->connect(host, port);
1171   if (rc == 0)
1172     return p.release();
1173   else if (rc == 127) // stapsh command not found
1174     return new ssh_legacy_remote(s, host, port); // try legacy instead
1175   return NULL;
1176 }
1177
1178 remote*
1179 ssh_remote::create(systemtap_session& s, const uri_decoder& ud)
1180 {
1181   if (!ud.has_authority || ud.authority.empty())
1182     throw runtime_error(_("ssh target requires a hostname"));
1183   if (!ud.path.empty() && ud.path != "/")
1184     throw runtime_error(_("ssh target URI doesn't support a /path"));
1185   if (ud.has_query)
1186     throw runtime_error(_("ssh target URI doesn't support a ?query"));
1187   if (ud.has_fragment)
1188     throw runtime_error(_("ssh target URI doesn't support a #fragment"));
1189
1190   return create(s, ud.authority);
1191 }
1192
1193
1194 remote*
1195 remote::create(systemtap_session& s, const string& uri, int idx)
1196 {
1197   remote *it = 0;
1198   try
1199     {
1200       if (uri.find(':') != string::npos)
1201         {
1202           const uri_decoder ud(uri);
1203
1204           // An ssh "host:port" is ambiguous with a URI "scheme:path".
1205           // So if it looks like a number, just assume ssh.
1206           if (!ud.has_authority && !ud.has_query &&
1207               !ud.has_fragment && !ud.path.empty() &&
1208               ud.path.find_first_not_of("1234567890") == string::npos)
1209             it = ssh_remote::create(s, uri);
1210           else if (ud.scheme == "direct")
1211             it = new direct(s);
1212           else if (ud.scheme == "stapsh")
1213             it = new direct_stapsh(s);
1214           else if (ud.scheme == "unix")
1215             it = new unix_stapsh(s, ud);
1216           else if (ud.scheme == "libvirt")
1217             it = new libvirt_stapsh(s, ud);
1218           else if (ud.scheme == "ssh")
1219             it = ssh_remote::create(s, ud);
1220           else
1221             throw runtime_error(_F("unrecognized URI scheme '%s' in remote: %s",
1222                                    ud.scheme.c_str(), uri.c_str()));
1223         }
1224       else
1225         // XXX assuming everything else is ssh for now...
1226         it = ssh_remote::create(s, uri);
1227     }
1228   catch (std::runtime_error& e)
1229     {
1230       cerr << e.what() << " on remote '" << uri << "'" << endl;
1231       it = 0;
1232     }
1233
1234   if (it && idx >= 0) // PR13354: remote metadata for staprun -r IDX:URI
1235     {
1236       stringstream r_arg;
1237       r_arg << idx << ":" << uri;
1238       it->staprun_r_arg = r_arg.str();
1239     }
1240
1241   return it;
1242 }
1243
1244 int
1245 remote::run(const vector<remote*>& remotes)
1246 {
1247   // NB: the first failure "wins"
1248   int ret = 0, rc = 0;
1249
1250   for (unsigned i = 0; i < remotes.size() && !pending_interrupts; ++i)
1251     {
1252       remote *r = remotes[i];
1253       r->s->verbose = r->s->perpass_verbose[4];
1254       if (r->s->use_remote_prefix)
1255         r->prefix = lex_cast(i) + ": ";
1256       rc = r->prepare();
1257       if (rc)
1258         return rc;
1259     }
1260
1261   for (unsigned i = 0; i < remotes.size() && !pending_interrupts; ++i)
1262     {
1263       rc = remotes[i]->start();
1264       if (!ret)
1265         ret = rc;
1266     }
1267
1268   // mask signals while we're preparing to poll
1269   {
1270     stap_sigmasker masked;
1271
1272     // polling loop for remotes that have fds to watch
1273     for (;;)
1274       {
1275         vector<pollfd> fds;
1276         for (unsigned i = 0; i < remotes.size(); ++i)
1277           remotes[i]->prepare_poll (fds);
1278         if (fds.empty())
1279           break;
1280
1281         rc = ppoll (&fds[0], fds.size(), NULL, &masked.old);
1282         if (rc < 0 && errno != EINTR)
1283           break;
1284
1285         for (unsigned i = 0; i < remotes.size(); ++i)
1286           remotes[i]->handle_poll (fds);
1287       }
1288   }
1289
1290   for (unsigned i = 0; i < remotes.size(); ++i)
1291     {
1292       rc = remotes[i]->finish();
1293       if (!ret)
1294         ret = rc;
1295     }
1296
1297   return ret;
1298 }
1299
1300
1301 /* vim: set sw=2 ts=8 cino=>4,n-2,{2,^-2,t0,(0,u0,w1,M1 : */