ApplicationPoolServer.h

00001 /*
00002  *  Phusion Passenger - http://www.modrails.com/
00003  *  Copyright (C) 2008  Phusion
00004  *
00005  *  This program is free software; you can redistribute it and/or modify
00006  *  it under the terms of the GNU General Public License as published by
00007  *  the Free Software Foundation; version 2 of the License.
00008  *
00009  *  This program is distributed in the hope that it will be useful,
00010  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00011  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00012  *  GNU General Public License for more details.
00013  *
00014  *  You should have received a copy of the GNU General Public License along
00015  *  with this program; if not, write to the Free Software Foundation, Inc.,
00016  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
00017  */
00018 #ifndef _PASSENGER_APPLICATION_POOL_SERVER_H_
00019 #define _PASSENGER_APPLICATION_POOL_SERVER_H_
00020 
00021 #include <boost/shared_ptr.hpp>
00022 
00023 #include <sys/types.h>
00024 #include <sys/wait.h>
00025 #include <sys/socket.h>
00026 #include <cstdlib>
00027 #include <errno.h>
00028 #include <unistd.h>
00029 #include <signal.h>
00030 
00031 #include "MessageChannel.h"
00032 #include "ApplicationPool.h"
00033 #include "Application.h"
00034 #include "Exceptions.h"
00035 #include "Logging.h"
00036 
00037 namespace Passenger {
00038 
00039 using namespace std;
00040 using namespace boost;
00041 
00042 
00043 /**
00044  * Multi-process usage support for ApplicationPool.
00045  *
00046  * ApplicationPoolServer implements a client/server architecture for ApplicationPool.
00047  * This allows one to use ApplicationPool in a multi-process environment (unlike
00048  * StandardApplicationPool). The cache/pool data is stored in the server. Different
00049  * processes can then access the pool through the server.
00050  *
00051  * ApplicationPoolServer itself does not inherit ApplicationPool. Instead, it returns
00052  * an ApplicationPool object via the connect() call. For example:
00053  * @code
00054  *   // Create an ApplicationPoolServer.
00055  *   ApplicationPoolServer server(...);
00056  *   
00057  *   // Now fork a child process, like Apache's prefork MPM eventually will.
00058  *   pid_t pid = fork();
00059  *   if (pid == 0) {
00060  *       // Child process
00061  *       
00062  *       // Connect to the server. After connection, we have an ApplicationPool
00063  *       // object!
00064  *       ApplicationPoolPtr pool(server.connect());
00065  *
00066  *       // We don't need to connect to the server anymore, so we detach from it.
00067  *       // This frees up some resources, such as file descriptors.
00068  *       server.detach();
00069  *
00070  *       ApplicationPool::SessionPtr session(pool->get("/home/webapps/foo"));
00071  *       do_something_with(session);
00072  *
00073  *       _exit(0);
00074  *   } else {
00075  *       // Parent process
00076  *       waitpid(pid, NULL, 0);
00077  *   }
00078  * @endcode
00079  *
00080  * <h2>Implementation notes</h2>
00081  *
00082  * <h3>Separate server executable</h3>
00083  * The actual server is implemented in ApplicationPoolServerExecutable.cpp, this class is
00084  * just a convenience class for starting/stopping the server executable and connecting
00085  * to it.
00086  *
00087  * In the past, the server logic itself was implemented in this class. This implies that
00088  * the ApplicationPool server ran inside the Apache process. This presented us with several
00089  * problems:
00090  * - Because of the usage of threads in the ApplicationPool server, the Apache VM size would
00091  *   go way up. This gave people the (wrong) impression that Passenger uses a lot of memory,
00092  *   or that it leaks memory.
00093  * - Although it's not entirely confirmed, we suspect that it caused heap fragmentation as
00094  *   well. Apache allocates lots and lots of small objects on the heap, and ApplicationPool
00095  *   server isn't exactly helping. This too gave people the (wrong) impression that
00096  *   Passenger leaks memory.
00097  * - It would unnecessarily bloat the VM size of Apache worker processes.
00098  * - We had to resort to all kinds of tricks to make sure that fork()ing a process doesn't
00099  *   result in file descriptor leaks.
00100  * - Despite everything, there was still a small chance that file descriptor leaks would
00101  *   occur, and this could not be fixed. The reason for this is that the Apache control
00102  *   process may call fork() right after the ApplicationPool server has established a new
00103  *   connection with a client.
00104  *
00105  * Because of these problems, it was decided to split the ApplicationPool server to a
00106  * separate executable. This comes with no performance hit.
00107  *
00108  * <h3>Anonymous server socket</h3>
00109  * Notice that ApplicationPoolServer does do not use TCP sockets at all, or even named Unix
00110  * sockets, despite being a server that can handle multiple clients! So ApplicationPoolServer
00111  * will expose no open ports or temporary Unix socket files. Only child processes are able
00112  * to use the ApplicationPoolServer.
00113  *
00114  * This is implemented through anonymous Unix sockets (<tt>socketpair()</tt>) and file descriptor
00115  * passing. It allows one to emulate <tt>accept()</tt>. ApplicationPoolServer is connected to
00116  * the server executable through a Unix socket pair. connect() sends a connect request to the
00117  * server through that socket. The server will then create a new socket pair, and pass one of
00118  * them back. This new socket pair represents the newly established connection.
00119  *
00120  * @ingroup Support
00121  */
00122 class ApplicationPoolServer {
00123 private:
00124         /**
00125          * Contains data shared between RemoteSession and Client.
00126          * Since RemoteSession and Client have different life times, i.e. one may be
00127          * destroyed before the other, they both use a smart pointer that points to
00128          * a SharedData. This way, the SharedData object is only destroyed when
00129          * both the RemoteSession and the Client object has been destroyed.
00130          */
00131         struct SharedData {
00132                 /**
00133                  * The socket connection to the ApplicationPool server, as was
00134                  * established by ApplicationPoolServer::connect().
00135                  */
00136                 int server;
00137                 
00138                 ~SharedData() {
00139                         close(server);
00140                 }
00141         };
00142         
00143         typedef shared_ptr<SharedData> SharedDataPtr;
00144         
00145         /**
00146          * An Application::Session which works together with ApplicationPoolServer.
00147          */
00148         class RemoteSession: public Application::Session {
00149         private:
00150                 SharedDataPtr data;
00151                 int id;
00152                 int reader;
00153                 int writer;
00154                 pid_t pid;
00155         public:
00156                 RemoteSession(SharedDataPtr data, pid_t pid, int id, int reader, int writer) {
00157                         this->data = data;
00158                         this->pid = pid;
00159                         this->id = id;
00160                         this->reader = reader;
00161                         this->writer = writer;
00162                 }
00163                 
00164                 virtual ~RemoteSession() {
00165                         closeReader();
00166                         closeWriter();
00167                         MessageChannel(data->server).write("close", toString(id).c_str(), NULL);
00168                 }
00169                 
00170                 virtual int getReader() const {
00171                         return reader;
00172                 }
00173                 
00174                 virtual void closeReader() {
00175                         if (reader != -1) {
00176                                 close(reader);
00177                                 reader = -1;
00178                         }
00179                 }
00180                 
00181                 virtual int getWriter() const {
00182                         return writer;
00183                 }
00184                 
00185                 virtual void closeWriter() {
00186                         if (writer != -1) {
00187                                 close(writer);
00188                                 writer = -1;
00189                         }
00190                 }
00191                 
00192                 virtual pid_t getPid() const {
00193                         return pid;
00194                 }
00195         };
00196         
00197         /**
00198          * An ApplicationPool implementation that works together with ApplicationPoolServer.
00199          * It doesn't do much by itself, its job is mostly to forward queries/commands to
00200          * the server and returning the result. Most of the logic is in the server executable.
00201          */
00202         class Client: public ApplicationPool {
00203         private:
00204                 SharedDataPtr data;
00205                 
00206         public:
00207                 /**
00208                  * Create a new Client.
00209                  *
00210                  * @param sock The newly established socket connection with the ApplicationPoolServer.
00211                  */
00212                 Client(int sock) {
00213                         data = ptr(new SharedData());
00214                         data->server = sock;
00215                 }
00216                 
00217                 virtual void clear() {
00218                         MessageChannel channel(data->server);
00219                         channel.write("clear", NULL);
00220                 }
00221                 
00222                 virtual void setMaxIdleTime(unsigned int seconds) {
00223                         MessageChannel channel(data->server);
00224                         channel.write("setMaxIdleTime", toString(seconds).c_str(), NULL);
00225                 }
00226                 
00227                 virtual void setMax(unsigned int max) {
00228                         MessageChannel channel(data->server);
00229                         channel.write("setMax", toString(max).c_str(), NULL);
00230                 }
00231                 
00232                 virtual unsigned int getActive() const {
00233                         MessageChannel channel(data->server);
00234                         vector<string> args;
00235                         
00236                         channel.write("getActive", NULL);
00237                         channel.read(args);
00238                         return atoi(args[0].c_str());
00239                 }
00240                 
00241                 virtual unsigned int getCount() const {
00242                         MessageChannel channel(data->server);
00243                         vector<string> args;
00244                         
00245                         channel.write("getCount", NULL);
00246                         channel.read(args);
00247                         return atoi(args[0].c_str());
00248                 }
00249                 
00250                 virtual pid_t getSpawnServerPid() const {
00251                         MessageChannel channel(data->server);
00252                         vector<string> args;
00253                         
00254                         channel.write("getSpawnServerPid", NULL);
00255                         channel.read(args);
00256                         return atoi(args[0].c_str());
00257                 }
00258                 
00259                 virtual Application::SessionPtr get(const string &appRoot, bool lowerPrivilege = true,
00260                            const string &lowestUser = "nobody") {
00261                         MessageChannel channel(data->server);
00262                         vector<string> args;
00263                         int reader, writer;
00264                         
00265                         channel.write("get", appRoot.c_str(),
00266                                 (lowerPrivilege) ? "true" : "false",
00267                                 lowestUser.c_str(), NULL);
00268                         if (!channel.read(args)) {
00269                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00270                         }
00271                         if (args[0] == "ok") {
00272                                 reader = channel.readFileDescriptor();
00273                                 writer = channel.readFileDescriptor();
00274                                 return ptr(new RemoteSession(data, atoi(args[1]), atoi(args[2]), reader, writer));
00275                         } else if (args[0] == "SpawnException") {
00276                                 if (args[2] == "true") {
00277                                         string errorPage;
00278                                         
00279                                         if (!channel.readScalar(errorPage)) {
00280                                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00281                                         }
00282                                         throw SpawnException(args[1], errorPage);
00283                                 } else {
00284                                         throw SpawnException(args[1]);
00285                                 }
00286                         } else if (args[0] == "IOException") {
00287                                 throw IOException(args[1]);
00288                         } else {
00289                                 throw IOException("The ApplicationPool server returned an unknown message.");
00290                         }
00291                 }
00292         };
00293         
00294         
00295         static const int SERVER_SOCKET_FD = 3;
00296         
00297         string m_serverExecutable;
00298         string m_spawnServerCommand;
00299         string m_logFile;
00300         string m_environment;
00301         string m_rubyCommand;
00302         string m_user;
00303         
00304         /**
00305          * The PID of the ApplicationPool server process. If no server process
00306          * is running, then <tt>serverPid == 0</tt>.
00307          *
00308          * @invariant
00309          *    if serverPid == 0:
00310          *       serverSocket == -1
00311          */
00312         pid_t serverPid;
00313         
00314         /**
00315          * The connection to the ApplicationPool server process. If no server
00316          * process is running, then <tt>serverSocket == -1</tt>.
00317          *
00318          * @invariant
00319          *    if serverPid == 0:
00320          *       serverSocket == -1
00321          */
00322         int serverSocket;
00323         
00324         /**
00325          * Shutdown the currently running ApplicationPool server process.
00326          *
00327          * @pre serverSocket != -1 && serverPid != 0
00328          * @post serverSocket == -1 && serverPid == 0
00329          */
00330         void shutdownServer() {
00331                 time_t begin;
00332                 bool done;
00333                 int ret;
00334                 
00335                 do {
00336                         ret = close(serverSocket);
00337                 } while (ret == -1 && errno == EINTR);
00338                 
00339                 P_DEBUG("Waiting for existing ApplicationPoolServerExecutable to exit...");
00340                 begin = time(NULL);
00341                 while (!done && time(NULL) < begin + 5) {
00342                         done = waitpid(serverPid, NULL, WNOHANG) > 0;
00343                         usleep(100000);
00344                 }
00345                 if (done) {
00346                         P_DEBUG("ApplicationPoolServerExecutable exited.");
00347                 } else {
00348                         P_DEBUG("ApplicationPoolServerExecutable not exited in time. Killing it...");
00349                         kill(serverPid, SIGTERM);
00350                         waitpid(serverPid, NULL, 0);
00351                 }
00352                 serverSocket = -1;
00353                 serverPid = 0;
00354         }
00355         
00356         /**
00357          * Start an ApplicationPool server process. If there's already one running,
00358          * then the currently running one will be shutdown.
00359          *
00360          * @post serverSocket != -1 && serverPid != 0
00361          * @throw SystemException Something went wrong.
00362          */
00363         void restartServer() {
00364                 int fds[2];
00365                 pid_t pid;
00366                 
00367                 if (serverPid != 0) {
00368                         shutdownServer();
00369                 }
00370                 
00371                 if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) {
00372                         throw SystemException("Cannot create a Unix socket pair", errno);
00373                 }
00374                 
00375                 pid = fork();
00376                 if (pid == 0) { // Child process.
00377                         dup2(fds[0], 3);
00378                         
00379                         // Close all unnecessary file descriptors
00380                         for (long i = sysconf(_SC_OPEN_MAX) - 1; i > SERVER_SOCKET_FD; i--) {
00381                                 close(i);
00382                         }
00383                         
00384                         execlp(
00385                                 #if 0
00386                                         "valgrind",
00387                                         "valgrind",
00388                                 #else
00389                                         m_serverExecutable.c_str(),
00390                                 #endif
00391                                 m_serverExecutable.c_str(),
00392                                 m_spawnServerCommand.c_str(),
00393                                 m_logFile.c_str(),
00394                                 m_environment.c_str(),
00395                                 m_rubyCommand.c_str(),
00396                                 m_user.c_str(),
00397                                 NULL);
00398                         int e = errno;
00399                         fprintf(stderr, "*** Passenger ERROR: Cannot execute %s: %s (%d)\n",
00400                                 m_serverExecutable.c_str(), strerror(e), e);
00401                         fflush(stderr);
00402                         _exit(1);
00403                 } else if (pid == -1) { // Error.
00404                         close(fds[0]);
00405                         close(fds[1]);
00406                         throw SystemException("Cannot create a new process", errno);
00407                 } else { // Parent process.
00408                         close(fds[0]);
00409                         serverSocket = fds[1];
00410                         serverPid = pid;
00411                 }
00412         }
00413 
00414 public:
00415         /**
00416          * Create a new ApplicationPoolServer object.
00417          *
00418          * @param serverExecutable The filename of the ApplicationPool server
00419          *            executable to use.
00420          * @param spawnServerCommand The filename of the spawn server to use.
00421          * @param logFile Specify a log file that the spawn server should use.
00422          *            Messages on its standard output and standard error channels
00423          *            will be written to this log file. If an empty string is
00424          *            specified, no log file will be used, and the spawn server
00425          *            will use the same standard output/error channels as the
00426          *            current process.
00427          * @param environment The RAILS_ENV environment that all RoR applications
00428          *            should use. If an empty string is specified, the current value
00429          *            of the RAILS_ENV environment variable will be used.
00430          * @param rubyCommand The Ruby interpreter's command.
00431          * @param user The user that the spawn manager should run as. This
00432          *             parameter only has effect if the current process is
00433          *             running as root. If the empty string is given, or if
00434          *             the <tt>user</tt> is not a valid username, then
00435          *             the spawn manager will be run as the current user.
00436          * @throws SystemException An error occured while trying to setup the spawn server
00437          *            or the server socket.
00438          * @throws IOException The specified log file could not be opened.
00439          */
00440         ApplicationPoolServer(const string &serverExecutable,
00441                      const string &spawnServerCommand,
00442                      const string &logFile = "",
00443                      const string &environment = "production",
00444                      const string &rubyCommand = "ruby",
00445                      const string &user = "")
00446         : m_serverExecutable(serverExecutable),
00447           m_spawnServerCommand(spawnServerCommand),
00448           m_logFile(logFile),
00449           m_environment(environment),
00450           m_rubyCommand(rubyCommand),
00451           m_user(user) {
00452                 serverSocket = -1;
00453                 serverPid = 0;
00454                 restartServer();
00455         }
00456         
00457         ~ApplicationPoolServer() {
00458                 if (serverSocket != -1) {
00459                         shutdownServer();
00460                 }
00461         }
00462         
00463         /**
00464          * Connects to the server and returns a usable ApplicationPool object.
00465          * All cache/pool data of this ApplicationPool is actually stored on
00466          * the server and shared with other clients, but that is totally
00467          * transparent to the user of the ApplicationPool object.
00468          *
00469          * @warning
00470          * One may only use the returned ApplicationPool object for handling
00471          * one session at a time. For example, don't do stuff like this:
00472          * @code
00473          *   ApplicationPoolPtr pool = server.connect();
00474          *   Application::SessionPtr session1 = pool->get(...);
00475          *   Application::SessionPtr session2 = pool->get(...);
00476          * @endcode
00477          * Otherwise, a deadlock can occur under certain circumstances.
00478          * @warning
00479          * Instead, one should call connect() multiple times:
00480          * @code
00481          *   ApplicationPoolPtr pool1 = server.connect();
00482          *   Application::SessionPtr session1 = pool1->get(...);
00483          *   
00484          *   ApplicationPoolPtr pool2 = server.connect();
00485          *   Application::SessionPtr session2 = pool2->get(...);
00486          * @endcode
00487          *
00488          * @throws SystemException Something went wrong.
00489          * @throws IOException Something went wrong.
00490          */
00491         ApplicationPoolPtr connect() {
00492                 MessageChannel channel(serverSocket);
00493                 int clientConnection;
00494                 
00495                 // Write some random data to wake up the server.
00496                 channel.writeRaw("x", 1);
00497                 
00498                 clientConnection = channel.readFileDescriptor();
00499                 return ptr(new Client(clientConnection));
00500         }
00501         
00502         /**
00503          * Detach the server, thereby telling it that we don't want to connect
00504          * to it anymore. This frees up some resources in the current process,
00505          * such as file descriptors.
00506          *
00507          * This method is particularily useful to Apache worker processes that
00508          * have just established a connection with the ApplicationPool server.
00509          * Any sessions that are opened prior to calling detach(), will keep
00510          * working even after a detach().
00511          *
00512          * This method may only be called once. The ApplicationPoolServer object
00513          * will become unusable once detach() has been called, so call connect()
00514          * before calling detach().
00515          */
00516         void detach() {
00517                 close(serverSocket);
00518                 serverSocket = -1;
00519         }
00520 };
00521 
00522 typedef shared_ptr<ApplicationPoolServer> ApplicationPoolServerPtr;
00523 
00524 } // namespace Passenger
00525 
00526 #endif /* _PASSENGER_APPLICATION_POOL_SERVER_H_ */

Generated on Wed May 7 13:56:13 2008 for Passenger by  doxygen 1.5.3