ApplicationPoolClientServer.h

00001 /*
00002  *  Phusion Passenger - http://www.modrails.com/
00003  *  Copyright (C) 2008  Phusion
00004  *
00005  *  This program is free software; you can redistribute it and/or modify
00006  *  it under the terms of the GNU General Public License as published by
00007  *  the Free Software Foundation; version 2 of the License.
00008  *
00009  *  This program is distributed in the hope that it will be useful,
00010  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00011  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00012  *  GNU General Public License for more details.
00013  *
00014  *  You should have received a copy of the GNU General Public License along
00015  *  with this program; if not, write to the Free Software Foundation, Inc.,
00016  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
00017  */
00018 #ifndef _PASSENGER_APPLICATION_POOL_CLIENT_SERVER_H_
00019 #define _PASSENGER_APPLICATION_POOL_CLIENT_SERVER_H_
00020 
00021 #include <boost/bind.hpp>
00022 #include <boost/thread/thread.hpp>
00023 
00024 #include <set>
00025 
00026 #include <sys/types.h>
00027 #include <sys/socket.h>
00028 #include <cstdlib>
00029 #include <errno.h>
00030 #include <unistd.h>
00031 
00032 #include "StandardApplicationPool.h"
00033 #include "MessageChannel.h"
00034 #include "Exceptions.h"
00035 #include "Logging.h"
00036 
00037 namespace Passenger {
00038 
00039 using namespace std;
00040 using namespace boost;
00041 
00042 /**
00043  * Multi-process usage support for ApplicationPool.
00044  *
00045  * ApplicationPoolServer implements a client/server architecture for ApplicationPool.
00046  * This allows one to use ApplicationPool in a multi-process environment (unlike
00047  * StandardApplicationPool). The cache/pool data is stored in the server. Different
00048  * processes can then access the pool through the server.
00049  *
00050  * ApplicationPoolServer itself does not inherit ApplicationPool. Instead, it returns
00051  * an ApplicationPool object via the connect() call. For example:
00052  * @code
00053  *   // Create an ApplicationPoolServer.
00054  *   ApplicationPoolServer server(...);
00055  *   
00056  *   // Now fork a child process, like Apache's prefork MPM eventually will.
00057  *   pid_t pid = fork();
00058  *   if (pid == 0) {
00059  *       // Child process
00060  *       
00061  *       // Connect to the server. After connection, we have an ApplicationPool
00062  *       // object!
00063  *       ApplicationPoolPtr pool(server.connect());
00064  *
00065  *       // The child process doesn't run a server (only the parent process does)
00066  *       // so we call detach to free the server resources (things like file
00067  *       // descriptors).
00068  *       server.detach();
00069  *
00070  *       ApplicationPool::SessionPtr session(pool->get("/home/webapps/foo"));
00071  *       do_something_with(session);
00072  *
00073  *       _exit(0);
00074  *   } else {
00075  *       // Parent process
00076  *       waitpid(pid, NULL, 0);
00077  *   }
00078  * @endcode
00079  *
00080  * @warning
00081  *   ApplicationPoolServer uses threads internally. Threads will disappear after a fork(),
00082  *   so ApplicationPoolServer will become usable as a server after a fork(). After a fork(),
00083  *   you can still call connect() (and, of course, detach()), but the same
00084  *   ApplicationPoolServer better still be running in the parent process. So in case of
00085  *   Apache with the prefork MPM, be sure to create an ApplicationPoolServer
00086  *   <em>after</em> Apache has daemonized.
00087  *
00088  * <h2>Implementation notes</h2>
00089  * Notice that ApplicationPoolServer does do not use TCP sockets at all, or even named Unix
00090  * sockets, depite being a server that can handle multiple clients! So ApplicationPoolServer
00091  * will expose no open ports or temporary Unix socket files. Only child processes are able
00092  * to use the ApplicationPoolServer.
00093  *
00094  * This is implemented through anonymous Unix sockets (<tt>socketpair()</tt>) and file descriptor
00095  * passing. It allows one to emulate <tt>accept()</tt>. During initialization,
00096  * ApplicationPoolServer creates a pair of Unix sockets, one called <tt>serverSocket</tt>
00097  * and the other called <tt>connectSocket</tt>. There is a thread which continuously
00098  * listens on serverSocket for incoming data. The data itself is not important, because it
00099  * only serves to wake up the thread. ApplicationPoolServer::connect() sends some data through
00100  * connectSocket, which wakes up the server thread. The server thread will then create
00101  * a pair of Unix sockets. One of them is passed through serverSocket. The other will be
00102  * handled by a newly created client thread. So the socket that was passed through serverSocket
00103  * is the client's connection to the server, while the other socket is the server's connection
00104  * to the client.
00105  *
00106  * Note that serverSocket and connectSocket are solely used for setting up new connections
00107  * ala accept(). They are not used for any actual data. In fact, they cannot be used in any
00108  * other way without some sort of inter-process synchronization mechanism, because all
00109  * child processes are connected to the same serverSocket. In contrast,
00110  * ApplicationPoolServer::connect() allows one to setup a private communicate channel between
00111  * the server and the current child process.
00112  *
00113  * Also note that each client is handled by a seperate thread. This is necessary because
00114  * ApplicationPoolServer internally uses StandardApplicationPool, and the current algorithm
00115  * for StandardApplicationPool::get() can block (in the case that the spawning limit has
00116  * been exceeded). While it is possible to get around this problem without using threads,
00117  * a thread-based implementation is easier to write.
00118  *
00119  * @ingroup Support
00120  */
00121 class ApplicationPoolServer {
00122 private:
00123         /**
00124          * Contains data shared between RemoteSession and Client.
00125          * Since RemoteSession and Client have different life times, i.e. one may be
00126          * destroyed before the other, they both use a smart pointer that points to
00127          * a SharedData. This way, the SharedData object is only destroyed when
00128          * both the RemoteSession and the Client object has been destroyed.
00129          */
00130         struct SharedData {
00131                 /**
00132                  * The socket connection to the server, as was established by
00133                  * ApplicationPoolServer::connect().
00134                  */
00135                 int server;
00136                 
00137                 ~SharedData() {
00138                         close(server);
00139                 }
00140         };
00141         
00142         typedef shared_ptr<SharedData> SharedDataPtr;
00143 
00144         /**
00145          * An Application::Session which works together with ApplicationPoolServer.
00146          */
00147         class RemoteSession: public Application::Session {
00148         private:
00149                 SharedDataPtr data;
00150                 int id;
00151                 int reader;
00152                 int writer;
00153                 pid_t pid;
00154         public:
00155                 RemoteSession(SharedDataPtr data, pid_t pid, int id, int reader, int writer) {
00156                         this->data = data;
00157                         this->pid = pid;
00158                         this->id = id;
00159                         this->reader = reader;
00160                         this->writer = writer;
00161                 }
00162                 
00163                 virtual ~RemoteSession() {
00164                         closeReader();
00165                         closeWriter();
00166                         MessageChannel(data->server).write("close", toString(id).c_str(), NULL);
00167                 }
00168                 
00169                 virtual int getReader() const {
00170                         return reader;
00171                 }
00172                 
00173                 virtual void closeReader() {
00174                         if (reader != -1) {
00175                                 close(reader);
00176                                 reader = -1;
00177                         }
00178                 }
00179                 
00180                 virtual int getWriter() const {
00181                         return writer;
00182                 }
00183                 
00184                 virtual void closeWriter() {
00185                         if (writer != -1) {
00186                                 close(writer);
00187                                 writer = -1;
00188                         }
00189                 }
00190                 
00191                 virtual pid_t getPid() const {
00192                         return pid;
00193                 }
00194         };
00195 
00196         /**
00197          * An ApplicationPool implementation that works together with ApplicationPoolServer.
00198          * It doesn't do much by itself, its job is mostly to forward queries/commands to
00199          * the server and returning the result. Most of the logic is in the server.
00200          */
00201         class Client: public ApplicationPool {
00202         private:
00203                 SharedDataPtr data;
00204                 
00205         public:
00206                 /**
00207                  * Create a new Client.
00208                  *
00209                  * @param sock The newly established socket connection with the ApplicationPoolServer.
00210                  */
00211                 Client(int sock) {
00212                         data = ptr(new SharedData());
00213                         data->server = sock;
00214                 }
00215                 
00216                 virtual void clear() {
00217                         MessageChannel channel(data->server);
00218                         channel.write("clear", NULL);
00219                 }
00220                 
00221                 virtual void setMaxIdleTime(unsigned int seconds) {
00222                         MessageChannel channel(data->server);
00223                         channel.write("setMaxIdleTime", toString(seconds).c_str(), NULL);
00224                 }
00225                 
00226                 virtual void setMax(unsigned int max) {
00227                         MessageChannel channel(data->server);
00228                         channel.write("setMax", toString(max).c_str(), NULL);
00229                 }
00230                 
00231                 virtual unsigned int getActive() const {
00232                         MessageChannel channel(data->server);
00233                         vector<string> args;
00234                         
00235                         channel.write("getActive", NULL);
00236                         channel.read(args);
00237                         return atoi(args[0].c_str());
00238                 }
00239                 
00240                 virtual unsigned int getCount() const {
00241                         MessageChannel channel(data->server);
00242                         vector<string> args;
00243                         
00244                         channel.write("getCount", NULL);
00245                         channel.read(args);
00246                         return atoi(args[0].c_str());
00247                 }
00248                 
00249                 virtual pid_t getSpawnServerPid() const {
00250                         MessageChannel channel(data->server);
00251                         vector<string> args;
00252                         
00253                         channel.write("getSpawnServerPid", NULL);
00254                         channel.read(args);
00255                         return atoi(args[0].c_str());
00256                 }
00257                 
00258                 virtual Application::SessionPtr get(const string &appRoot, bool lowerPrivilege = true, const string &lowestUser = "nobody") {
00259                         MessageChannel channel(data->server);
00260                         vector<string> args;
00261                         int reader, writer;
00262                         
00263                         channel.write("get", appRoot.c_str(),
00264                                 (lowerPrivilege) ? "true" : "false",
00265                                 lowestUser.c_str(), NULL);
00266                         if (!channel.read(args)) {
00267                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00268                         }
00269                         if (args[0] == "ok") {
00270                                 reader = channel.readFileDescriptor();
00271                                 writer = channel.readFileDescriptor();
00272                                 return ptr(new RemoteSession(data, atoi(args[1]), atoi(args[2]), reader, writer));
00273                         } else if (args[0] == "SpawnException") {
00274                                 if (args[2] == "true") {
00275                                         string errorPage;
00276                                         
00277                                         if (!channel.readScalar(errorPage)) {
00278                                                 throw IOException("The ApplicationPool server unexpectedly closed the connection.");
00279                                         }
00280                                         throw SpawnException(args[1], errorPage);
00281                                 } else {
00282                                         throw SpawnException(args[1]);
00283                                 }
00284                         } else if (args[0] == "IOException") {
00285                                 throw IOException(args[1]);
00286                         } else {
00287                                 throw IOException("The ApplicationPool server returned an unknown message.");
00288                         }
00289                 }
00290         };
00291         
00292         /**
00293          * Contains information about exactly one client.
00294          */
00295         struct ClientInfo {
00296                 /** The connection to the client. */
00297                 int fd;
00298                 /** The thread which handles the client. */
00299                 thread *thr;
00300                 bool detached;
00301                 
00302                 ClientInfo() {
00303                         detached = false;
00304                 }
00305                 
00306                 void detach() {
00307                         detached = true;
00308                         close(fd);
00309                         fd = -1;
00310                 }
00311                 
00312                 ~ClientInfo() {
00313                         /* For some reason, joining or deleting (detaching)
00314                          * the thread after fork() will cause a segfault.
00315                          * I haven't figured out why that happens, so for now
00316                          * I'll just ignore the thread (which isn't running
00317                          * anyway).
00318                          */
00319                         if (!detached) {
00320                                 close(fd);
00321                                 delete thr;
00322                         }
00323                 }
00324         };
00325         
00326         typedef shared_ptr<ClientInfo> ClientInfoPtr;
00327         
00328         StandardApplicationPool pool;
00329         int serverSocket;
00330         int connectSocket;
00331         bool done, detached;
00332         
00333         mutex lock;
00334         thread *serverThread;
00335         set<ClientInfoPtr> clients;
00336         
00337         /* TODO: the current design makes it possible to leak file descriptors.
00338          * For example, suppose that a fork() happens right after
00339          * serverThreadMainLoop() created a socketpair. Uh-oh. The problem is that
00340          * Apache can fork no matter what the threads are currently doing.
00341          *
00342          * This problem can be solved by running the server thread main loop in
00343          * its own process, instead of a thread in the Apache control process.
00344          *
00345          * This situation is a corner case though, and doesn't happen very often.
00346          * When it does happen, the problem isn't that great: an Apache worker
00347          * process will eventually get killed, thus freeing all its file
00348          * descriptors. So it should be acceptable to fix this problem in
00349          * a post-1.0.1 release.
00350          */
00351         
00352         /**
00353          * The entry point of the server thread which sets up private connections.
00354          * See the class overview's implementation notes for details.
00355          */
00356         void serverThreadMainLoop() {
00357                 while (!done) {
00358                         int fds[2], ret;
00359                         char x;
00360                         
00361                         // The received data only serves to wake up the server socket,
00362                         // and is not important.
00363                         do {
00364                                 ret = read(serverSocket, &x, 1);
00365                         } while (ret == -1 && errno == EINTR);
00366                         if (ret == 0) {
00367                                 break;
00368                         }
00369                         
00370                         // Incoming connect request.
00371                         do {
00372                                 ret = socketpair(AF_UNIX, SOCK_STREAM, 0, fds);
00373                         } while (ret == -1 && errno == EINTR);
00374                         if (ret == -1) {
00375                                 int e = errno;
00376                                 P_ERROR("Cannot create an anonymous Unix socket: " <<
00377                                         strerror(e) << " (" << e << ") --- aborting!");
00378                                 abort();
00379                                 
00380                                 // Shut up compiler warning.
00381                                 bool x = false;
00382                                 if (x) {
00383                                         printf("%d", e);
00384                                 }
00385                         }
00386                         
00387                         try {
00388                                 MessageChannel(serverSocket).writeFileDescriptor(fds[1]);
00389                                 do {
00390                                         ret = close(fds[1]);
00391                                 } while (ret == -1 && errno == EINTR);
00392                         } catch (SystemException &e) {
00393                                 P_ERROR("Cannot send a file descriptor: " << e.sys() <<
00394                                         " --- aborting!");
00395                                 abort();
00396                         } catch (const exception &e) {
00397                                 P_ERROR("Cannot send a file descriptor: " << e.what() <<
00398                                         " --- aborting!");
00399                                 abort();
00400                         }
00401                         
00402                         ClientInfoPtr info(new ClientInfo());
00403                         info->fd = fds[0];
00404                         info->thr = new thread(bind(&ApplicationPoolServer::clientThreadMainLoop, this, info));
00405                         mutex::scoped_lock l(lock);
00406                         clients.insert(info);
00407                 }
00408         }
00409         
00410         /**
00411          * The entry point of a thread which handles exactly one client.
00412          */
00413         void clientThreadMainLoop(ClientInfoPtr client) {
00414                 MessageChannel channel(client->fd);
00415                 vector<string> args;
00416                 map<int, Application::SessionPtr> sessions;
00417                 int lastID = 0;
00418 
00419                 try {
00420                         while (!done) {
00421                                 if (!channel.read(args)) {
00422                                         break;
00423                                 }
00424                                 
00425                                 P_TRACE(3, "Client " << this << ": received message: " <<
00426                                         toString(args));
00427                                 if (args[0] == "get" && args.size() == 4) {
00428                                         Application::SessionPtr session;
00429                                         bool failed = false;
00430                                         try {
00431                                                 session = pool.get(args[1], args[2] == "true", args[3]);
00432                                                 sessions[lastID] = session;
00433                                                 lastID++;
00434                                         } catch (const SpawnException &e) {
00435                                                 if (e.hasErrorPage()) {
00436                                                         P_TRACE(3, "Client " << this << ": SpawnException "
00437                                                                 "occured (with error page)");
00438                                                         channel.write("SpawnException", e.what(), "true", NULL);
00439                                                         channel.writeScalar(e.getErrorPage());
00440                                                 } else {
00441                                                         P_TRACE(3, "Client " << this << ": SpawnException "
00442                                                                 "occured (no error page)");
00443                                                         channel.write("SpawnException", e.what(), "false", NULL);
00444                                                 }
00445                                                 failed = true;
00446                                         } catch (const IOException &e) {
00447                                                 channel.write("IOException", e.what(), NULL);
00448                                                 failed = true;
00449                                         }
00450                                         if (!failed) {
00451                                                 try {
00452                                                         channel.write("ok", toString(session->getPid()).c_str(),
00453                                                                 toString(lastID - 1).c_str(), NULL);
00454                                                         channel.writeFileDescriptor(session->getReader());
00455                                                         channel.writeFileDescriptor(session->getWriter());
00456                                                         session->closeReader();
00457                                                         session->closeWriter();
00458                                                 } catch (const exception &) {
00459                                                         P_TRACE(3, "Client " << this << ": something went wrong "
00460                                                                 "while sending 'ok' back to the client.");
00461                                                         sessions.erase(lastID - 1);
00462                                                         throw;
00463                                                 }
00464                                         }
00465                                 
00466                                 } else if (args[0] == "close" && args.size() == 2) {
00467                                         sessions.erase(atoi(args[1]));
00468                                 
00469                                 } else if (args[0] == "clear" && args.size() == 1) {
00470                                         pool.clear();
00471                                 
00472                                 } else if (args[0] == "setMaxIdleTime" && args.size() == 2) {
00473                                         pool.setMaxIdleTime(atoi(args[1]));
00474                                 
00475                                 } else if (args[0] == "setMax" && args.size() == 2) {
00476                                         pool.setMax(atoi(args[1]));
00477                                 
00478                                 } else if (args[0] == "getActive" && args.size() == 1) {
00479                                         channel.write(toString(pool.getActive()).c_str(), NULL);
00480                                 
00481                                 } else if (args[0] == "getCount" && args.size() == 1) {
00482                                         channel.write(toString(pool.getCount()).c_str(), NULL);
00483                                 
00484                                 } else if (args[0] == "getSpawnServerPid" && args.size() == 1) {
00485                                         channel.write(toString(pool.getSpawnServerPid()).c_str(), NULL);
00486                                 
00487                                 } else {
00488                                         string name;
00489                                         if (args.empty()) {
00490                                                 name = "(null)";
00491                                         } else {
00492                                                 name = args[0];
00493                                         }
00494                                         P_WARN("An ApplicationPoolServer client sent an invalid command: "
00495                                                 << name << " (" << args.size() << " elements)");
00496                                         done = true;
00497                                 }
00498                         }
00499                 } catch (const exception &e) {
00500                         P_WARN("Uncaught exception in ApplicationPoolServer client thread: " <<
00501                                 e.what());
00502                 }
00503                 
00504                 mutex::scoped_lock l(lock);
00505                 clients.erase(client);
00506         }
00507         
00508 public:
00509         /**
00510          * Create a new ApplicationPoolServer object.
00511          *
00512          * @param spawnServerCommand The filename of the spawn server to use.
00513          * @param logFile Specify a log file that the spawn server should use.
00514          *            Messages on its standard output and standard error channels
00515          *            will be written to this log file. If an empty string is
00516          *            specified, no log file will be used, and the spawn server
00517          *            will use the same standard output/error channels as the
00518          *            current process.
00519          * @param environment The RAILS_ENV environment that all RoR applications
00520          *            should use. If an empty string is specified, the current value
00521          *            of the RAILS_ENV environment variable will be used.
00522          * @param rubyCommand The Ruby interpreter's command.
00523          * @param user The user that the spawn manager should run as. This
00524          *             parameter only has effect if the current process is
00525          *             running as root. If the empty string is given, or if
00526          *             the <tt>user</tt> is not a valid username, then
00527          *             the spawn manager will be run as the current user.
00528          * @throws SystemException An error occured while trying to setup the spawn server
00529          *            or the server socket.
00530          * @throws IOException The specified log file could not be opened.
00531          * @throws boost::thread_resource_error A threading resource could not be
00532          *            allocated or initialized.
00533          */
00534         ApplicationPoolServer(const string &spawnServerCommand,
00535                      const string &logFile = "",
00536                      const string &environment = "production",
00537                      const string &rubyCommand = "ruby",
00538                      const string &user = "")
00539         : pool(spawnServerCommand, logFile, environment, rubyCommand, user) {
00540                 int fds[2];
00541                 
00542                 if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) {
00543                         throw SystemException("Cannot create a Unix socket pair", errno);
00544                 }
00545                 serverSocket = fds[0];
00546                 connectSocket = fds[1];
00547                 done = false;
00548                 detached = false;
00549                 try {
00550                         serverThread = new thread(bind(&ApplicationPoolServer::serverThreadMainLoop, this));
00551                 } catch (const thread_resource_error &e) {
00552                         throw thread_resource_error("Could not create the ApplicationPoolServer "
00553                                 "server main loop thread", e.native_error());
00554                 }
00555         }
00556         
00557         ~ApplicationPoolServer() {
00558                 if (!detached) {
00559                         done = true;
00560                         close(connectSocket);
00561                         serverThread->join();
00562                         delete serverThread;
00563                         close(serverSocket);
00564                         
00565                         set<ClientInfoPtr> clientsCopy;
00566                         {
00567                                 mutex::scoped_lock l(lock);
00568                                 clientsCopy = clients;
00569                         }
00570                         set<ClientInfoPtr>::iterator it;
00571                         for (it = clientsCopy.begin(); it != clientsCopy.end(); it++) {
00572                                 (*it)->thr->join();
00573                         }
00574                 }
00575         }
00576         
00577         /**
00578          * Connects to the server and returns a usable ApplicationPool object.
00579          * All cache/pool data of this ApplicationPool is actually stored on
00580          * the server and shared with other clients, but that is totally
00581          * transparent to the user of the ApplicationPool object.
00582          *
00583          * @warning
00584          * One may only use the returned ApplicationPool object for handling
00585          * one session at a time. For example, don't do stuff like this:
00586          * @code
00587          *   ApplicationPoolPtr pool = server.connect();
00588          *   Application::SessionPtr session1 = pool->get(...);
00589          *   Application::SessionPtr session2 = pool->get(...);
00590          * @endcode
00591          * Otherwise, a deadlock can occur under certain circumstances.
00592          * @warning
00593          * Instead, one should call connect() multiple times:
00594          * @code
00595          *   ApplicationPoolPtr pool1 = server.connect();
00596          *   Application::SessionPtr session1 = pool1->get(...);
00597          *   
00598          *   ApplicationPoolPtr pool2 = server.connect();
00599          *   Application::SessionPtr session2 = pool2->get(...);
00600          * @endcode
00601          *
00602          * @throws SystemException Something went wrong.
00603          * @throws IOException Something went wrong.
00604          */
00605         ApplicationPoolPtr connect() {
00606                 MessageChannel channel(connectSocket);
00607                 int fd;
00608                 
00609                 // Write some random data to wake up the server.
00610                 channel.writeRaw("x", 1);
00611                 
00612                 fd = channel.readFileDescriptor();
00613                 return ptr(new Client(fd));
00614         }
00615         
00616         /**
00617          * Detach the server by freeing up some server resources such as file descriptors.
00618          * This should be called by child processes that wish to use a server, but do
00619          * not run the server itself.
00620          *
00621          * This method may only be called once. The ApplicationPoolServer object
00622          * will become unusable once detach() has been called.
00623          *
00624          * @warning Never call this method in the process in which this
00625          *    ApplicationPoolServer was created!
00626          */
00627         void detach() {
00628                 detached = true;
00629                 close(connectSocket);
00630                 close(serverSocket);
00631                 delete serverThread;
00632                 
00633                 /* A client thread might have a reference to a ClientInfo
00634                  * object. And because that thread doesn't run anymore after a
00635                  * fork(), the reference never gets removed and the ClientInfo
00636                  * object never gets destroyed. This results in file descriptor
00637                  * leaks. So we forcefully close the file descriptors.
00638                  */
00639                 set<ClientInfoPtr>::iterator it;
00640                 for (it = clients.begin(); it != clients.end(); it++) {
00641                         (*it)->detach();
00642                 }
00643                 clients.clear();
00644                 
00645                 pool.detach();
00646         }
00647 };
00648 
00649 typedef shared_ptr<ApplicationPoolServer> ApplicationPoolServerPtr;
00650 
00651 } // namespace Passenger
00652 
00653 #endif /* _PASSENGER_APPLICATION_POOL_CLIENT_SERVER_H_ */

Generated on Wed May 7 20:28:18 2008 for Passenger by  doxygen 1.5.3