ENH: Python server can now run multiple servers

 - the SERVER_PORTS is now a list of ports and when you run
   python_server.py it will run as many servers as desired.
 - python_server.py now will create multiple servers via
   multiprocessing.
 - the xmlrpc_clients.py is changed to deal with these multiple servers.
   This allows us to handle many incoming requests.

These changes allow us to run the online test for many users.  We had
over 400 simultaneous users and a total of about 650 users using the app
with these modifications.

3 files changed, 74 insertions, 12 deletions

diff --git a/exam/xmlrpc_clients.py b/exam/xmlrpc_clients.py
index 5dc51c6..115ee6e 100644
--- a/exam/xmlrpc_clients.py
+++ b/exam/xmlrpc_clients.py
@@ -1,5 +1,41 @@
 from xmlrpclib import ServerProxy
-from settings import SERVER_PORT
+from settings import SERVER_PORTS
+import random
+import socket
 
-# Connect to the python server.
-python_server = ServerProxy('http://localhost:%d'%(SERVER_PORT))
+
+class PythonServer(object):
+    """A class that manages accesing the farm of Python servers and making
+    calls to them such that no one XMLRPC server is overloaded.
+    """
+    def __init__(self):
+        servers = [ServerProxy('http://localhost:%d'%(x)) for x in SERVER_PORTS]
+        self.servers = servers
+        self.indices = range(len(SERVER_PORTS))
+    
+    def run_code(self, answer, test_code, user_dir):
+        """See the documentation of the method of the same name in 
+        python_server.py.
+        """
+        done = False
+        result = [False, 'Unable to connect to any Python servers!']
+        # Try to connect a few times if not, quit.
+        count = 5
+        while (not done) and (count > 0):
+            try:
+                server = self._get_server()
+                result = server.run_code(answer, test_code, user_dir)
+            except socket.error:
+                count -= 1
+            else:
+                done = True
+        return result
+
+    def _get_server(self):
+        # pick a suitable server at random from our pool of servers.
+        index = random.choice(self.indices)
+        return self.servers[index]
+
+# views.py calls this Python server which forwards the request to one
+# of the running servers.
+python_server = PythonServer()
diff --git a/python_server.py b/python_server.py
index 20d7440..d33ee47 100755
--- a/python_server.py
+++ b/python_server.py
@@ -1,7 +1,19 @@
 #!/usr/bin/env python
 """This server runs an XMLRPC server that can be submitted code and tests
 and returns the output.  It *should* be run as root and will run as the user 
-'nobody' so as to minimize any damange by errant code.
+'nobody' so as to minimize any damange by errant code.  This can be configured 
+by editing settings.py to run as many servers as desired.  One can also
+specify the ports on the command line.  Here are examples::
+
+  $ sudo ./python_server.py
+  # Runs servers based on settings.py:SERVER_PORTS one server per port given.
+  
+or::
+
+  $ sudo ./python_server.py 8001 8002 8003 8004 8005
+  # Runs 5 servers on ports specified.
+
+All these servers should be running as nobody.
 """
 import sys
 import traceback
@@ -10,11 +22,15 @@ import pwd
 import os
 from os.path import isdir
 import signal
-from settings import SERVER_PORT, SERVER_TIMEOUT
+from multiprocessing import Process
+
+# Local imports.
+from settings import SERVER_PORTS, SERVER_TIMEOUT
 
 
 def run_as_nobody():
-    # Set the effective uid
+    """Runs the current process as nobody."""
+    # Set the effective uid to that of nobody.
     nobody = pwd.getpwnam('nobody')
     os.setegid(nobody.pw_gid)
     os.seteuid(nobody.pw_uid)
@@ -25,6 +41,7 @@ class TimeoutException(Exception):
     pass
     
 def timeout_handler(signum, frame):
+    """A handler for the ALARM signal."""
     raise TimeoutException('Code took too long to run.')
 
 
@@ -80,13 +97,21 @@ def run_code(answer, test_code, in_dir=None):
 
     return success, err
 
+def run_server(port):
+    server = SimpleXMLRPCServer(("localhost", port))
+    server.register_function(run_code)
+    server.serve_forever()
 
 def main():
     run_as_nobody()
-    server = SimpleXMLRPCServer(("localhost", SERVER_PORT))
-    server.register_function(run_code)
-    server.serve_forever()
+    if len(sys.argv) == 1:
+        ports = SERVER_PORTS
+    else:
+        ports = [int(x) for x in sys.argv[1:]]
+
+    for port in ports:
+        p = Process(target=run_server,  args=(port,))
+        p.start()
     
 if __name__ == '__main__':
     main()
-    
\ No newline at end of file
diff --git a/settings.py b/settings.py
index 4e9b390..e5e3e69 100644
--- a/settings.py
+++ b/settings.py
@@ -5,8 +5,9 @@ from os.path import dirname, join, basename, abspath
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
-# The port the Python server should run on.
-SERVER_PORT = 8001
+# The ports the Python server should run on.  This will run one separate 
+# server for each port listed in the following list.
+SERVER_PORTS = [8001] # range(8001, 8026)
 
 # Timeout for the code to run in seconds.  This is an integer!
 SERVER_TIMEOUT = 2