ENH: Creating a ServerPool for code checks

Changed the server so we use a pool of servers managed with a Queue of
available servers.  The XML/RPC client is also changed to handle
connection failures gracefully.  This is because XML/RPC cannot have
more than 2 connections at a given time, so if there is an error, we
wait for a random amount of time and try again.  This allows us to
handle fairly large loads nicely.

1 files changed, 34 insertions, 20 deletions

diff --git a/exam/xmlrpc_clients.py b/exam/xmlrpc_clients.py
index 01172d7..817e37d 100644
--- a/exam/xmlrpc_clients.py
+++ b/exam/xmlrpc_clients.py
@@ -1,17 +1,24 @@
 from xmlrpclib import ServerProxy
-from settings import SERVER_PORTS
+import time
 import random
 import socket
 
+from settings import SERVER_PORTS, SERVER_POOL_PORT
 
-class CodeServer(object):
+
+class ConnectionError(Exception):
+    pass
+
+################################################################################
+# `CodeServerProxy` class.
+################################################################################
+class CodeServerProxy(object):
     """A class that manages accesing the farm of Python servers and making
     calls to them such that no one XMLRPC server is overloaded.
     """
     def __init__(self):
-        servers = [ServerProxy('http://localhost:%d'%(x)) for x in SERVER_PORTS]
-        self.servers = servers
-        self.indices = range(len(SERVER_PORTS))
+        pool_url = 'http://localhost:%d'%(SERVER_POOL_PORT)
+        self.pool_server = ServerProxy(pool_url)
         self.methods = {"python": 'run_python_code',
                         "bash": 'run_bash_code'}
     
@@ -38,27 +45,34 @@ class CodeServer(object):
         A tuple: (success, error message).
         """
         method_name = self.methods[language]
-        done = False
-        result = [False, 'Unable to connect to any code servers!']
-        # Try to connect a few times if not, quit.
-        count = 5
-        while (not done) and (count > 0):
+        try:
+            server = self._get_server()
+            method = getattr(server, method_name)
+            result = method(answer, test_code, user_dir)
+        except ConnectionError:
+            result = [False, 'Unable to connect to any code servers!']
+        return result
+
+    def _get_server(self):
+        # Get a suitable server from our pool of servers.  This may block.  We
+        # try about 60 times, essentially waiting at most for about 30 seconds.
+        done, count = False, 60
+
+        while not done and count > 0:
             try:
-                server = self._get_server()
-                method = getattr(server, method_name)
-                result = method(answer, test_code, user_dir)
+                port = self.pool_server.get_server_port()
             except socket.error:
+                # Wait a while try again.
+                time.sleep(random.random())
                 count -= 1
             else:
                 done = True
-        return result
-
-    def _get_server(self):
-        # pick a suitable server at random from our pool of servers.
-        index = random.choice(self.indices)
-        return self.servers[index]
+        if not done:
+            raise ConnectionError("Couldn't connect to a server!") 
+        proxy = ServerProxy('http://localhost:%d'%port)
+        return proxy
 
 # views.py calls this Python server which forwards the request to one
 # of the running servers.
-code_server = CodeServer()
+code_server = CodeServerProxy()