From 3738c8fefa8ac69508bb6daeee045c1f5ea0cb17 Mon Sep 17 00:00:00 2001 From: Prabhu Ramachandran Date: Fri, 12 Aug 2016 16:57:03 +0530 Subject: Add test case for code server. This currently fails when multiple threads ask for a code evaluation at the same time. --- yaksh/tests/test_code_server.py | 117 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 yaksh/tests/test_code_server.py (limited to 'yaksh/tests/test_code_server.py') diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py new file mode 100644 index 0000000..18510c6 --- /dev/null +++ b/yaksh/tests/test_code_server.py @@ -0,0 +1,117 @@ +import json +from multiprocessing import Process +try: + from Queue import Queue +except ImportError: + from queue import Queue +from threading import Thread +import unittest + + +from yaksh.code_server import ServerPool, SERVER_POOL_PORT + +from yaksh import settings +from yaksh.xmlrpc_clients import code_server + + +class TestCodeServer(unittest.TestCase): + + @classmethod + def setUpClass(cls): + settings.code_evaluators['python']['standardtestcase'] = \ + "yaksh.python_assertion_evaluator.PythonAssertionEvaluator" + ports = range(8001, 8006) + server_pool = ServerPool(ports=ports, pool_port=SERVER_POOL_PORT) + cls.server_pool = server_pool + cls.server_proc = p = Process(target=server_pool.run) + p.start() + + + @classmethod + def tearDownClass(cls): + cls.server_pool.stop() + cls.server_proc.terminate() + settings.code_evaluators['python']['standardtestcase'] = \ + "python_assertion_evaluator.PythonAssertionEvaluator" + + def test_inifinite_loop(self): + # Given + testdata = {'user_answer': 'while True: pass', + 'test_case_data': [{'test_case':'assert 1==2'}]} + + # When + result = code_server.run_code( + 'python', 'standardtestcase', json.dumps(testdata), '' + ) + + # Then + data = json.loads(result) + self.assertFalse(data['success']) + self.assertTrue('infinite loop' in data['error']) + + def test_correct_answer(self): + # Given + testdata = {'user_answer': 'def f(): return 1', + 'test_case_data': [{'test_case':'assert f() == 1'}]} + + # When + result = code_server.run_code( + 'python', 'standardtestcase', json.dumps(testdata), '' + ) + + # Then + data = json.loads(result) + self.assertTrue(data['success']) + self.assertEqual(data['error'], 'Correct answer') + + def test_wrong_answer(self): + # Given + testdata = {'user_answer': 'def f(): return 1', + 'test_case_data': [{'test_case':'assert f() == 2'}]} + + # When + result = code_server.run_code( + 'python', 'standardtestcase', json.dumps(testdata), '' + ) + + # Then + data = json.loads(result) + self.assertFalse(data['success']) + self.assertTrue('AssertionError' in data['error']) + + def test_multiple_simultaneous_hits(self): + # Given + results = Queue() + + def run_code(): + """Run an infinite loop.""" + testdata = {'user_answer': 'while True: pass', + 'test_case_data': [{'test_case':'assert 1==2'}]} + result = code_server.run_code( + 'python', 'standardtestcase', json.dumps(testdata), '' + ) + results.put(json.loads(result)) + + N = 5 + # When + import time + threads = [] + for i in range(N): + t = Thread(target=run_code) + threads.append(t) + t.start() + + for t in threads: + if t.isAlive(): + t.join() + + # Then + self.assertEqual(results.qsize(), N) + for i in range(N): + data = results.get() + self.assertFalse(data['success']) + self.assertTrue('infinite loop' in data['error']) + + +if __name__ == '__main__': + unittest.main() -- cgit From b39eb538c08d4dd6761d6a970aadf58bde58a7ce Mon Sep 17 00:00:00 2001 From: Prabhu Ramachandran Date: Fri, 12 Aug 2016 21:01:08 +0530 Subject: Use a tornado based server for the pool server. With the previous XMLRPC based server, an XMLRPC server would respond to a request for an available port. This does not work as the server can only take about 2 simultaneous connections. The server pool now uses a HTTP server via tornado which works extremely well. The django code should not change at all as this is an internal change. This change should make the code server far more robust and work for a very large number of simultaneous users. The http server also has a simple status page to indicate the current load. This will not be correct on OSX due to limitations of the multi-processing Queue implementation on OSX. --- yaksh/tests/test_code_server.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) (limited to 'yaksh/tests/test_code_server.py') diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 18510c6..a73f073 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -1,17 +1,15 @@ import json -from multiprocessing import Process try: from Queue import Queue except ImportError: from queue import Queue from threading import Thread import unittest - +import urllib from yaksh.code_server import ServerPool, SERVER_POOL_PORT - from yaksh import settings -from yaksh.xmlrpc_clients import code_server +from yaksh.xmlrpc_clients import CodeServerProxy class TestCodeServer(unittest.TestCase): @@ -23,24 +21,26 @@ class TestCodeServer(unittest.TestCase): ports = range(8001, 8006) server_pool = ServerPool(ports=ports, pool_port=SERVER_POOL_PORT) cls.server_pool = server_pool - cls.server_proc = p = Process(target=server_pool.run) - p.start() - + cls.server_thread = t = Thread(target=server_pool.run) + t.start() @classmethod def tearDownClass(cls): cls.server_pool.stop() - cls.server_proc.terminate() + cls.server_thread.join() settings.code_evaluators['python']['standardtestcase'] = \ "python_assertion_evaluator.PythonAssertionEvaluator" + def setUp(self): + self.code_server = CodeServerProxy() + def test_inifinite_loop(self): # Given testdata = {'user_answer': 'while True: pass', 'test_case_data': [{'test_case':'assert 1==2'}]} # When - result = code_server.run_code( + result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) @@ -55,7 +55,7 @@ class TestCodeServer(unittest.TestCase): 'test_case_data': [{'test_case':'assert f() == 1'}]} # When - result = code_server.run_code( + result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) @@ -70,7 +70,7 @@ class TestCodeServer(unittest.TestCase): 'test_case_data': [{'test_case':'assert f() == 2'}]} # When - result = code_server.run_code( + result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) @@ -87,12 +87,12 @@ class TestCodeServer(unittest.TestCase): """Run an infinite loop.""" testdata = {'user_answer': 'while True: pass', 'test_case_data': [{'test_case':'assert 1==2'}]} - result = code_server.run_code( + result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) results.put(json.loads(result)) - N = 5 + N = 10 # When import time threads = [] @@ -112,6 +112,19 @@ class TestCodeServer(unittest.TestCase): self.assertFalse(data['success']) self.assertTrue('infinite loop' in data['error']) + def test_server_pool_status(self): + # Given + url = "http://localhost:%s/status"%SERVER_POOL_PORT + + # When + data = urllib.urlopen(url).read() + + # Then + expect = 'out of 5 are free' + self.assertTrue(expect in data) + expect = 'Load:' + self.assertTrue(expect in data) + if __name__ == '__main__': unittest.main() -- cgit