From dee13fa4f8006d5266c02d6290b0e98d31413a9f Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 7 Dec 2016 14:59:07 +0530 Subject: Refactor code server and python evaluator --- yaksh/code_evaluator.py | 79 ++- yaksh/code_server.py | 21 +- .../evaluator_tests/old_test_python_evaluation.py | 549 ++++++++++++++++ yaksh/evaluator_tests/test_python_evaluation.py | 687 +++++++++++++++++---- yaksh/language_registry.py | 15 +- yaksh/old_code_server.py | 221 +++++++ yaksh/python_assertion_evaluator.py | 63 +- yaksh/settings.py | 36 +- 8 files changed, 1491 insertions(+), 180 deletions(-) mode change 100755 => 100644 yaksh/code_server.py create mode 100644 yaksh/evaluator_tests/old_test_python_evaluation.py create mode 100755 yaksh/old_code_server.py (limited to 'yaksh') diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index afe18c3..5ede63d 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -19,9 +19,12 @@ except ImportError: # Local imports from .settings import SERVER_TIMEOUT +from .language_registry import create_evaluator_instance + MY_DIR = abspath(dirname(__file__)) +registry = None # Raised when the code times-out. # c.f. http://pguides.net/python/timeout-a-function @@ -63,7 +66,8 @@ class CodeEvaluator(object): self.timeout_msg = msg self.in_dir = in_dir - def evaluate(self, **kwargs): + + def evaluate(self, kwargs): #language, test_case_type, """Evaluates given code with the test cases based on given arguments in test_case_data. @@ -85,6 +89,9 @@ class CodeEvaluator(object): A tuple: (success, error message, weight). """ + # self.language = language + # self.test_case_type = test_case_type + self.setup() success, error, weight = self.safe_evaluate(**kwargs) self.teardown() @@ -99,11 +106,13 @@ class CodeEvaluator(object): os.makedirs(self.in_dir) self._change_dir(self.in_dir) - def safe_evaluate(self, user_answer, partial_grading, test_case_data, file_paths=None): + def safe_evaluate(self, **kwargs): #user_answer, partial_grading, test_case_data, file_paths=None """ Handles code evaluation along with compilation, signal handling and Exception handling """ + metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths + test_case_data = kwargs.get('test_case_data') # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() @@ -114,14 +123,16 @@ class CodeEvaluator(object): # Do whatever testing needed. try: + # Run evaluator selection registry here for idx, test_case in enumerate(test_case_data): + test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case test_case_success = False - self.compile_code(user_answer, file_paths, **test_case) - test_case_success, err, test_case_weight = self.check_code(user_answer, - file_paths, - partial_grading, - **test_case - ) + test_case_instance.compile_code() #user_answer, file_paths, test_case + test_case_success, err, test_case_weight = test_case_instance.check_code() #**kwargs + # user_answer, + # file_paths, + # partial_grading, + # **test_case if test_case_success: weight += test_case_weight @@ -135,7 +146,7 @@ class CodeEvaluator(object): except OSError: msg = traceback.format_exc(limit=0) error = "Error: {0}".format(msg) - except Exception: + except Exception as e: exc_type, exc_value, exc_tb = sys.exc_info() tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) if len(tb_list) > 2: @@ -147,6 +158,56 @@ class CodeEvaluator(object): return success, error, weight + # def safe_evaluate(self, user_answer, partial_grading, test_case_data, file_paths=None): + # """ + # Handles code evaluation along with compilation, signal handling + # and Exception handling + # """ + + # # Add a new signal handler for the execution of this code. + # prev_handler = create_signal_handler() + # success = False + # test_case_success_status = [False] * len(test_case_data) + # error = "" + # weight = 0.0 + + # # Do whatever testing needed. + # try: + # for idx, test_case in enumerate(test_case_data): + # test_case_success = False + # self.compile_code(user_answer, file_paths, **test_case) + # test_case_success, err, test_case_weight = self.check_code(user_answer, + # file_paths, + # partial_grading, + # **test_case + # ) + # if test_case_success: + # weight += test_case_weight + + # error += err + "\n" + # test_case_success_status[idx] = test_case_success + + # success = all(test_case_success_status) + + # except TimeoutException: + # error = self.timeout_msg + # except OSError: + # msg = traceback.format_exc(limit=0) + # error = "Error: {0}".format(msg) + # except Exception as e: + # print "HELLOOOOO", e + # exc_type, exc_value, exc_tb = sys.exc_info() + # tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) + # if len(tb_list) > 2: + # del tb_list[1:3] + # error = "Error: {0}".format("".join(tb_list)) + # finally: + # # Set back any original signal handler. + # set_original_signal_handler(prev_handler) + + # return success, error, weight + + def teardown(self): # Cancel the signal delete_signal_handler() diff --git a/yaksh/code_server.py b/yaksh/code_server.py old mode 100755 new mode 100644 index b3c9c30..abe7cd8 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -53,7 +53,7 @@ from tornado.web import Application, RequestHandler # Local imports from .settings import SERVER_PORTS, SERVER_POOL_PORT -from .language_registry import create_evaluator_instance, unpack_json +from .language_registry import create_evaluator_instance MY_DIR = abspath(dirname(__file__)) @@ -84,19 +84,24 @@ class CodeServer(object): """Calls relevant EvaluateCode class based on language to check the answer code """ - code_evaluator = create_evaluator_instance(language, - test_case_type, - json_data, - in_dir - ) - data = unpack_json(json_data) - result = code_evaluator.evaluate(**data) + # code_evaluator = create_evaluator_instance(language, + # test_case_type, + # json_data, + # in_dir + # ) + data = unpack_json_to_python_obj(json_data) + code_eval_instance = CodeEvaluator(in_dir) + result = code_eval_instance.evaluate(**data) #language, test_case_type, # Put us back into the server pool queue since we are free now. self.queue.put(self.port) return json.dumps(result) + def unpack_json_to_python_obj(self, json_data): + data = json.loads(json_data) + return data + def run(self): """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("0.0.0.0", self.port)) diff --git a/yaksh/evaluator_tests/old_test_python_evaluation.py b/yaksh/evaluator_tests/old_test_python_evaluation.py new file mode 100644 index 0000000..9796fa2 --- /dev/null +++ b/yaksh/evaluator_tests/old_test_python_evaluation.py @@ -0,0 +1,549 @@ +from __future__ import unicode_literals +import unittest +import os +import tempfile +import shutil +from textwrap import dedent + +# Local import +from yaksh.python_assertion_evaluator import PythonAssertionEvaluator +from yaksh.python_stdio_evaluator import PythonStdioEvaluator +from yaksh.settings import SERVER_TIMEOUT + + +class PythonAssertionEvaluationTestCases(unittest.TestCase): + def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path + self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, + {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, + {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, + ] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + self.file_paths = None + + def tearDown(self): + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn a + b" + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertIn("Correct answer", result.get('error')) + + def test_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn a - b" + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertIn('AssertionError in: assert(add(1,2)==3)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) + + def test_partial_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" + test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, + {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, + {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': True + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('weight'), 2.0) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) + + def test_infinite_loop(self): + # Given + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) + + def test_syntax_error(self): + # Given + user_answer = dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "", + "SyntaxError", + "invalid syntax" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) + + def test_indent_error(self): + # Given + user_answer = dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = ["Traceback", "call", + "File", + "line", + "", + "IndentationError", + "indented block" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in indent_error_msg: + self.assertIn(msg, result.get("error")) + + def test_name_error(self): + # Given + user_answer = "" + name_error_msg = ["Traceback", + "call", + "NameError", + "name", + "defined" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(3, len(err)) + for msg in name_error_msg: + self.assertIn(msg, result.get("error")) + + def test_recursion_error(self): + # Given + user_answer = dedent(""" + def add(a, b): + return add(3, 3) + """) + recursion_error_msg = ["Traceback", + "call", + "maximum recursion depth exceeded" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + for msg in recursion_error_msg: + self.assertIn(msg, result.get("error")) + + def test_type_error(self): + # Given + user_answer = dedent(""" + def add(a): + return a + b + """) + type_error_msg = ["Traceback", + "call", + "TypeError", + "argument" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(3, len(err)) + for msg in type_error_msg: + self.assertIn(msg, result.get("error")) + + def test_value_error(self): + # Given + user_answer = dedent(""" + def add(a, b): + c = 'a' + return int(a) + int(b) + int(c) + """) + value_error_msg = ["Traceback", + "call", + "ValueError", + "invalid literal", + "base" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(4, len(err)) + for msg in value_error_msg: + self.assertIn(msg, result.get("error")) + + def test_file_based_assert(self): + # Given + self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] + self.file_paths = [('/tmp/test.txt', False)] + user_answer = dedent(""" + def ans(): + with open("test.txt") as f: + return f.read()[0] + """) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertIn("Correct answer", result.get('error')) + self.assertTrue(result.get('success')) + + def test_single_testcase_error(self): + # Given + """ Tests the user answer with just an incorrect test case """ + + user_answer = "def palindrome(a):\n\treturn a == a[::-1]" + test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + "weight": 0.0 + } + ] + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "", + "SyntaxError", + "invalid syntax" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) + + + def test_multiple_testcase_error(self): + """ Tests the user answer with an correct test case + first and then with an incorrect test case """ + # Given + user_answer = "def palindrome(a):\n\treturn a == a[::-1]" + test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + "weight": 0.0 + }, + {"test_case": 's="abbb"\nassert palindrome(S)==False', + "weight": 0.0 + } + ] + name_error_msg = ["Traceback", + "call", + "File", + "line", + "", + "NameError", + "name 'S' is not defined" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(3, len(err)) + for msg in name_error_msg: + self.assertIn(msg, result.get("error")) + + +class PythonStdIOEvaluationTestCases(unittest.TestCase): + def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + self.file_paths = None + + def test_correct_answer_integer(self): + # Given + self.test_case_data = [{"expected_input": "1\n2", + "expected_output": "3", + "weight": 0.0 + }] + user_answer = dedent(""" + a = int(input()) + b = int(input()) + print(a+b) + """ + ) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertIn("Correct answer", result.get('error')) + + def test_correct_answer_list(self): + # Given + self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", + "expected_output": "[1, 2, 3, 5, 6, 7]", + "weight": 0.0 + }] + user_answer = dedent(""" + from six.moves import input + input_a = input() + input_b = input() + a = [int(i) for i in input_a.split(',')] + b = [int(i) for i in input_b.split(',')] + print(a+b) + """ + ) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertIn("Correct answer", result.get('error')) + + def test_correct_answer_string(self): + # Given + self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), + "expected_output": "2", + "weight": 0.0 + }] + user_answer = dedent(""" + from six.moves import input + a = str(input()) + b = str(input()) + print(a.count(b)) + """ + ) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertIn("Correct answer", result.get('error')) + + def test_incorrect_answer_integer(self): + # Given + self.test_case_data = [{"expected_input": "1\n2", + "expected_output": "3", + "weight": 0.0 + }] + user_answer = dedent(""" + a = int(input()) + b = int(input()) + print(a-b) + """ + ) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertIn("Incorrect answer", result.get('error')) + + def test_file_based_answer(self): + # Given + self.test_case_data = [{"expected_input": "", + "expected_output": "2", + "weight": 0.0 + }] + self.file_paths = [('/tmp/test.txt', False)] + + user_answer = dedent(""" + with open("test.txt") as f: + a = f.read() + print(a[0]) + """ + ) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertEqual(result.get('error'), "Correct answer\n") + self.assertTrue(result.get('success')) + + def test_infinite_loop(self): + # Given + test_case_data = [{"expected_input": "1\n2", + "expected_output": "3", + "weight": 0.0 + }] + timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + user_answer = "while True:\n\tpass" + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'partial_grading': False + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertEqual(result.get('error'), timeout_msg) + self.assertFalse(result.get('success')) + +if __name__ == '__main__': + unittest.main() diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index acf5d0a..43cd0ea 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -6,25 +6,27 @@ import shutil from textwrap import dedent # Local import +from yaksh.code_evaluator import CodeEvaluator from yaksh.python_assertion_evaluator import PythonAssertionEvaluator from yaksh.python_stdio_evaluator import PythonStdioEvaluator from yaksh.settings import SERVER_TIMEOUT - class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, + self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) self.file_paths = None + self.language = 'python' + self.test_case_type = 'standardtestcase' def tearDown(self): os.remove('/tmp/test.txt') @@ -33,15 +35,24 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False, + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -50,15 +61,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + # evaluator = PythonAssertionEvaluator() + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -72,111 +88,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result.get('error') ) - def test_partial_incorrect_answer(self): - # Given - user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" - test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, - {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': True - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertEqual(result.get('weight'), 2.0) - self.assertIn('AssertionError in: assert(add(-1,2)==1)', - result.get('error') - ) - self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', - result.get('error') - ) - - def test_infinite_loop(self): - # Given - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), self.timeout_msg) - - def test_syntax_error(self): - # Given - user_answer = dedent(""" - def add(a, b); - return a + b - """) - syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "", - "SyntaxError", - "invalid syntax" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) - for msg in syntax_error_msg: - self.assertIn(msg, result.get("error")) - - def test_indent_error(self): - # Given - user_answer = dedent(""" - def add(a, b): - return a + b - """) - indent_error_msg = ["Traceback", "call", - "File", - "line", - "", - "IndentationError", - "indented block" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) - for msg in indent_error_msg: - self.assertIn(msg, result.get("error")) def test_name_error(self): # Given @@ -542,5 +453,539 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): self.assertEqual(result.get('error'), timeout_msg) self.assertFalse(result.get('success')) +# class PythonAssertionEvaluationTestCases(unittest.TestCase): +# def setUp(self): +# with open('/tmp/test.txt', 'wb') as f: +# f.write('2'.encode('ascii')) +# tmp_in_dir_path = tempfile.mkdtemp() +# self.in_dir = tmp_in_dir_path +# self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, +# {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, +# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, +# ] +# self.timeout_msg = ("Code took more than {0} seconds to run. " +# "You probably have an infinite loop in" +# " your code.").format(SERVER_TIMEOUT) +# self.file_paths = None + +# def tearDown(self): +# os.remove('/tmp/test.txt') +# shutil.rmtree(self.in_dir) + +# def test_correct_answer(self): +# # Given +# user_answer = "def add(a,b):\n\treturn a + b" +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertTrue(result.get('success')) +# self.assertIn("Correct answer", result.get('error')) + +# def test_incorrect_answer(self): +# # Given +# user_answer = "def add(a,b):\n\treturn a - b" +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertFalse(result.get('success')) +# self.assertIn('AssertionError in: assert(add(1,2)==3)', +# result.get('error') +# ) +# self.assertIn('AssertionError in: assert(add(-1,2)==1)', +# result.get('error') +# ) +# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', +# result.get('error') +# ) + +# def test_partial_incorrect_answer(self): +# # Given +# user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" +# test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, +# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, +# {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': True +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertFalse(result.get('success')) +# self.assertEqual(result.get('weight'), 2.0) +# self.assertIn('AssertionError in: assert(add(-1,2)==1)', +# result.get('error') +# ) +# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', +# result.get('error') +# ) + +# def test_infinite_loop(self): +# # Given +# user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertFalse(result.get('success')) +# self.assertEqual(result.get('error'), self.timeout_msg) + +# def test_syntax_error(self): +# # Given +# user_answer = dedent(""" +# def add(a, b); +# return a + b +# """) +# syntax_error_msg = ["Traceback", +# "call", +# "File", +# "line", +# "", +# "SyntaxError", +# "invalid syntax" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(5, len(err)) +# for msg in syntax_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_indent_error(self): +# # Given +# user_answer = dedent(""" +# def add(a, b): +# return a + b +# """) +# indent_error_msg = ["Traceback", "call", +# "File", +# "line", +# "", +# "IndentationError", +# "indented block" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False + +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(5, len(err)) +# for msg in indent_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_name_error(self): +# # Given +# user_answer = "" +# name_error_msg = ["Traceback", +# "call", +# "NameError", +# "name", +# "defined" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(3, len(err)) +# for msg in name_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_recursion_error(self): +# # Given +# user_answer = dedent(""" +# def add(a, b): +# return add(3, 3) +# """) +# recursion_error_msg = ["Traceback", +# "call", +# "maximum recursion depth exceeded" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# for msg in recursion_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_type_error(self): +# # Given +# user_answer = dedent(""" +# def add(a): +# return a + b +# """) +# type_error_msg = ["Traceback", +# "call", +# "TypeError", +# "argument" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(3, len(err)) +# for msg in type_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_value_error(self): +# # Given +# user_answer = dedent(""" +# def add(a, b): +# c = 'a' +# return int(a) + int(b) + int(c) +# """) +# value_error_msg = ["Traceback", +# "call", +# "ValueError", +# "invalid literal", +# "base" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(4, len(err)) +# for msg in value_error_msg: +# self.assertIn(msg, result.get("error")) + +# def test_file_based_assert(self): +# # Given +# self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] +# self.file_paths = [('/tmp/test.txt', False)] +# user_answer = dedent(""" +# def ans(): +# with open("test.txt") as f: +# return f.read()[0] +# """) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertIn("Correct answer", result.get('error')) +# self.assertTrue(result.get('success')) + +# def test_single_testcase_error(self): +# # Given +# """ Tests the user answer with just an incorrect test case """ + +# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" +# test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', +# "weight": 0.0 +# } +# ] +# syntax_error_msg = ["Traceback", +# "call", +# "File", +# "line", +# "", +# "SyntaxError", +# "invalid syntax" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(5, len(err)) +# for msg in syntax_error_msg: +# self.assertIn(msg, result.get("error")) + + +# def test_multiple_testcase_error(self): +# """ Tests the user answer with an correct test case +# first and then with an incorrect test case """ +# # Given +# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" +# test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', +# "weight": 0.0 +# }, +# {"test_case": 's="abbb"\nassert palindrome(S)==False', +# "weight": 0.0 +# } +# ] +# name_error_msg = ["Traceback", +# "call", +# "File", +# "line", +# "", +# "NameError", +# "name 'S' is not defined" +# ] +# kwargs = {'user_answer': user_answer, +# 'test_case_data': test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonAssertionEvaluator() +# result = evaluator.evaluate(**kwargs) +# err = result.get("error").splitlines() + +# # Then +# self.assertFalse(result.get("success")) +# self.assertEqual(3, len(err)) +# for msg in name_error_msg: +# self.assertIn(msg, result.get("error")) + + +# class PythonStdIOEvaluationTestCases(unittest.TestCase): +# def setUp(self): +# with open('/tmp/test.txt', 'wb') as f: +# f.write('2'.encode('ascii')) +# self.file_paths = None + +# def test_correct_answer_integer(self): +# # Given +# self.test_case_data = [{"expected_input": "1\n2", +# "expected_output": "3", +# "weight": 0.0 +# }] +# user_answer = dedent(""" +# a = int(input()) +# b = int(input()) +# print(a+b) +# """ +# ) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertTrue(result.get('success')) +# self.assertIn("Correct answer", result.get('error')) + +# def test_correct_answer_list(self): +# # Given +# self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", +# "expected_output": "[1, 2, 3, 5, 6, 7]", +# "weight": 0.0 +# }] +# user_answer = dedent(""" +# from six.moves import input +# input_a = input() +# input_b = input() +# a = [int(i) for i in input_a.split(',')] +# b = [int(i) for i in input_b.split(',')] +# print(a+b) +# """ +# ) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertTrue(result.get('success')) +# self.assertIn("Correct answer", result.get('error')) + +# def test_correct_answer_string(self): +# # Given +# self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), +# "expected_output": "2", +# "weight": 0.0 +# }] +# user_answer = dedent(""" +# from six.moves import input +# a = str(input()) +# b = str(input()) +# print(a.count(b)) +# """ +# ) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertTrue(result.get('success')) +# self.assertIn("Correct answer", result.get('error')) + +# def test_incorrect_answer_integer(self): +# # Given +# self.test_case_data = [{"expected_input": "1\n2", +# "expected_output": "3", +# "weight": 0.0 +# }] +# user_answer = dedent(""" +# a = int(input()) +# b = int(input()) +# print(a-b) +# """ +# ) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertFalse(result.get('success')) +# self.assertIn("Incorrect answer", result.get('error')) + +# def test_file_based_answer(self): +# # Given +# self.test_case_data = [{"expected_input": "", +# "expected_output": "2", +# "weight": 0.0 +# }] +# self.file_paths = [('/tmp/test.txt', False)] + +# user_answer = dedent(""" +# with open("test.txt") as f: +# a = f.read() +# print(a[0]) +# """ +# ) +# kwargs = {'user_answer': user_answer, +# 'test_case_data': self.test_case_data, +# 'file_paths': self.file_paths, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertEqual(result.get('error'), "Correct answer\n") +# self.assertTrue(result.get('success')) + +# def test_infinite_loop(self): +# # Given +# test_case_data = [{"expected_input": "1\n2", +# "expected_output": "3", +# "weight": 0.0 +# }] +# timeout_msg = ("Code took more than {0} seconds to run. " +# "You probably have an infinite loop in" +# " your code.").format(SERVER_TIMEOUT) +# user_answer = "while True:\n\tpass" +# kwargs = {'user_answer': user_answer, +# 'test_case_data': test_case_data, +# 'partial_grading': False +# } + +# # When +# evaluator = PythonStdioEvaluator() +# result = evaluator.evaluate(**kwargs) + +# # Then +# self.assertEqual(result.get('error'), timeout_msg) +# self.assertFalse(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 0e0140b..8d3aad2 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -14,15 +14,16 @@ def get_registry(): registry = _LanguageRegistry() return registry -def unpack_json(json_data): - data = json.loads(json_data) - return data +# def unpack_json_to_python_obj(json_data): +# data = json.loads(json_data) +# return data -def create_evaluator_instance(language, test_case_type, json_data, in_dir): +def create_evaluator_instance(metadata, test_case): #create_evaluator_instance """Create instance of relevant EvaluateCode class based on language""" registry = get_registry() - cls = registry.get_class(language, test_case_type) - instance = cls(in_dir) + cls = registry.get_class(metadata.get('language'), test_case.get('test_case_type')) + # instance = cls(in_dir) + instance = cls(metadata, test_case) return instance class _LanguageRegistry(object): @@ -36,8 +37,8 @@ class _LanguageRegistry(object): """ Get the code evaluator class for the given language """ if not self._register.get(language): self._register[language] = code_evaluators.get(language) - test_case_register = self._register[language] + cls = test_case_register.get(test_case_type) module_name, class_name = cls.rsplit(".", 1) # load the module, will raise ImportError if module cannot be loaded diff --git a/yaksh/old_code_server.py b/yaksh/old_code_server.py new file mode 100755 index 0000000..b3c9c30 --- /dev/null +++ b/yaksh/old_code_server.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python + +"""This server runs an HTTP server (using tornado) and several code servers +using XMLRPC that can be submitted code +and tests and returns the output. It *should* be run as root and will run as +the user 'nobody' so as to minimize any damange by errant code. This can be +configured by editing settings.py to run as many servers as desired. One can +also specify the ports on the command line. Here are examples:: + + $ sudo ./code_server.py + # Runs servers based on settings.py:SERVER_PORTS one server per port given. + +or:: + + $ sudo ./code_server.py 8001 8002 8003 8004 8005 + # Runs 5 servers on ports specified. + +All these servers should be running as nobody. This will also start a server +pool that defaults to port 50000 and is configurable in +settings.py:SERVER_POOL_PORT. This port exposes a `get_server_port` function +that returns an available server. + +""" + +# Standard library imports +from __future__ import unicode_literals +import json +from multiprocessing import Process, Queue +import os +from os.path import isdir, dirname, abspath, join, isfile +import pwd +import re +import signal +import stat +import subprocess +import sys + +try: + from SimpleXMLRPCServer import SimpleXMLRPCServer +except ImportError: + # The above import will not work on Python-3.x. + from xmlrpc.server import SimpleXMLRPCServer + +try: + from urllib import unquote +except ImportError: + # The above import will not work on Python-3.x. + from urllib.parse import unquote + +# Library imports +from tornado.ioloop import IOLoop +from tornado.web import Application, RequestHandler + +# Local imports +from .settings import SERVER_PORTS, SERVER_POOL_PORT +from .language_registry import create_evaluator_instance, unpack_json + + +MY_DIR = abspath(dirname(__file__)) + + +# Private Protocol ########## +def run_as_nobody(): + """Runs the current process as nobody.""" + # Set the effective uid and to that of nobody. + nobody = pwd.getpwnam('nobody') + os.setegid(nobody.pw_gid) + os.seteuid(nobody.pw_uid) + + +############################################################################### +# `CodeServer` class. +############################################################################### +class CodeServer(object): + """A code server that executes user submitted test code, tests it and + reports if the code was correct or not. + """ + def __init__(self, port, queue): + self.port = port + self.queue = queue + + # Public Protocol ########## + def check_code(self, language, test_case_type, json_data, in_dir=None): + """Calls relevant EvaluateCode class based on language to check the + answer code + """ + code_evaluator = create_evaluator_instance(language, + test_case_type, + json_data, + in_dir + ) + data = unpack_json(json_data) + result = code_evaluator.evaluate(**data) + + # Put us back into the server pool queue since we are free now. + self.queue.put(self.port) + + return json.dumps(result) + + def run(self): + """Run XMLRPC server, serving our methods.""" + server = SimpleXMLRPCServer(("0.0.0.0", self.port)) + self.server = server + server.register_instance(self) + self.queue.put(self.port) + server.serve_forever() + + +############################################################################### +# `ServerPool` class. +############################################################################### +class ServerPool(object): + """Manages a pool of CodeServer objects.""" + def __init__(self, ports, pool_port=50000): + """Create a pool of servers. Uses a shared Queue to get available + servers. + + Parameters + ---------- + + ports : list(int) + List of ports at which the CodeServer's should run. + + pool_port : int + Port at which the server pool should serve. + """ + self.my_port = pool_port + self.ports = ports + queue = Queue(maxsize=len(self.ports)) + self.queue = queue + servers = [] + processes = [] + for port in self.ports: + server = CodeServer(port, queue) + servers.append(server) + p = Process(target=server.run) + processes.append(p) + self.servers = servers + self.processes = processes + self.app = self._make_app() + + def _make_app(self): + app = Application([ + (r"/.*", MainHandler, dict(server=self)), + ]) + app.listen(self.my_port) + return app + + def _start_code_servers(self): + for proc in self.processes: + if proc.pid is None: + proc.start() + + # Public Protocol ########## + + def get_server_port(self): + """Get available server port from ones in the pool. This will block + till it gets an available server. + """ + return self.queue.get() + + def get_status(self): + """Returns current queue size and total number of ports used.""" + try: + qs = self.queue.qsize() + except NotImplementedError: + # May not work on OS X so we return a dummy. + qs = len(self.ports) + + return qs, len(self.ports) + + def run(self): + """Run server which returns an available server port where code + can be executed. + """ + # We start the code servers here to ensure they are run as nobody. + self._start_code_servers() + IOLoop.current().start() + + def stop(self): + """Stop all the code server processes. + """ + for proc in self.processes: + proc.terminate() + IOLoop.current().stop() + + +class MainHandler(RequestHandler): + def initialize(self, server): + self.server = server + + def get(self): + path = self.request.path[1:] + if len(path) == 0: + port = self.server.get_server_port() + self.write(str(port)) + elif path == "status": + q_size, total = self.server.get_status() + result = "%d servers out of %d are free.\n"%(q_size, total) + load = float(total - q_size)/total*100 + result += "Load: %s%%\n"%load + self.write(result) + + +############################################################################### +def main(args=None): + if args: + ports = [int(x) for x in args] + else: + ports = SERVER_PORTS + + server_pool = ServerPool(ports=ports, pool_port=SERVER_POOL_PORT) + # This is done *after* the server pool is created because when the tornado + # app calls listen(), it cannot be nobody. + run_as_nobody() + + server_pool.run() + +if __name__ == '__main__': + args = sys.argv[1:] + main(args) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 986dbf2..5f1b29e 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -11,32 +11,60 @@ from .code_evaluator import CodeEvaluator, TimeoutException from .file_utils import copy_files, delete_files -class PythonAssertionEvaluator(CodeEvaluator): +class PythonAssertionEvaluator(object): """Tests the Python code obtained from Code Server""" - def setup(self): - super(PythonAssertionEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.exec_scope = None self.files = [] - def teardown(self): + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('test_case') + self.weight = test_case_data.get('weight') + + def __del__(self): # Delete the created file. if self.files: delete_files(self.files) - super(PythonAssertionEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, test_case, weight): - if file_paths: - self.files = copy_files(file_paths) + + # def setup(self): + # super(PythonAssertionEvaluator, self).setup() + # self.exec_scope = None + # self.files = [] + + + # def teardown(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) + # super(PythonAssertionEvaluator, self).teardown() + + # def teardown(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) + # # Cancel the signal + # delete_signal_handler() + # self._change_dir(dirname(MY_DIR)) + + def compile_code(self): + if self.file_paths: + self.files = copy_files(self.file_paths) if self.exec_scope: return None else: - submitted = compile(user_answer, '', mode='exec') + submitted = compile(self.user_answer, '', mode='exec') self.exec_scope = {} exec(submitted, self.exec_scope) return self.exec_scope - def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): + def check_code(self): """ Function validates user answer by running an assertion based test case against it @@ -61,26 +89,21 @@ class PythonAssertionEvaluator(CodeEvaluator): test_case_weight = 0.0 try: tb = None - _tests = compile(test_case, '', mode='exec') + _tests = compile(self.test_case, '', mode='exec') exec(_tests, self.exec_scope) except AssertionError: type, value, tb = sys.exc_info() info = traceback.extract_tb(tb) fname, lineno, func, text = info[-1] - text = str(test_case).splitlines()[lineno-1] + text = str(self.test_case).splitlines()[lineno-1] err = ("-----\nExpected Test Case:\n{0}\n" - "Error - {1} {2} in: {3}\n-----").format(test_case, - type.__name__, - str(value), text - ) - except TimeoutException: - raise + "Error - {1} {2} in: {3}\n-----").format(self.test_case, type.__name__, str(value), text) except Exception: msg = traceback.format_exc(limit=0) err = "Error in Test case: {0}".format(msg) else: success = True - err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(test_case) - test_case_weight = float(weight) if partial_grading else 0.0 + err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(self.test_case) + test_case_weight = float(self.weight) if self.partial_grading else 0.0 del tb return success, err, test_case_weight diff --git a/yaksh/settings.py b/yaksh/settings.py index 6383999..690ddb1 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -19,22 +19,28 @@ SERVER_TIMEOUT = 4 # host.org/foo/exam set URL_ROOT='/foo' URL_ROOT = '' +# code_evaluators = { +# "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", +# "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" +# }, +# "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", +# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" +# }, +# "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", +# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" +# }, +# "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", +# "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, + +# "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", +# "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" +# }, + +# "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, +# } + code_evaluators = { "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" - }, - "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" - }, - "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" - }, - "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", - "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, - - "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", - "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" - }, - - "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, + } } -- cgit From b59e11188609ef10150f76d75f75882f8ae20269 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 8 Dec 2016 14:41:49 +0530 Subject: Fix test cases to reflect code server and python evaluator changes --- yaksh/evaluator_tests/test_python_evaluation.py | 1012 +++++++++-------------- yaksh/python_assertion_evaluator.py | 8 +- yaksh/python_stdio_evaluator.py | 49 +- 3 files changed, 411 insertions(+), 658 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 43cd0ea..688002f 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -88,6 +88,148 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result.get('error') ) + def test_partial_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" + test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': True + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('weight'), 2.0) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) + + def test_infinite_loop(self): + # Given + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) + + def test_syntax_error(self): + # Given + user_answer = dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "", + "SyntaxError", + "invalid syntax" + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) + + def test_indent_error(self): + # Given + user_answer = dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = ["Traceback", "call", + "File", + "line", + "", + "IndentationError", + "indented block" + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in indent_error_msg: + self.assertIn(msg, result.get("error")) def test_name_error(self): # Given @@ -98,20 +240,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "name", "defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) @@ -125,15 +277,25 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "call", "maximum recursion depth exceeded" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -152,20 +314,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "TypeError", "argument" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(3, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error")) @@ -182,41 +354,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid literal", "base" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(4, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] + self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertIn("Correct answer", result.get('error')) @@ -227,7 +419,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nasert palindrome(s)==False', "weight": 0.0 } ] @@ -239,20 +432,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "SyntaxError", "invalid syntax" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(6, len(err)) + self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error")) @@ -262,45 +465,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(palindrome("abba")==True)', "weight": 0.0 }, - {"test_case": 's="abbb"\nassert palindrome(S)==False', + {"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nassert palindrome(S)==False', "weight": 0.0 } ] name_error_msg = ["Traceback", "call", + "File", + "line", + "", "NameError", "name 'S' is not defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(7, len(err)) + self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) - class PythonStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) self.file_paths = None + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path def test_correct_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -310,14 +529,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -325,7 +552,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given - self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]", "weight": 0.0 }] @@ -338,14 +566,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -353,7 +590,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given - self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2", "weight": 0.0 }] @@ -364,14 +602,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a.count(b)) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -379,7 +626,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_incorrect_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -389,14 +637,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a-b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -404,7 +660,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given - self.test_case_data = [{"expected_input": "", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "", "expected_output": "2", "weight": 0.0 }] @@ -416,15 +673,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a[0]) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertEqual(result.get('error'), "Correct answer\n") @@ -432,7 +697,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): # Given - test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -440,552 +706,28 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'partial_grading': False + # } + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertEqual(result.get('error'), timeout_msg) self.assertFalse(result.get('success')) -# class PythonAssertionEvaluationTestCases(unittest.TestCase): -# def setUp(self): -# with open('/tmp/test.txt', 'wb') as f: -# f.write('2'.encode('ascii')) -# tmp_in_dir_path = tempfile.mkdtemp() -# self.in_dir = tmp_in_dir_path -# self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, -# {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, -# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, -# ] -# self.timeout_msg = ("Code took more than {0} seconds to run. " -# "You probably have an infinite loop in" -# " your code.").format(SERVER_TIMEOUT) -# self.file_paths = None - -# def tearDown(self): -# os.remove('/tmp/test.txt') -# shutil.rmtree(self.in_dir) - -# def test_correct_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn a + b" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_incorrect_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn a - b" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertIn('AssertionError in: assert(add(1,2)==3)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,2)==1)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', -# result.get('error') -# ) - -# def test_partial_incorrect_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" -# test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, -# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, -# {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': True -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertEqual(result.get('weight'), 2.0) -# self.assertIn('AssertionError in: assert(add(-1,2)==1)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', -# result.get('error') -# ) - -# def test_infinite_loop(self): -# # Given -# user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertEqual(result.get('error'), self.timeout_msg) - -# def test_syntax_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b); -# return a + b -# """) -# syntax_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "", -# "SyntaxError", -# "invalid syntax" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in syntax_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_indent_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# return a + b -# """) -# indent_error_msg = ["Traceback", "call", -# "File", -# "line", -# "", -# "IndentationError", -# "indented block" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False - -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in indent_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_name_error(self): -# # Given -# user_answer = "" -# name_error_msg = ["Traceback", -# "call", -# "NameError", -# "name", -# "defined" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in name_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_recursion_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# return add(3, 3) -# """) -# recursion_error_msg = ["Traceback", -# "call", -# "maximum recursion depth exceeded" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# for msg in recursion_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_type_error(self): -# # Given -# user_answer = dedent(""" -# def add(a): -# return a + b -# """) -# type_error_msg = ["Traceback", -# "call", -# "TypeError", -# "argument" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in type_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_value_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# c = 'a' -# return int(a) + int(b) + int(c) -# """) -# value_error_msg = ["Traceback", -# "call", -# "ValueError", -# "invalid literal", -# "base" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(4, len(err)) -# for msg in value_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_file_based_assert(self): -# # Given -# self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] -# self.file_paths = [('/tmp/test.txt', False)] -# user_answer = dedent(""" -# def ans(): -# with open("test.txt") as f: -# return f.read()[0] -# """) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertIn("Correct answer", result.get('error')) -# self.assertTrue(result.get('success')) - -# def test_single_testcase_error(self): -# # Given -# """ Tests the user answer with just an incorrect test case """ - -# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" -# test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', -# "weight": 0.0 -# } -# ] -# syntax_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "", -# "SyntaxError", -# "invalid syntax" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in syntax_error_msg: -# self.assertIn(msg, result.get("error")) - - -# def test_multiple_testcase_error(self): -# """ Tests the user answer with an correct test case -# first and then with an incorrect test case """ -# # Given -# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" -# test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', -# "weight": 0.0 -# }, -# {"test_case": 's="abbb"\nassert palindrome(S)==False', -# "weight": 0.0 -# } -# ] -# name_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "", -# "NameError", -# "name 'S' is not defined" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in name_error_msg: -# self.assertIn(msg, result.get("error")) - - -# class PythonStdIOEvaluationTestCases(unittest.TestCase): -# def setUp(self): -# with open('/tmp/test.txt', 'wb') as f: -# f.write('2'.encode('ascii')) -# self.file_paths = None - -# def test_correct_answer_integer(self): -# # Given -# self.test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# a = int(input()) -# b = int(input()) -# print(a+b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_correct_answer_list(self): -# # Given -# self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", -# "expected_output": "[1, 2, 3, 5, 6, 7]", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# from six.moves import input -# input_a = input() -# input_b = input() -# a = [int(i) for i in input_a.split(',')] -# b = [int(i) for i in input_b.split(',')] -# print(a+b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_correct_answer_string(self): -# # Given -# self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), -# "expected_output": "2", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# from six.moves import input -# a = str(input()) -# b = str(input()) -# print(a.count(b)) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_incorrect_answer_integer(self): -# # Given -# self.test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# a = int(input()) -# b = int(input()) -# print(a-b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertIn("Incorrect answer", result.get('error')) - -# def test_file_based_answer(self): -# # Given -# self.test_case_data = [{"expected_input": "", -# "expected_output": "2", -# "weight": 0.0 -# }] -# self.file_paths = [('/tmp/test.txt', False)] - -# user_answer = dedent(""" -# with open("test.txt") as f: -# a = f.read() -# print(a[0]) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertEqual(result.get('error'), "Correct answer\n") -# self.assertTrue(result.get('success')) - -# def test_infinite_loop(self): -# # Given -# test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# timeout_msg = ("Code took more than {0} seconds to run. " -# "You probably have an infinite loop in" -# " your code.").format(SERVER_TIMEOUT) -# user_answer = "while True:\n\tpass" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertEqual(result.get('error'), timeout_msg) -# self.assertFalse(result.get('success')) if __name__ == '__main__': unittest.main() diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 5f1b29e..3e172ec 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -27,10 +27,10 @@ class PythonAssertionEvaluator(object): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - def __del__(self): - # Delete the created file. - if self.files: - delete_files(self.files) + # def __del__(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) # def setup(self): diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 1506685..b618a0b 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -31,24 +31,36 @@ def redirect_stdout(): class PythonStdioEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def setup(self): - super(PythonStdioEvaluator, self).setup() + # def setup(self): + # super(PythonStdioEvaluator, self).setup() + # self.files = [] + + # def teardown(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) + # super(PythonStdioEvaluator, self).teardown() + def __init__(self, metadata, test_case_data): self.files = [] - def teardown(self): - # Delete the created file. - if self.files: - delete_files(self.files) - super(PythonStdioEvaluator, self).teardown() + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.expected_input = test_case_data.get('expected_input') + self.expected_output = test_case_data.get('expected_output') + self.weight = test_case_data.get('weight') - def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): - if file_paths: - self.files = copy_files(file_paths) - submitted = compile(user_answer, '', mode='exec') - if expected_input: + def compile_code(self): # user_answer, file_paths, expected_input, expected_output, weight): + if self.file_paths: + self.files = copy_files(self.file_paths) + submitted = compile(self.user_answer, '', mode='exec') + if self.expected_input: input_buffer = StringIO() - input_buffer.write(expected_input) + input_buffer.write(self.expected_input) input_buffer.seek(0) sys.stdin = input_buffer with redirect_stdout() as output_buffer: @@ -57,16 +69,15 @@ class PythonStdioEvaluator(CodeEvaluator): self.output_value = output_buffer.getvalue().rstrip("\n") return self.output_value - def check_code(self, user_answer, file_paths, partial_grading, expected_input, - expected_output, weight): + def check_code(self): # user_answer, file_paths, partial_grading, expected_input, expected_output, weight): success = False test_case_weight = 0.0 tb = None - if self.output_value == expected_output: + if self.output_value == self.expected_output: success = True err = "Correct answer" - test_case_weight = weight + test_case_weight = self.weight else: success = False err = dedent(""" @@ -74,8 +85,8 @@ class PythonStdioEvaluator(CodeEvaluator): Given input - {0} Expected output - {1} Your output - {2} - """.format(expected_input, - expected_output, + """.format(self.expected_input, + self.expected_output, self.output_value ) ) -- cgit From 80a4feef3c209e044e8cbe31e44c81d69136e100 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 15 Dec 2016 16:34:18 +0530 Subject: Add further changes to code evaluator --- yaksh/base_evaluator.py | 66 +++++++++++++ yaksh/code_evaluator.py | 118 +++++++++++++----------- yaksh/code_server.py | 5 +- yaksh/cpp_code_evaluator.py | 46 ++++++--- yaksh/evaluator_tests/test_python_evaluation.py | 13 +-- yaksh/models.py | 41 +++++--- yaksh/python_assertion_evaluator.py | 12 ++- yaksh/python_stdio_evaluator.py | 4 +- 8 files changed, 205 insertions(+), 100 deletions(-) create mode 100644 yaksh/base_evaluator.py (limited to 'yaksh') diff --git a/yaksh/base_evaluator.py b/yaksh/base_evaluator.py new file mode 100644 index 0000000..c8177b7 --- /dev/null +++ b/yaksh/base_evaluator.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +from __future__ import unicode_literals +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess + +class BaseEvaluator(object): + """Base Evaluator class containing generic attributes and callable methods""" + + def __init__(self): + pass + + def check_code(self): + raise NotImplementedError("check_code method not implemented") + + def compile_code(self): + pass + + def _run_command(self, cmd_args, *args, **kw): + """Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stdout and stderr. + """ + try: + proc = subprocess.Popen(cmd_args, *args, **kw) + stdout, stderr = proc.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc.kill() + # Re-raise exception. + raise + return proc, stdout.decode('utf-8'), stderr.decode('utf-8') + + def _remove_null_substitute_char(self, string): + """Returns a string without any null and substitute characters""" + stripped = "" + for c in string: + if ord(c) is not 26 and ord(c) is not 0: + stripped = stripped + c + return ''.join(stripped) + + def create_submit_code_file(self, file_name): + """ Set the file path for code (`answer`)""" + submit_path = abspath(file_name) + if not exists(submit_path): + submit_f = open(submit_path, 'w') + submit_f.close() + + return submit_path + + def write_to_submit_code_file(self, file_path, user_answer): + """ Write the code (`answer`) to a file""" + submit_f = open(file_path, 'w') + submit_f.write(user_answer.lstrip()) + submit_f.close() + + def _set_test_code_file_path(self, ref_path=None, test_case_path=None): + if ref_path and not ref_path.startswith('/'): + ref_path = join(MY_DIR, ref_path) + + if test_case_path and not test_case_path.startswith('/'): + test_case_path = join(MY_DIR, test_case_path) + + return ref_path, test_case_path diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 5ede63d..e5b8853 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -89,11 +89,9 @@ class CodeEvaluator(object): A tuple: (success, error message, weight). """ - # self.language = language - # self.test_case_type = test_case_type - + test_case_instances = self.get_evaluator_objects(kwargs) self.setup() - success, error, weight = self.safe_evaluate(**kwargs) + success, error, weight = self.safe_evaluate(test_case_instances) self.teardown() result = {'success': success, 'error': error, 'weight': weight} @@ -106,29 +104,43 @@ class CodeEvaluator(object): os.makedirs(self.in_dir) self._change_dir(self.in_dir) - def safe_evaluate(self, **kwargs): #user_answer, partial_grading, test_case_data, file_paths=None + def get_evaluator_objects(self, kwargs): + metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths + test_case_data = kwargs.get('test_case_data') + test_case_instances = [] + + for test_case in test_case_data: + test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case + test_case_instances.append(test_case_instance) + + return test_case_instances + + + def safe_evaluate(self, test_case_instances): #user_answer, partial_grading, test_case_data, file_paths=None """ Handles code evaluation along with compilation, signal handling and Exception handling """ - metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths - test_case_data = kwargs.get('test_case_data') + # metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths + # test_case_data = kwargs.get('test_case_data') # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False - test_case_success_status = [False] * len(test_case_data) + test_case_success_status = [False] * len(test_case_instances) error = "" weight = 0.0 # Do whatever testing needed. try: # Run evaluator selection registry here - for idx, test_case in enumerate(test_case_data): - test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case + for idx, test_case_instance in enumerate(test_case_instances): + # test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case + # self.setup() test_case_success = False test_case_instance.compile_code() #user_answer, file_paths, test_case test_case_success, err, test_case_weight = test_case_instance.check_code() #**kwargs + # self.teardown() # user_answer, # file_paths, # partial_grading, @@ -213,64 +225,64 @@ class CodeEvaluator(object): delete_signal_handler() self._change_dir(dirname(MY_DIR)) - def check_code(self): - raise NotImplementedError("check_code method not implemented") + # def check_code(self): + # raise NotImplementedError("check_code method not implemented") - def compile_code(self, user_answer, file_paths, **kwargs): - pass + # def compile_code(self, user_answer, file_paths, **kwargs): + # pass - def create_submit_code_file(self, file_name): - """ Set the file path for code (`answer`)""" - submit_path = abspath(file_name) - if not exists(submit_path): - submit_f = open(submit_path, 'w') - submit_f.close() + # def create_submit_code_file(self, file_name): + # """ Set the file path for code (`answer`)""" + # submit_path = abspath(file_name) + # if not exists(submit_path): + # submit_f = open(submit_path, 'w') + # submit_f.close() - return submit_path + # return submit_path - def write_to_submit_code_file(self, file_path, user_answer): - """ Write the code (`answer`) to a file""" - submit_f = open(file_path, 'w') - submit_f.write(user_answer.lstrip()) - submit_f.close() + # def write_to_submit_code_file(self, file_path, user_answer): + # """ Write the code (`answer`) to a file""" + # submit_f = open(file_path, 'w') + # submit_f.write(user_answer.lstrip()) + # submit_f.close() def _set_file_as_executable(self, fname): os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) - def _set_test_code_file_path(self, ref_path=None, test_case_path=None): - if ref_path and not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) + # def _set_test_code_file_path(self, ref_path=None, test_case_path=None): + # if ref_path and not ref_path.startswith('/'): + # ref_path = join(MY_DIR, ref_path) - if test_case_path and not test_case_path.startswith('/'): - test_case_path = join(MY_DIR, test_case_path) + # if test_case_path and not test_case_path.startswith('/'): + # test_case_path = join(MY_DIR, test_case_path) - return ref_path, test_case_path + # return ref_path, test_case_path - def _run_command(self, cmd_args, *args, **kw): - """Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stdout and stderr. - """ - try: - proc = subprocess.Popen(cmd_args, *args, **kw) - stdout, stderr = proc.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc.kill() - # Re-raise exception. - raise - return proc, stdout.decode('utf-8'), stderr.decode('utf-8') + # def _run_command(self, cmd_args, *args, **kw): + # """Run a command in a subprocess while blocking, the process is killed + # if it takes more than 2 seconds to run. Return the Popen object, the + # stdout and stderr. + # """ + # try: + # proc = subprocess.Popen(cmd_args, *args, **kw) + # stdout, stderr = proc.communicate() + # except TimeoutException: + # # Runaway code, so kill it. + # proc.kill() + # # Re-raise exception. + # raise + # return proc, stdout.decode('utf-8'), stderr.decode('utf-8') def _change_dir(self, in_dir): if in_dir is not None and isdir(in_dir): os.chdir(in_dir) - def _remove_null_substitute_char(self, string): - """Returns a string without any null and substitute characters""" - stripped = "" - for c in string: - if ord(c) is not 26 and ord(c) is not 0: - stripped = stripped + c - return ''.join(stripped) + # def _remove_null_substitute_char(self, string): + # """Returns a string without any null and substitute characters""" + # stripped = "" + # for c in string: + # if ord(c) is not 26 and ord(c) is not 0: + # stripped = stripped + c + # return ''.join(stripped) diff --git a/yaksh/code_server.py b/yaksh/code_server.py index abe7cd8..3c1a3e3 100644 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -54,6 +54,7 @@ from tornado.web import Application, RequestHandler # Local imports from .settings import SERVER_PORTS, SERVER_POOL_PORT from .language_registry import create_evaluator_instance +from .code_evaluator import CodeEvaluator MY_DIR = abspath(dirname(__file__)) @@ -89,9 +90,9 @@ class CodeServer(object): # json_data, # in_dir # ) - data = unpack_json_to_python_obj(json_data) + data = self.unpack_json_to_python_obj(json_data) code_eval_instance = CodeEvaluator(in_dir) - result = code_eval_instance.evaluate(**data) #language, test_case_type, + result = code_eval_instance.evaluate(data) #language, test_case_type, # Put us back into the server pool queue since we are free now. self.queue.put(self.port) diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 716a522..806fe67 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -5,17 +5,15 @@ import pwd import os from os.path import join, isfile import subprocess -import importlib # Local imports -from .code_evaluator import CodeEvaluator from .file_utils import copy_files, delete_files +from .base_evaluator import BaseEvaluator -class CppCodeEvaluator(CodeEvaluator): +class CppCodeEvaluator(BaseEvaluator): """Tests the C code obtained from Code Server""" - def setup(self): - super(CppCodeEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] self.submit_code_path = self.create_submit_code_file('submit.c') self.compiled_user_answer = None @@ -23,16 +21,34 @@ class CppCodeEvaluator(CodeEvaluator): self.user_output_path = "" self.ref_output_path = "" - def teardown(self): - # Delete the created file. - os.remove(self.submit_code_path) - if os.path.exists(self.ref_output_path): - os.remove(self.ref_output_path) - if os.path.exists(self.user_output_path): - os.remove(self.user_output_path) - if self.files: - delete_files(self.files) - super(CppCodeEvaluator, self).teardown() + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('test_case') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(CppCodeEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('submit.c') + # self.compiled_user_answer = None + # self.compiled_test_code = None + # self.user_output_path = "" + # self.ref_output_path = "" + + # def teardown(self): + # # Delete the created file. + # os.remove(self.submit_code_path) + # if os.path.exists(self.ref_output_path): + # os.remove(self.ref_output_path) + # if os.path.exists(self.user_output_path): + # os.remove(self.user_output_path) + # if self.files: + # delete_files(self.files) + # super(CppCodeEvaluator, self).teardown() def set_file_paths(self): diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 688002f..4bf0032 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -263,7 +263,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) + self.assertEqual(9, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) @@ -337,7 +337,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) + self.assertEqual(9, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error")) @@ -377,7 +377,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(4, len(err)) + self.assertEqual(9, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) @@ -455,7 +455,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) + self.assertEqual(6, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error")) @@ -476,9 +476,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] name_error_msg = ["Traceback", "call", - "File", - "line", - "", "NameError", "name 'S' is not defined" ] @@ -504,7 +501,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) + self.assertEqual(7, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) diff --git a/yaksh/models.py b/yaksh/models.py index 7fae305..4951836 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -255,6 +255,7 @@ class Question(models.Model): def consolidate_answer_data(self, user_answer): question_data = {} + metadata = {} test_case_data = [] test_cases = self.get_test_cases() @@ -264,12 +265,15 @@ class Question(models.Model): test_case_data.append(test_case_as_dict) question_data['test_case_data'] = test_case_data - question_data['user_answer'] = user_answer - question_data['partial_grading'] = self.partial_grading + metadata['user_answer'] = user_answer + metadata['language'] = self.language + metadata['partial_grading'] = self.partial_grading files = FileUpload.objects.filter(question=self) if files: - question_data['file_paths'] = [(file.file.path, file.extract) + metadata['file_paths'] = [(file.file.path, file.extract) for file in files] + question_data['metadata'] = metadata + return json.dumps(question_data) @@ -309,15 +313,20 @@ class Question(models.Model): delete_files(files_list, file_path) def get_test_cases(self, **kwargs): - test_case_ctype = ContentType.objects.get(app_label="yaksh", - model=self.test_case_type - ) - test_cases = test_case_ctype.get_all_objects_for_this_type( - question=self, - **kwargs - ) - - return test_cases + # test_case_ctype = ContentType.objects.get(app_label="yaksh", + # model=self.test_case_type + # ) + # test_cases = test_case_ctype.get_all_objects_for_this_type( + # question=self, + # **kwargs + # ) + tc_list = [] + for tc in self.testcase_set.all(): + tc_type = tc.type + obj = getattr(tc, tc_type) + tc_list.append(obj) + + return tc_list def get_test_case(self, **kwargs): test_case_ctype = ContentType.objects.get(app_label="yaksh", @@ -1137,7 +1146,8 @@ class StandardTestCase(TestCase): weight = models.FloatField(default=1.0) def get_field_value(self): - return {"test_case": self.test_case, + return {"test_case_type": "standardtestcase", + "test_case": self.test_case, "weight": self.weight} def __str__(self): @@ -1152,7 +1162,8 @@ class StdioBasedTestCase(TestCase): weight = models.IntegerField(default=1.0) def get_field_value(self): - return {"expected_output": self.expected_output, + return {"test_case_type": "stdiobasedtestcase", + "expected_output": self.expected_output, "expected_input": self.expected_input, "weight": self.weight} @@ -1167,7 +1178,7 @@ class McqTestCase(TestCase): correct = models.BooleanField(default=False) def get_field_value(self): - return {"options": self.options, "correct": self.correct} + return {"test_case_type": "mcqtestcase", "options": self.options, "correct": self.correct} def __str__(self): return u'Question: {0} | Correct: {1}'.format(self.question, diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 3e172ec..eb13f53 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -7,11 +7,12 @@ from os.path import join import importlib # Local imports -from .code_evaluator import CodeEvaluator, TimeoutException from .file_utils import copy_files, delete_files +from .base_evaluator import BaseEvaluator +from .code_evaluator import TimeoutException -class PythonAssertionEvaluator(object): +class PythonAssertionEvaluator(BaseEvaluator): """Tests the Python code obtained from Code Server""" def __init__(self, metadata, test_case_data): @@ -32,7 +33,6 @@ class PythonAssertionEvaluator(object): # if self.files: # delete_files(self.files) - # def setup(self): # super(PythonAssertionEvaluator, self).setup() # self.exec_scope = None @@ -98,9 +98,11 @@ class PythonAssertionEvaluator(object): text = str(self.test_case).splitlines()[lineno-1] err = ("-----\nExpected Test Case:\n{0}\n" "Error - {1} {2} in: {3}\n-----").format(self.test_case, type.__name__, str(value), text) + except TimeoutException: + raise except Exception: - msg = traceback.format_exc(limit=0) - err = "Error in Test case: {0}".format(msg) + msg = traceback.format_exc(limit=0) + err = "Error in Test case: {0}".format(msg) else: success = True err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(self.test_case) diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index b618a0b..7ef3a7c 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -14,8 +14,8 @@ except ImportError: from io import StringIO # Local imports -from .code_evaluator import CodeEvaluator from .file_utils import copy_files, delete_files +from .base_evaluator import BaseEvaluator @contextmanager @@ -28,7 +28,7 @@ def redirect_stdout(): sys.stdout = old_target # restore to the previous value -class PythonStdioEvaluator(CodeEvaluator): +class PythonStdioEvaluator(BaseEvaluator): """Tests the Python code obtained from Code Server""" # def setup(self): -- cgit From f1da39aded67efa3da145851f0e9f687a3e434e5 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 19 Dec 2016 11:44:55 +0530 Subject: Change all evaluator structure and make sure eval test cases pass --- yaksh/base_evaluator.py | 11 + yaksh/bash_code_evaluator.py | 54 +- yaksh/bash_stdio_evaluator.py | 45 +- yaksh/code_evaluator.py | 34 +- yaksh/cpp_code_evaluator.py | 34 +- yaksh/cpp_stdio_evaluator.py | 39 +- .../evaluator_tests/old_test_python_evaluation.py | 549 --------------------- yaksh/evaluator_tests/test_bash_evaluation.py | 211 ++++++-- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 468 ++++++++++++++---- yaksh/evaluator_tests/test_code_evaluation.py | 12 +- yaksh/evaluator_tests/test_java_evaluation.py | 371 ++++++++++---- yaksh/evaluator_tests/test_python_evaluation.py | 2 - yaksh/evaluator_tests/test_scilab_evaluation.py | 107 +++- yaksh/java_code_evaluator.py | 46 +- yaksh/java_stdio_evaluator.py | 41 +- yaksh/old_code_server.py | 221 --------- yaksh/python_assertion_evaluator.py | 17 +- yaksh/python_stdio_evaluator.py | 9 +- yaksh/scilab_code_evaluator.py | 44 +- yaksh/settings.py | 14 +- yaksh/stdio_evaluator.py | 12 +- 21 files changed, 1132 insertions(+), 1209 deletions(-) delete mode 100644 yaksh/evaluator_tests/old_test_python_evaluation.py delete mode 100755 yaksh/old_code_server.py (limited to 'yaksh') diff --git a/yaksh/base_evaluator.py b/yaksh/base_evaluator.py index c8177b7..b290ba4 100644 --- a/yaksh/base_evaluator.py +++ b/yaksh/base_evaluator.py @@ -4,7 +4,13 @@ import traceback import pwd import os from os.path import join, isfile +from os.path import isdir, dirname, abspath, join, isfile, exists import subprocess +import stat + + +# Local imports +from .code_evaluator import MY_DIR, TimeoutException class BaseEvaluator(object): """Base Evaluator class containing generic attributes and callable methods""" @@ -64,3 +70,8 @@ class BaseEvaluator(object): test_case_path = join(MY_DIR, test_case_path) return ref_path, test_case_path + + def _set_file_as_executable(self, fname): + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index b5974d2..03ec16a 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -9,26 +9,37 @@ import subprocess import importlib # local imports -from .code_evaluator import CodeEvaluator +from .base_evaluator import BaseEvaluator from .file_utils import copy_files, delete_files -class BashCodeEvaluator(CodeEvaluator): +class BashCodeEvaluator(BaseEvaluator): # Private Protocol ########## - def setup(self): - super(BashCodeEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = self.create_submit_code_file('submit.sh') - self._set_file_as_executable(self.submit_code_path) + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('test_case') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(BashCodeEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('submit.sh') + # self._set_file_as_executable(self.submit_code_path) def teardown(self): # Delete the created file. os.remove(self.submit_code_path) if self.files: delete_files(self.files) - super(BashCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): + def check_code(self): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -53,18 +64,21 @@ class BashCodeEvaluator(CodeEvaluator): Returns (False, error_msg, 0.0): If mandatory arguments are not files or if the required permissions are not given to the file(s). """ - ref_code_path = test_case + ref_code_path = self.test_case success = False test_case_weight = 0.0 + self.submit_code_path = self.create_submit_code_file('submit.sh') + self._set_file_as_executable(self.submit_code_path) + get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() get_test_case_path = get_test_case_path.strip() clean_ref_code_path, clean_test_case_path = \ self._set_test_code_file_path(get_ref_path, get_test_case_path) - if file_paths: - self.files = copy_files(file_paths) + if self.file_paths: + self.files = copy_files(self.file_paths) if not isfile(clean_ref_code_path): msg = "No file at %s or Incorrect path" % clean_ref_code_path return False, msg, 0.0 @@ -78,8 +92,8 @@ class BashCodeEvaluator(CodeEvaluator): msg = "Script %s is not executable" % self.submit_code_path return False, msg, 0.0 - user_answer = user_answer.replace("\r", "") - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_answer = self.user_answer.replace("\r", "") + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) if clean_test_case_path is None or "": ret = self._run_command(clean_ref_code_path, @@ -95,7 +109,7 @@ class BashCodeEvaluator(CodeEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - test_case_weight = float(weight) if partial_grading else 0.0 + test_case_weight = float(self.weight) if self.partial_grading else 0.0 return True, "Correct answer", test_case_weight else: err = "Error: expected %s, got %s" % (inst_stderr, @@ -116,21 +130,21 @@ class BashCodeEvaluator(CodeEvaluator): loop_count = 0 test_cases = open(clean_test_case_path).readlines() num_lines = len(test_cases) - for test_case in test_cases: + for tc in test_cases: loop_count += 1 if valid_answer: args = [clean_ref_code_path] + \ - [x for x in test_case.split()] + [x for x in tc.split()] ret = self._run_command(args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) proc, inst_stdout, inst_stderr = ret - if file_paths: - self.files = copy_files(file_paths) + if self.file_paths: + self.files = copy_files(self.file_paths) args = [self.submit_code_path] + \ - [x for x in test_case.split()] + [x for x in tc.split()] ret = self._run_command(args, stdin=None, stdout=subprocess.PIPE, @@ -138,7 +152,7 @@ class BashCodeEvaluator(CodeEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - test_case_weight = float(weight) if partial_grading else 0.0 + test_case_weight = float(self.weight) if self.partial_grading else 0.0 return True, "Correct answer", test_case_weight else: err = ("Error:expected" diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 1dd9fd5..3344c57 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -11,43 +11,54 @@ from .file_utils import copy_files, delete_files class BashStdioEvaluator(StdIOEvaluator): """Evaluates Bash StdIO based code""" - - def setup(self): - super(BashStdioEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = self.create_submit_code_file('Test.sh') + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.expected_input = test_case_data.get('expected_input') + self.expected_output = test_case_data.get('expected_output') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(BashStdioEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('Test.sh') def teardown(self): os.remove(self.submit_code_path) if self.files: delete_files(self.files) - super(BashStdioEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): - if file_paths: - self.files = copy_files(file_paths) + def compile_code(self): + self.submit_code_path = self.create_submit_code_file('Test.sh') + if self.file_paths: + self.files = copy_files(self.file_paths) if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path return False, msg user_code_directory = os.getcwd() + '/' - user_answer = user_answer.replace("\r", "") - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_answer = self.user_answer.replace("\r", "") + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) - def check_code(self, user_answer, file_paths, partial_grading, - expected_input, expected_output, weight): + def check_code(self): success = False test_case_weight = 0.0 - expected_input = str(expected_input).replace('\r', '') + self.expected_input = str(self.expected_input).replace('\r', '') proc = subprocess.Popen("bash ./Test.sh", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - success, err = self.evaluate_stdio(user_answer, proc, - expected_input, - expected_output + success, err = self.evaluate_stdio(self.user_answer, proc, + self.expected_input, + self.expected_output ) - test_case_weight = float(weight) if partial_grading and success else 0.0 + test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 return success, err, test_case_weight diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index e5b8853..f1ac5b7 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -4,6 +4,7 @@ import sys import pwd import os import stat +import contextlib from os.path import isdir, dirname, abspath, join, isfile, exists import signal import traceback @@ -31,6 +32,15 @@ registry = None class TimeoutException(Exception): pass +@contextlib.contextmanager +def change_dir(path): + cur_dir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(cur_dir) + def timeout_handler(signum, frame): """A handler for the ALARM signal.""" @@ -89,9 +99,10 @@ class CodeEvaluator(object): A tuple: (success, error message, weight). """ - test_case_instances = self.get_evaluator_objects(kwargs) self.setup() - success, error, weight = self.safe_evaluate(test_case_instances) + test_case_instances = self.get_evaluator_objects(kwargs) + with change_dir(self.in_dir): + success, error, weight = self.safe_evaluate(test_case_instances) self.teardown() result = {'success': success, 'error': error, 'weight': weight} @@ -102,7 +113,7 @@ class CodeEvaluator(object): if self.in_dir: if not os.path.exists(self.in_dir): os.makedirs(self.in_dir) - self._change_dir(self.in_dir) + # self._change_dir(self.in_dir) def get_evaluator_objects(self, kwargs): metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths @@ -140,6 +151,7 @@ class CodeEvaluator(object): test_case_success = False test_case_instance.compile_code() #user_answer, file_paths, test_case test_case_success, err, test_case_weight = test_case_instance.check_code() #**kwargs + test_case_instance.teardown() # self.teardown() # user_answer, # file_paths, @@ -223,7 +235,7 @@ class CodeEvaluator(object): def teardown(self): # Cancel the signal delete_signal_handler() - self._change_dir(dirname(MY_DIR)) + # self._change_dir(dirname(MY_DIR)) # def check_code(self): # raise NotImplementedError("check_code method not implemented") @@ -246,10 +258,10 @@ class CodeEvaluator(object): # submit_f.write(user_answer.lstrip()) # submit_f.close() - def _set_file_as_executable(self, fname): - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + # def _set_file_as_executable(self, fname): + # os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + # | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + # | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) # def _set_test_code_file_path(self, ref_path=None, test_case_path=None): # if ref_path and not ref_path.startswith('/'): @@ -275,9 +287,9 @@ class CodeEvaluator(object): # raise # return proc, stdout.decode('utf-8'), stderr.decode('utf-8') - def _change_dir(self, in_dir): - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) + # def _change_dir(self, in_dir): + # if in_dir is not None and isdir(in_dir): + # os.chdir(in_dir) # def _remove_null_substitute_char(self, string): # """Returns a string without any null and substitute characters""" diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 806fe67..c6f5a7e 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -39,17 +39,15 @@ class CppCodeEvaluator(BaseEvaluator): # self.user_output_path = "" # self.ref_output_path = "" - # def teardown(self): - # # Delete the created file. - # os.remove(self.submit_code_path) - # if os.path.exists(self.ref_output_path): - # os.remove(self.ref_output_path) - # if os.path.exists(self.user_output_path): - # os.remove(self.user_output_path) - # if self.files: - # delete_files(self.files) - # super(CppCodeEvaluator, self).teardown() - + def teardown(self): + # Delete the created file. + os.remove(self.submit_code_path) + if os.path.exists(self.ref_output_path): + os.remove(self.ref_output_path) + if os.path.exists(self.user_output_path): + os.remove(self.user_output_path) + if self.files: + delete_files(self.files) def set_file_paths(self): user_output_path = os.getcwd() + '/output_file' @@ -66,15 +64,15 @@ class CppCodeEvaluator(BaseEvaluator): ref_output_path) return compile_command, compile_main - def compile_code(self, user_answer, file_paths, test_case, weight): + def compile_code(self): if self.compiled_user_answer and self.compiled_test_code: return None else: - ref_code_path = test_case + ref_code_path = self.test_case clean_ref_code_path, clean_test_case_path = \ self._set_test_code_file_path(ref_code_path) - if file_paths: - self.files = copy_files(file_paths) + if self.file_paths: + self.files = copy_files(self.file_paths) if not isfile(clean_ref_code_path): msg = "No file at %s or Incorrect path" % clean_ref_code_path return False, msg @@ -82,7 +80,7 @@ class CppCodeEvaluator(BaseEvaluator): msg = "No file at %s or Incorrect path" % self.submit_code_path return False, msg - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) self.user_output_path, self.ref_output_path = self.set_file_paths() self.compile_command, self.compile_main = self.get_commands( clean_ref_code_path, @@ -105,7 +103,7 @@ class CppCodeEvaluator(BaseEvaluator): return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): + def check_code(self): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -145,7 +143,7 @@ class CppCodeEvaluator(BaseEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" - test_case_weight = float(weight) if partial_grading else 0.0 + test_case_weight = float(self.weight) if self.partial_grading else 0.0 else: err = "{0} \n {1}".format(stdout, stderr) else: diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 00fad92..a57afbe 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -11,17 +11,29 @@ from .file_utils import copy_files, delete_files class CppStdioEvaluator(StdIOEvaluator): """Evaluates C StdIO based code""" - - def setup(self): - super(CppStdioEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = self.create_submit_code_file('main.c') + self.submit_code_path = self.create_submit_code_file('submit.c') + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.expected_input = test_case_data.get('expected_input') + self.expected_output = test_case_data.get('expected_output') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(CppStdioEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('main.c') def teardown(self): os.remove(self.submit_code_path) if self.files: delete_files(self.files) - super(CppStdioEvaluator, self).teardown() def set_file_paths(self): user_output_path = os.getcwd() + '/output_file' @@ -35,13 +47,13 @@ class CppStdioEvaluator(StdIOEvaluator): ref_output_path) return compile_command, compile_main - def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): - if file_paths: + def compile_code(self): + if self.file_paths: self.files = copy_files(file_paths) if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path return False, msg - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) self.user_output_path, self.ref_output_path = self.set_file_paths() self.compile_command, self.compile_main = self.get_commands( self.user_output_path, @@ -61,8 +73,7 @@ class CppStdioEvaluator(StdIOEvaluator): ) return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, partial_grading, - expected_input, expected_output, weight): + def check_code(self): success = False test_case_weight = 0.0 @@ -78,9 +89,9 @@ class CppStdioEvaluator(StdIOEvaluator): stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - success, err = self.evaluate_stdio(user_answer, proc, - expected_input, - expected_output + success, err = self.evaluate_stdio(self.user_answer, proc, + self.expected_input, + self.expected_output ) os.remove(self.ref_output_path) else: @@ -106,5 +117,5 @@ class CppStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - test_case_weight = float(weight) if partial_grading and success else 0.0 + test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 return success, err, test_case_weight diff --git a/yaksh/evaluator_tests/old_test_python_evaluation.py b/yaksh/evaluator_tests/old_test_python_evaluation.py deleted file mode 100644 index 9796fa2..0000000 --- a/yaksh/evaluator_tests/old_test_python_evaluation.py +++ /dev/null @@ -1,549 +0,0 @@ -from __future__ import unicode_literals -import unittest -import os -import tempfile -import shutil -from textwrap import dedent - -# Local import -from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdio_evaluator import PythonStdioEvaluator -from yaksh.settings import SERVER_TIMEOUT - - -class PythonAssertionEvaluationTestCases(unittest.TestCase): - def setUp(self): - with open('/tmp/test.txt', 'wb') as f: - f.write('2'.encode('ascii')) - tmp_in_dir_path = tempfile.mkdtemp() - self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, - ] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) - self.file_paths = None - - def tearDown(self): - os.remove('/tmp/test.txt') - shutil.rmtree(self.in_dir) - - def test_correct_answer(self): - # Given - user_answer = "def add(a,b):\n\treturn a + b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) - - def test_incorrect_answer(self): - # Given - user_answer = "def add(a,b):\n\treturn a - b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertIn('AssertionError in: assert(add(1,2)==3)', - result.get('error') - ) - self.assertIn('AssertionError in: assert(add(-1,2)==1)', - result.get('error') - ) - self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', - result.get('error') - ) - - def test_partial_incorrect_answer(self): - # Given - user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" - test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, - {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': True - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertEqual(result.get('weight'), 2.0) - self.assertIn('AssertionError in: assert(add(-1,2)==1)', - result.get('error') - ) - self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', - result.get('error') - ) - - def test_infinite_loop(self): - # Given - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), self.timeout_msg) - - def test_syntax_error(self): - # Given - user_answer = dedent(""" - def add(a, b); - return a + b - """) - syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "", - "SyntaxError", - "invalid syntax" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) - for msg in syntax_error_msg: - self.assertIn(msg, result.get("error")) - - def test_indent_error(self): - # Given - user_answer = dedent(""" - def add(a, b): - return a + b - """) - indent_error_msg = ["Traceback", "call", - "File", - "line", - "", - "IndentationError", - "indented block" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) - for msg in indent_error_msg: - self.assertIn(msg, result.get("error")) - - def test_name_error(self): - # Given - user_answer = "" - name_error_msg = ["Traceback", - "call", - "NameError", - "name", - "defined" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) - for msg in name_error_msg: - self.assertIn(msg, result.get("error")) - - def test_recursion_error(self): - # Given - user_answer = dedent(""" - def add(a, b): - return add(3, 3) - """) - recursion_error_msg = ["Traceback", - "call", - "maximum recursion depth exceeded" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - for msg in recursion_error_msg: - self.assertIn(msg, result.get("error")) - - def test_type_error(self): - # Given - user_answer = dedent(""" - def add(a): - return a + b - """) - type_error_msg = ["Traceback", - "call", - "TypeError", - "argument" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) - for msg in type_error_msg: - self.assertIn(msg, result.get("error")) - - def test_value_error(self): - # Given - user_answer = dedent(""" - def add(a, b): - c = 'a' - return int(a) + int(b) + int(c) - """) - value_error_msg = ["Traceback", - "call", - "ValueError", - "invalid literal", - "base" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(4, len(err)) - for msg in value_error_msg: - self.assertIn(msg, result.get("error")) - - def test_file_based_assert(self): - # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] - self.file_paths = [('/tmp/test.txt', False)] - user_answer = dedent(""" - def ans(): - with open("test.txt") as f: - return f.read()[0] - """) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertIn("Correct answer", result.get('error')) - self.assertTrue(result.get('success')) - - def test_single_testcase_error(self): - # Given - """ Tests the user answer with just an incorrect test case """ - - user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', - "weight": 0.0 - } - ] - syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "", - "SyntaxError", - "invalid syntax" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) - for msg in syntax_error_msg: - self.assertIn(msg, result.get("error")) - - - def test_multiple_testcase_error(self): - """ Tests the user answer with an correct test case - first and then with an incorrect test case """ - # Given - user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', - "weight": 0.0 - }, - {"test_case": 's="abbb"\nassert palindrome(S)==False', - "weight": 0.0 - } - ] - name_error_msg = ["Traceback", - "call", - "File", - "line", - "", - "NameError", - "name 'S' is not defined" - ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) - err = result.get("error").splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(3, len(err)) - for msg in name_error_msg: - self.assertIn(msg, result.get("error")) - - -class PythonStdIOEvaluationTestCases(unittest.TestCase): - def setUp(self): - with open('/tmp/test.txt', 'wb') as f: - f.write('2'.encode('ascii')) - self.file_paths = None - - def test_correct_answer_integer(self): - # Given - self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3", - "weight": 0.0 - }] - user_answer = dedent(""" - a = int(input()) - b = int(input()) - print(a+b) - """ - ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) - - def test_correct_answer_list(self): - # Given - self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", - "expected_output": "[1, 2, 3, 5, 6, 7]", - "weight": 0.0 - }] - user_answer = dedent(""" - from six.moves import input - input_a = input() - input_b = input() - a = [int(i) for i in input_a.split(',')] - b = [int(i) for i in input_b.split(',')] - print(a+b) - """ - ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) - - def test_correct_answer_string(self): - # Given - self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), - "expected_output": "2", - "weight": 0.0 - }] - user_answer = dedent(""" - from six.moves import input - a = str(input()) - b = str(input()) - print(a.count(b)) - """ - ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) - - def test_incorrect_answer_integer(self): - # Given - self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3", - "weight": 0.0 - }] - user_answer = dedent(""" - a = int(input()) - b = int(input()) - print(a-b) - """ - ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertIn("Incorrect answer", result.get('error')) - - def test_file_based_answer(self): - # Given - self.test_case_data = [{"expected_input": "", - "expected_output": "2", - "weight": 0.0 - }] - self.file_paths = [('/tmp/test.txt', False)] - - user_answer = dedent(""" - with open("test.txt") as f: - a = f.read() - print(a[0]) - """ - ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertEqual(result.get('error'), "Correct answer\n") - self.assertTrue(result.get('success')) - - def test_infinite_loop(self): - # Given - test_case_data = [{"expected_input": "1\n2", - "expected_output": "3", - "weight": 0.0 - }] - timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) - user_answer = "while True:\n\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'partial_grading': False - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertEqual(result.get('error'), timeout_msg) - self.assertFalse(result.get('success')) - -if __name__ == '__main__': - unittest.main() diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 99e5122..8888ee6 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -3,6 +3,7 @@ import unittest import os import shutil import tempfile +from yaksh.code_evaluator import CodeEvaluator from yaksh.bash_code_evaluator import BashCodeEvaluator from yaksh.bash_stdio_evaluator import BashStdioEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -15,6 +16,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] @@ -32,39 +34,78 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") def test_error(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -72,17 +113,31 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] user_answer = ("#!/bin/bash\ncat $1") - get_class = BashCodeEvaluator() - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") @@ -92,6 +147,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) + self.file_paths = None + def test_correct_answer(self): user_answer = dedent(""" #!/bin/bash @@ -102,14 +159,28 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -123,15 +194,29 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', - 'expected_input': '1,2,3\n4,5,6\n7,8,9', - 'weight': 0.0 + 'expected_input': '1,2,3\n4,5,6\n7,8,9', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -144,14 +229,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -164,14 +262,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '10', 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d5193d3..9080e88 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -3,10 +3,14 @@ import unittest import os import shutil import tempfile +from textwrap import dedent + +# Local import +from yaksh.code_evaluator import CodeEvaluator from yaksh.cpp_code_evaluator import CppCodeEvaluator from yaksh.cpp_stdio_evaluator import CppStdioEvaluator from yaksh.settings import SERVER_TIMEOUT -from textwrap import dedent + class CAssertionEvaluationTestCases(unittest.TestCase): @@ -15,6 +19,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "c_cpp_files/main.cpp", + "test_case_type": "standardtestcase", "weight": 0.0 }] self.in_dir = tmp_in_dir_path @@ -29,25 +34,49 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': False, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a-b;}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect:", result.get('error')) @@ -55,31 +84,57 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + # get_class = CppCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [{"test_case": "c_cpp_files/file_data.c", + "test_case_type": "standardtestcase", "weight": 0.0 }] user_answer = dedent(""" @@ -94,13 +149,26 @@ class CAssertionEvaluationTestCases(unittest.TestCase): return buff[0]; } """) - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + # get_class = CppCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") @@ -108,12 +176,14 @@ class CppStdioEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = dedent(""" @@ -123,19 +193,33 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%d%d",&a,&b); printf("%d",a+b); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include @@ -146,19 +230,33 @@ class CppStdioEvaluationTestCases(unittest.TestCase): for(i=0;i<3;i++){ printf("%d",a[i]);} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include @@ -167,12 +265,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%s",a); printf("%s",a); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -183,12 +294,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) @@ -201,12 +325,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a) }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -217,19 +354,33 @@ class CppStdioEvaluationTestCases(unittest.TestCase): while(0==0){ printf("abc");} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include @@ -237,12 +388,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=5,b=6; printf("%d",a+b); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + # get_class = CppStdioEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': False, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -255,19 +419,33 @@ class CppStdioEvaluationTestCases(unittest.TestCase): cin>>a>>b; cout< @@ -279,19 +457,33 @@ class CppStdioEvaluationTestCases(unittest.TestCase): for(i=0;i<3;i++){ cout< @@ -301,12 +493,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): cin>>a; cout< @@ -375,12 +620,25 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=5,b=6; cout< 1) def test_error(self): - user_answer = dedent(""" class Test { System.out.print("a"); }""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = JavaStdioEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -229,19 +351,33 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): { System.out.print("a");} }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = JavaStdioEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': '', - 'weight': 0.0 + 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" class Test @@ -250,19 +386,33 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): int b = 6; System.out.print(a+b); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = JavaStdioEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'HelloWorld', - 'expected_input': 'Hello\nWorld', - 'weight': 0.0 + 'expected_input': 'Hello\nWorld', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; @@ -273,20 +423,34 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): String b = s.nextLine(); System.out.print(a+b); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = JavaStdioEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_file_based_stdout(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [{'expected_output': '2', - 'expected_input': '', - 'weight': 0.0 + 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" import java.io.BufferedReader; @@ -306,13 +470,26 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): br.close(); }}} """) - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = JavaStdioEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 4bf0032..fb762f9 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -25,8 +25,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) self.file_paths = None - self.language = 'python' - self.test_case_type = 'standardtestcase' def tearDown(self): os.remove('/tmp/test.txt') diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index c30f652..de7368f 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -4,7 +4,8 @@ import os import shutil import tempfile -from yaksh import code_evaluator as evaluator +from yaksh import code_evaluator +from yaksh.code_evaluator import CodeEvaluator from yaksh.scilab_code_evaluator import ScilabCodeEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -12,40 +13,69 @@ class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "scilab_files/test_add.sce", + "test_case_type": "standardtestcase", "weight": 0.0 }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) + code_evaluator.SERVER_TIMEOUT = 9 self.file_paths = None def tearDown(self): + code_evaluator.SERVER_TIMEOUT = 4 shutil.rmtree(self.in_dir) def test_correct_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = ScilabCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_error(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\ndis(\tendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = ScilabCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue('error' in result.get("error")) @@ -53,28 +83,55 @@ class ScilabEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a-b;\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = ScilabCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Message", result.get('error')) self.assertTrue(lines_of_error > 1) def test_infinite_loop(self): + code_evaluator.SERVER_TIMEOUT = 4 user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = ScilabCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index d87e6e3..f2ca53b 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -8,21 +8,38 @@ import subprocess import importlib # Local imports -from .code_evaluator import CodeEvaluator +from .base_evaluator import BaseEvaluator from .file_utils import copy_files, delete_files -class JavaCodeEvaluator(CodeEvaluator): +class JavaCodeEvaluator(BaseEvaluator): """Tests the Java code obtained from Code Server""" - def setup(self): - super(JavaCodeEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = self.create_submit_code_file('Test.java') + # self.submit_code_path = self.create_submit_code_file('Test.java') self.compiled_user_answer = None self.compiled_test_code = None self.user_output_path = "" self.ref_output_path = "" + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('test_case') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(JavaCodeEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('Test.java') + # self.compiled_user_answer = None + # self.compiled_test_code = None + # self.user_output_path = "" + # self.ref_output_path = "" + def teardown(self): # Delete the created file. os.remove(self.submit_code_path) @@ -32,8 +49,6 @@ class JavaCodeEvaluator(CodeEvaluator): os.remove(self.ref_output_path) if self.files: delete_files(self.files) - super(JavaCodeEvaluator, self).teardown() - def get_commands(self, clean_ref_code_path, user_code_directory): compile_command = 'javac {0}'.format(self.submit_code_path), @@ -47,15 +62,16 @@ class JavaCodeEvaluator(CodeEvaluator): output_path = "{0}{1}.class".format(directory, file_name) return output_path - def compile_code(self, user_answer, file_paths, test_case, weight): + def compile_code(self): # , user_answer, file_paths, test_case, weight if self.compiled_user_answer and self.compiled_test_code: return None else: - ref_code_path = test_case + self.submit_code_path = self.create_submit_code_file('Test.java') + ref_code_path = self.test_case clean_ref_code_path, clean_test_case_path = \ self._set_test_code_file_path(ref_code_path) - if file_paths: - self.files = copy_files(file_paths) + if self.file_paths: + self.files = copy_files(self.file_paths) if not isfile(clean_ref_code_path): msg = "No file at %s or Incorrect path" % clean_ref_code_path return False, msg @@ -65,7 +81,7 @@ class JavaCodeEvaluator(CodeEvaluator): user_code_directory = os.getcwd() + '/' self.write_to_submit_code_file(self.submit_code_path, - user_answer + self.user_answer ) ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] self.user_output_path = self.set_file_paths(user_code_directory, @@ -82,6 +98,7 @@ class JavaCodeEvaluator(CodeEvaluator): user_code_directory, ref_file_name ) + self.compiled_user_answer = self._run_command(compile_command, shell=True, stdout=subprocess.PIPE, @@ -96,7 +113,7 @@ class JavaCodeEvaluator(CodeEvaluator): return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): + def check_code(self): # user_answer, file_paths, partial_grading, test_case, weight """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -136,7 +153,7 @@ class JavaCodeEvaluator(CodeEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" - test_case_weight = float(weight) if partial_grading else 0.0 + test_case_weight = float(seelf.weight) if self.partial_grading else 0.0 else: err = stdout + "\n" + stderr else: @@ -161,4 +178,5 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr + return success, err, test_case_weight diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index 88d4c88..78bc20e 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -11,17 +11,28 @@ from .file_utils import copy_files, delete_files class JavaStdioEvaluator(StdIOEvaluator): """Evaluates Java StdIO based code""" - - def setup(self): - super(JavaStdioEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = self.create_submit_code_file('Test.java') + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.expected_input = test_case_data.get('expected_input') + self.expected_output = test_case_data.get('expected_output') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(JavaStdioEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = self.create_submit_code_file('Test.java') def teardown(self): os.remove(self.submit_code_path) if self.files: delete_files(self.files) - super(JavaStdioEvaluator, self).teardown() def set_file_paths(self, directory, file_name): output_path = "{0}{1}.class".format(directory, file_name) @@ -31,14 +42,15 @@ class JavaStdioEvaluator(StdIOEvaluator): compile_command = 'javac {0}'.format(self.submit_code_path) return compile_command - def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): + def compile_code(self): + self.submit_code_path = self.create_submit_code_file('Test.java') if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path return False, msg - if file_paths: - self.files = copy_files(file_paths) + if self.file_paths: + self.files = copy_files(self.file_paths) user_code_directory = os.getcwd() + '/' - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) self.user_output_path = self.set_file_paths(user_code_directory, 'Test' ) @@ -50,8 +62,7 @@ class JavaStdioEvaluator(StdIOEvaluator): ) return self.compiled_user_answer - def check_code(self, user_answer, file_paths, partial_grading, - expected_input, expected_output, weight): + def check_code(self): success = False test_case_weight = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer @@ -63,9 +74,9 @@ class JavaStdioEvaluator(StdIOEvaluator): stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - success, err = self.evaluate_stdio(user_answer, proc, - expected_input, - expected_output + success, err = self.evaluate_stdio(self.user_answer, proc, + self.expected_input, + self.expected_output ) os.remove(self.user_output_path) else: @@ -79,5 +90,5 @@ class JavaStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - test_case_weight = float(weight) if partial_grading and success else 0.0 + test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 return success, err, test_case_weight diff --git a/yaksh/old_code_server.py b/yaksh/old_code_server.py deleted file mode 100755 index b3c9c30..0000000 --- a/yaksh/old_code_server.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python - -"""This server runs an HTTP server (using tornado) and several code servers -using XMLRPC that can be submitted code -and tests and returns the output. It *should* be run as root and will run as -the user 'nobody' so as to minimize any damange by errant code. This can be -configured by editing settings.py to run as many servers as desired. One can -also specify the ports on the command line. Here are examples:: - - $ sudo ./code_server.py - # Runs servers based on settings.py:SERVER_PORTS one server per port given. - -or:: - - $ sudo ./code_server.py 8001 8002 8003 8004 8005 - # Runs 5 servers on ports specified. - -All these servers should be running as nobody. This will also start a server -pool that defaults to port 50000 and is configurable in -settings.py:SERVER_POOL_PORT. This port exposes a `get_server_port` function -that returns an available server. - -""" - -# Standard library imports -from __future__ import unicode_literals -import json -from multiprocessing import Process, Queue -import os -from os.path import isdir, dirname, abspath, join, isfile -import pwd -import re -import signal -import stat -import subprocess -import sys - -try: - from SimpleXMLRPCServer import SimpleXMLRPCServer -except ImportError: - # The above import will not work on Python-3.x. - from xmlrpc.server import SimpleXMLRPCServer - -try: - from urllib import unquote -except ImportError: - # The above import will not work on Python-3.x. - from urllib.parse import unquote - -# Library imports -from tornado.ioloop import IOLoop -from tornado.web import Application, RequestHandler - -# Local imports -from .settings import SERVER_PORTS, SERVER_POOL_PORT -from .language_registry import create_evaluator_instance, unpack_json - - -MY_DIR = abspath(dirname(__file__)) - - -# Private Protocol ########## -def run_as_nobody(): - """Runs the current process as nobody.""" - # Set the effective uid and to that of nobody. - nobody = pwd.getpwnam('nobody') - os.setegid(nobody.pw_gid) - os.seteuid(nobody.pw_uid) - - -############################################################################### -# `CodeServer` class. -############################################################################### -class CodeServer(object): - """A code server that executes user submitted test code, tests it and - reports if the code was correct or not. - """ - def __init__(self, port, queue): - self.port = port - self.queue = queue - - # Public Protocol ########## - def check_code(self, language, test_case_type, json_data, in_dir=None): - """Calls relevant EvaluateCode class based on language to check the - answer code - """ - code_evaluator = create_evaluator_instance(language, - test_case_type, - json_data, - in_dir - ) - data = unpack_json(json_data) - result = code_evaluator.evaluate(**data) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return json.dumps(result) - - def run(self): - """Run XMLRPC server, serving our methods.""" - server = SimpleXMLRPCServer(("0.0.0.0", self.port)) - self.server = server - server.register_instance(self) - self.queue.put(self.port) - server.serve_forever() - - -############################################################################### -# `ServerPool` class. -############################################################################### -class ServerPool(object): - """Manages a pool of CodeServer objects.""" - def __init__(self, ports, pool_port=50000): - """Create a pool of servers. Uses a shared Queue to get available - servers. - - Parameters - ---------- - - ports : list(int) - List of ports at which the CodeServer's should run. - - pool_port : int - Port at which the server pool should serve. - """ - self.my_port = pool_port - self.ports = ports - queue = Queue(maxsize=len(self.ports)) - self.queue = queue - servers = [] - processes = [] - for port in self.ports: - server = CodeServer(port, queue) - servers.append(server) - p = Process(target=server.run) - processes.append(p) - self.servers = servers - self.processes = processes - self.app = self._make_app() - - def _make_app(self): - app = Application([ - (r"/.*", MainHandler, dict(server=self)), - ]) - app.listen(self.my_port) - return app - - def _start_code_servers(self): - for proc in self.processes: - if proc.pid is None: - proc.start() - - # Public Protocol ########## - - def get_server_port(self): - """Get available server port from ones in the pool. This will block - till it gets an available server. - """ - return self.queue.get() - - def get_status(self): - """Returns current queue size and total number of ports used.""" - try: - qs = self.queue.qsize() - except NotImplementedError: - # May not work on OS X so we return a dummy. - qs = len(self.ports) - - return qs, len(self.ports) - - def run(self): - """Run server which returns an available server port where code - can be executed. - """ - # We start the code servers here to ensure they are run as nobody. - self._start_code_servers() - IOLoop.current().start() - - def stop(self): - """Stop all the code server processes. - """ - for proc in self.processes: - proc.terminate() - IOLoop.current().stop() - - -class MainHandler(RequestHandler): - def initialize(self, server): - self.server = server - - def get(self): - path = self.request.path[1:] - if len(path) == 0: - port = self.server.get_server_port() - self.write(str(port)) - elif path == "status": - q_size, total = self.server.get_status() - result = "%d servers out of %d are free.\n"%(q_size, total) - load = float(total - q_size)/total*100 - result += "Load: %s%%\n"%load - self.write(result) - - -############################################################################### -def main(args=None): - if args: - ports = [int(x) for x in args] - else: - ports = SERVER_PORTS - - server_pool = ServerPool(ports=ports, pool_port=SERVER_POOL_PORT) - # This is done *after* the server pool is created because when the tornado - # app calls listen(), it cannot be nobody. - run_as_nobody() - - server_pool.run() - -if __name__ == '__main__': - args = sys.argv[1:] - main(args) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index eb13f53..003e001 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -39,19 +39,10 @@ class PythonAssertionEvaluator(BaseEvaluator): # self.files = [] - # def teardown(self): - # # Delete the created file. - # if self.files: - # delete_files(self.files) - # super(PythonAssertionEvaluator, self).teardown() - - # def teardown(self): - # # Delete the created file. - # if self.files: - # delete_files(self.files) - # # Cancel the signal - # delete_signal_handler() - # self._change_dir(dirname(MY_DIR)) + def teardown(self): + # Delete the created file. + if self.files: + delete_files(self.files) def compile_code(self): if self.file_paths: diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 7ef3a7c..991fd63 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -35,11 +35,6 @@ class PythonStdioEvaluator(BaseEvaluator): # super(PythonStdioEvaluator, self).setup() # self.files = [] - # def teardown(self): - # # Delete the created file. - # if self.files: - # delete_files(self.files) - # super(PythonStdioEvaluator, self).teardown() def __init__(self, metadata, test_case_data): self.files = [] @@ -53,6 +48,10 @@ class PythonStdioEvaluator(BaseEvaluator): self.expected_output = test_case_data.get('expected_output') self.weight = test_case_data.get('weight') + def teardown(self): + # Delete the created file. + if self.files: + delete_files(self.files) def compile_code(self): # user_answer, file_paths, expected_input, expected_output, weight): if self.file_paths: diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 3c2d44c..97e40a8 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -8,38 +8,49 @@ import re import importlib # Local imports -from .code_evaluator import CodeEvaluator +from .base_evaluator import BaseEvaluator from .file_utils import copy_files, delete_files -class ScilabCodeEvaluator(CodeEvaluator): +class ScilabCodeEvaluator(BaseEvaluator): """Tests the Scilab code obtained from Code Server""" - def setup(self): - super(ScilabCodeEvaluator, self).setup() + def __init__(self, metadata, test_case_data): self.files = [] - self.submit_code_path = \ - self.create_submit_code_file('function.sci') + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('test_case') + self.weight = test_case_data.get('weight') + + # def setup(self): + # super(ScilabCodeEvaluator, self).setup() + # self.files = [] + # self.submit_code_path = \ + # self.create_submit_code_file('function.sci') def teardown(self): # Delete the created file. os.remove(self.submit_code_path) if self.files: delete_files(self.files) - super(ScilabCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): - if file_paths: - self.files = copy_files(file_paths) - ref_code_path = test_case + def check_code(self): + self.submit_code_path = self.create_submit_code_file('function.sci') + if self.file_paths: + self.files = copy_files(self.file_paths) + ref_code_path = self.test_case clean_ref_path, clean_test_case_path = \ self._set_test_code_file_path(ref_code_path) - user_answer, terminate_commands = \ - self._remove_scilab_exit(user_answer.lstrip()) + self.user_answer, terminate_commands = \ + self._remove_scilab_exit(self.user_answer.lstrip()) success = False test_case_weight = 0.0 - - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.write_to_submit_code_file(self.submit_code_path, self.user_answer) # Throw message if there are commmands that terminates scilab add_err = "" if terminate_commands: @@ -65,11 +76,12 @@ class ScilabCodeEvaluator(CodeEvaluator): stdout = self._strip_output(stdout) if proc.returncode == 5: success, err = True, "Correct answer" - test_case_weight = float(weight) if partial_grading else 0.0 + test_case_weight = float(self.weight) if self.partial_grading else 0.0 else: err = add_err + stdout else: err = add_err + stderr + return success, err, test_case_weight def _remove_scilab_exit(self, string): diff --git a/yaksh/settings.py b/yaksh/settings.py index 690ddb1..e18d310 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -42,5 +42,17 @@ URL_ROOT = '' code_evaluators = { "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" - } + }, + "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + }, + "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + }, + "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", + "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, + "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", + "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" + }, + "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py index 7530b96..106facd 100644 --- a/yaksh/stdio_evaluator.py +++ b/yaksh/stdio_evaluator.py @@ -1,18 +1,10 @@ from __future__ import unicode_literals # Local imports -from .code_evaluator import CodeEvaluator +from .base_evaluator import BaseEvaluator -class StdIOEvaluator(CodeEvaluator): - def setup(self): - super(StdIOEvaluator, self).setup() - pass - - def teardown(self): - super(StdIOEvaluator, self).teardown() - pass - +class StdIOEvaluator(BaseEvaluator): def evaluate_stdio(self, user_answer, proc, expected_input, expected_output): success = False ip = expected_input.replace(",", " ") -- cgit From 81d373983806b78d651e17b966bae0a8ccf3c4bb Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 19 Dec 2016 17:43:41 +0530 Subject: Fix model test cases --- yaksh/code_evaluator.py | 2 +- yaksh/models.py | 56 ++++++++++++++++++++++++++++++++--------- yaksh/test_models.py | 45 ++++++++++++++++++++------------- yaksh/tests/test_code_server.py | 48 ++++++++++++++++++++++------------- 4 files changed, 103 insertions(+), 48 deletions(-) (limited to 'yaksh') diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index f1ac5b7..52720df 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -74,7 +74,7 @@ class CodeEvaluator(object): msg = 'Code took more than %s seconds to run. You probably '\ 'have an infinite loop in your code.' % SERVER_TIMEOUT self.timeout_msg = msg - self.in_dir = in_dir + self.in_dir = in_dir if in_dir else MY_DIR def evaluate(self, kwargs): #language, test_case_type, diff --git a/yaksh/models.py b/yaksh/models.py index 4951836..272ec75 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -306,9 +306,13 @@ class Question(models.Model): que, result = Question.objects.get_or_create(**question) if file_names: que._add_files_to_db(file_names, file_path) - model_class = get_model_class(que.test_case_type) for test_case in test_cases: - model_class.objects.get_or_create(question=que, **test_case) + test_case_type = test_case.pop('test_case_type') + model_class = get_model_class(test_case_type) + # TestCase.objects.get_or_create(question=que, type=) + new_test_case, obj_create_status = model_class.objects.get_or_create(question=que, **test_case) + new_test_case.type = test_case_type + new_test_case.save() if files_list: delete_files(files_list, file_path) @@ -320,22 +324,50 @@ class Question(models.Model): # question=self, # **kwargs # ) + # tc_list = [] + # for tc in self.testcase_set.filter(**kwargs): + # tc_type = str(tc.type) + # obj = getattr(tc, tc_type) + # tc_list.append(obj) + + # return tc_list + tc_list = [] for tc in self.testcase_set.all(): - tc_type = tc.type - obj = getattr(tc, tc_type) - tc_list.append(obj) + test_case_type = tc.type + test_case_ctype = ContentType.objects.get(app_label="yaksh", + model=test_case_type + ) + test_case = test_case_ctype.get_object_for_this_type( + question=self, + **kwargs + ) + tc_list.append(test_case) return tc_list def get_test_case(self, **kwargs): - test_case_ctype = ContentType.objects.get(app_label="yaksh", - model=self.test_case_type - ) - test_case = test_case_ctype.get_object_for_this_type( - question=self, - **kwargs - ) + # test_case_ctype = ContentType.objects.get(app_label="yaksh", + # model=self.test_case_type + # ) + # test_case = test_case_ctype.get_object_for_this_type( + # question=self, + # **kwargs + # ) + # tc = self.testcase_set.get(**kwargs) + # tc_type = str(tc.type) + # test_case = getattr(tc, tc_type) + + # return test_case + for tc in self.testcase_set.all(): + test_case_type = tc.type + test_case_ctype = ContentType.objects.get(app_label="yaksh", + model=self.test_case_type + ) + test_case = test_case_ctype.get_object_for_this_type( + question=self, + **kwargs + ) return test_case diff --git a/yaksh/test_models.py b/yaksh/test_models.py index 522da89..0db1b82 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -134,21 +134,23 @@ class QuestionTestCases(unittest.TestCase): self.question1.tags.add('python', 'function') self.assertion_testcase = StandardTestCase(question=self.question1, - test_case='assert myfunc(12, 13) == 15' + test_case='assert myfunc(12, 13) == 15', + type='standardtestcase' ) self.upload_test_case = StandardTestCase(question=self.question2, - test_case='assert fact(3) == 6' + test_case='assert fact(3) == 6', + type='standardtestcase' ) self.upload_test_case.save() self.user_answer = "demo_answer" self.test_case_upload_data = [{"test_case": "assert fact(3)==6", + "test_case_type": "standardtestcase", "weight": 1.0 }] questions_data = [{"snippet": "def fact()", "active": True, "points": 1.0, "description": "factorial of a no", "language": "Python", "type": "Code", - "test_case_type": "standardtestcase", "testcase": self.test_case_upload_data, "files": [[file1, 0]], "summary": "Json Demo"}] @@ -213,7 +215,6 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(question_data.points, 1.0) self.assertTrue(question_data.active) self.assertEqual(question_data.snippet, 'def fact()') - self.assertEqual(question_data.test_case_type, 'standardtestcase') self.assertEqual(os.path.basename(file.file.path), "test.txt") self.assertEqual([case.get_field_value() for case in test_case], self.test_case_upload_data) @@ -511,19 +512,24 @@ class AnswerPaperTestCases(unittest.TestCase): self.question3.save() self.assertion_testcase = StandardTestCase( question=self.question1, - test_case='assert add(1, 3) == 4' + test_case='assert add(1, 3) == 4', + type = 'standardtestcase' + ) self.assertion_testcase.save() self.mcq_based_testcase = McqTestCase( options = 'a', question=self.question2, - correct = True + correct = True, + type = 'mcqtestcase' + ) self.mcq_based_testcase.save() self.mcc_based_testcase = McqTestCase( question=self.question3, options = 'a', - correct = True + correct = True, + type = 'mcqtestcase' ) self.mcc_based_testcase.save() @@ -870,21 +876,26 @@ class TestCaseTestCases(unittest.TestCase): self.question2.save() self.assertion_testcase = StandardTestCase( question=self.question1, - test_case='assert myfunc(12, 13) == 15' + test_case='assert myfunc(12, 13) == 15', + type='standardtestcase' ) self.stdout_based_testcase = StdioBasedTestCase( question=self.question2, - expected_output='Hello World' + expected_output='Hello World', + type='standardtestcase' + ) self.assertion_testcase.save() self.stdout_based_testcase.save() - answer_data = {"user_answer": "demo_answer", - "test_case_data": [ - {"test_case": "assert myfunc(12, 13) == 15", - "weight": 1.0 - } - ] - } + answer_data = {'metadata': { 'user_answer': 'demo_answer', + 'language': 'python', + 'partial_grading': False + }, + 'test_case_data': [{'test_case': 'assert myfunc(12, 13) == 15', + 'test_case_type': 'standardtestcase', + 'weight': 1.0 + }] + } self.answer_data_json = json.dumps(answer_data) def test_assertion_testcase(self): @@ -907,5 +918,5 @@ class TestCaseTestCases(unittest.TestCase): ) actual_data = json.loads(result) exp_data = json.loads(self.answer_data_json) - self.assertEqual(actual_data['user_answer'], exp_data['user_answer']) + self.assertEqual(actual_data['metadata']['user_answer'], exp_data['metadata']['user_answer']) self.assertEqual(actual_data['test_case_data'], exp_data['test_case_data']) diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 7efd20b..d46c9dd 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -37,12 +37,15 @@ class TestCodeServer(unittest.TestCase): def test_infinite_loop(self): # Given - testdata = {'user_answer': 'while True: pass', - 'partial_grading': False, + testdata = {'metadata': {'user_answer': 'while True: pass', + 'language': 'python', + 'partial_grading': False + }, 'test_case_data': [{'test_case':'assert 1==2', + 'test_case_type': 'standardtestcase', 'weight': 0.0 - } - ]} + }] + } # When result = self.code_server.run_code( @@ -56,12 +59,15 @@ class TestCodeServer(unittest.TestCase): def test_correct_answer(self): # Given - testdata = {'user_answer': 'def f(): return 1', - 'partial_grading': False, + testdata = {'metadata': { 'user_answer': 'def f(): return 1', + 'language': 'python', + 'partial_grading': False + }, 'test_case_data': [{'test_case':'assert f() == 1', + 'test_case_type': 'standardtestcase', 'weight': 0.0 - } - ]} + }] + } # When result = self.code_server.run_code( @@ -75,12 +81,15 @@ class TestCodeServer(unittest.TestCase): def test_wrong_answer(self): # Given - testdata = {'user_answer': 'def f(): return 1', - 'partial_grading': False, + testdata = {'metadata': { 'user_answer': 'def f(): return 1', + 'language': 'python', + 'partial_grading': False + }, 'test_case_data': [{'test_case':'assert f() == 2', + 'test_case_type': 'standardtestcase', 'weight': 0.0 - } - ]} + }] + } # When result = self.code_server.run_code( @@ -98,12 +107,15 @@ class TestCodeServer(unittest.TestCase): def run_code(): """Run an infinite loop.""" - testdata = {'user_answer': 'while True: pass', - 'partial_grading': False, - 'test_case_data': [{'test_case':'assert 1==2', - 'weight': 0.0 - } - ]} + testdata = {'metadata': { 'user_answer': 'while True: pass', + 'language': 'python', + 'partial_grading': False + }, + 'test_case_data': [{'test_case':'assert 1==2', + 'test_case_type': 'standardtestcase', + 'weight': 0.0 + }] + } result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) -- cgit From 4a0f94084bc26559ef3e26470619e87314f9d70e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 19 Dec 2016 19:18:35 +0530 Subject: Remove commented code --- yaksh/bash_code_evaluator.py | 6 -- yaksh/bash_stdio_evaluator.py | 5 - yaksh/code_evaluator.py | 138 ++---------------------- yaksh/code_server.py | 5 - yaksh/cpp_code_evaluator.py | 9 -- yaksh/cpp_stdio_evaluator.py | 5 - yaksh/evaluator_tests/test_bash_evaluation.py | 52 --------- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 116 -------------------- yaksh/evaluator_tests/test_java_evaluation.py | 86 --------------- yaksh/evaluator_tests/test_python_evaluation.py | 88 --------------- yaksh/evaluator_tests/test_scilab_evaluation.py | 28 ----- yaksh/java_code_evaluator.py | 14 +-- yaksh/java_stdio_evaluator.py | 5 - yaksh/models.py | 28 ----- yaksh/python_assertion_evaluator.py | 11 -- yaksh/python_stdio_evaluator.py | 9 +- 16 files changed, 12 insertions(+), 593 deletions(-) (limited to 'yaksh') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 03ec16a..43d7be7 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -27,12 +27,6 @@ class BashCodeEvaluator(BaseEvaluator): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - # def setup(self): - # super(BashCodeEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = self.create_submit_code_file('submit.sh') - # self._set_file_as_executable(self.submit_code_path) - def teardown(self): # Delete the created file. os.remove(self.submit_code_path) diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 3344c57..38b48e6 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -24,11 +24,6 @@ class BashStdioEvaluator(StdIOEvaluator): self.expected_output = test_case_data.get('expected_output') self.weight = test_case_data.get('weight') - # def setup(self): - # super(BashStdioEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = self.create_submit_code_file('Test.sh') - def teardown(self): os.remove(self.submit_code_path) if self.files: diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 52720df..ae61752 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -113,28 +113,24 @@ class CodeEvaluator(object): if self.in_dir: if not os.path.exists(self.in_dir): os.makedirs(self.in_dir) - # self._change_dir(self.in_dir) def get_evaluator_objects(self, kwargs): - metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths + metadata = kwargs.get('metadata') test_case_data = kwargs.get('test_case_data') test_case_instances = [] for test_case in test_case_data: - test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case + test_case_instance = create_evaluator_instance(metadata, test_case) test_case_instances.append(test_case_instance) return test_case_instances - def safe_evaluate(self, test_case_instances): #user_answer, partial_grading, test_case_data, file_paths=None + def safe_evaluate(self, test_case_instances): """ Handles code evaluation along with compilation, signal handling and Exception handling """ - # metadata = kwargs.get('metadata') # metadata contains user_answer, language, partial_grading, file_paths - # test_case_data = kwargs.get('test_case_data') - # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False @@ -146,17 +142,9 @@ class CodeEvaluator(object): try: # Run evaluator selection registry here for idx, test_case_instance in enumerate(test_case_instances): - # test_case_instance = create_evaluator_instance(metadata, test_case) #language, test_case - # self.setup() test_case_success = False - test_case_instance.compile_code() #user_answer, file_paths, test_case - test_case_success, err, test_case_weight = test_case_instance.check_code() #**kwargs - test_case_instance.teardown() - # self.teardown() - # user_answer, - # file_paths, - # partial_grading, - # **test_case + test_case_instance.compile_code() + test_case_success, err, test_case_weight = test_case_instance.check_code() if test_case_success: weight += test_case_weight @@ -165,6 +153,9 @@ class CodeEvaluator(object): success = all(test_case_success_status) + for test_case_instance in test_case_instances: + test_case_instance.teardown() + except TimeoutException: error = self.timeout_msg except OSError: @@ -182,119 +173,6 @@ class CodeEvaluator(object): return success, error, weight - # def safe_evaluate(self, user_answer, partial_grading, test_case_data, file_paths=None): - # """ - # Handles code evaluation along with compilation, signal handling - # and Exception handling - # """ - - # # Add a new signal handler for the execution of this code. - # prev_handler = create_signal_handler() - # success = False - # test_case_success_status = [False] * len(test_case_data) - # error = "" - # weight = 0.0 - - # # Do whatever testing needed. - # try: - # for idx, test_case in enumerate(test_case_data): - # test_case_success = False - # self.compile_code(user_answer, file_paths, **test_case) - # test_case_success, err, test_case_weight = self.check_code(user_answer, - # file_paths, - # partial_grading, - # **test_case - # ) - # if test_case_success: - # weight += test_case_weight - - # error += err + "\n" - # test_case_success_status[idx] = test_case_success - - # success = all(test_case_success_status) - - # except TimeoutException: - # error = self.timeout_msg - # except OSError: - # msg = traceback.format_exc(limit=0) - # error = "Error: {0}".format(msg) - # except Exception as e: - # print "HELLOOOOO", e - # exc_type, exc_value, exc_tb = sys.exc_info() - # tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) - # if len(tb_list) > 2: - # del tb_list[1:3] - # error = "Error: {0}".format("".join(tb_list)) - # finally: - # # Set back any original signal handler. - # set_original_signal_handler(prev_handler) - - # return success, error, weight - - def teardown(self): # Cancel the signal delete_signal_handler() - # self._change_dir(dirname(MY_DIR)) - - # def check_code(self): - # raise NotImplementedError("check_code method not implemented") - - # def compile_code(self, user_answer, file_paths, **kwargs): - # pass - - # def create_submit_code_file(self, file_name): - # """ Set the file path for code (`answer`)""" - # submit_path = abspath(file_name) - # if not exists(submit_path): - # submit_f = open(submit_path, 'w') - # submit_f.close() - - # return submit_path - - # def write_to_submit_code_file(self, file_path, user_answer): - # """ Write the code (`answer`) to a file""" - # submit_f = open(file_path, 'w') - # submit_f.write(user_answer.lstrip()) - # submit_f.close() - - # def _set_file_as_executable(self, fname): - # os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - # | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - # | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) - - # def _set_test_code_file_path(self, ref_path=None, test_case_path=None): - # if ref_path and not ref_path.startswith('/'): - # ref_path = join(MY_DIR, ref_path) - - # if test_case_path and not test_case_path.startswith('/'): - # test_case_path = join(MY_DIR, test_case_path) - - # return ref_path, test_case_path - - # def _run_command(self, cmd_args, *args, **kw): - # """Run a command in a subprocess while blocking, the process is killed - # if it takes more than 2 seconds to run. Return the Popen object, the - # stdout and stderr. - # """ - # try: - # proc = subprocess.Popen(cmd_args, *args, **kw) - # stdout, stderr = proc.communicate() - # except TimeoutException: - # # Runaway code, so kill it. - # proc.kill() - # # Re-raise exception. - # raise - # return proc, stdout.decode('utf-8'), stderr.decode('utf-8') - - # def _change_dir(self, in_dir): - # if in_dir is not None and isdir(in_dir): - # os.chdir(in_dir) - - # def _remove_null_substitute_char(self, string): - # """Returns a string without any null and substitute characters""" - # stripped = "" - # for c in string: - # if ord(c) is not 26 and ord(c) is not 0: - # stripped = stripped + c - # return ''.join(stripped) diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 3c1a3e3..815eb55 100644 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -85,11 +85,6 @@ class CodeServer(object): """Calls relevant EvaluateCode class based on language to check the answer code """ - # code_evaluator = create_evaluator_instance(language, - # test_case_type, - # json_data, - # in_dir - # ) data = self.unpack_json_to_python_obj(json_data) code_eval_instance = CodeEvaluator(in_dir) result = code_eval_instance.evaluate(data) #language, test_case_type, diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index c6f5a7e..f97f274 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -30,15 +30,6 @@ class CppCodeEvaluator(BaseEvaluator): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - # def setup(self): - # super(CppCodeEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = self.create_submit_code_file('submit.c') - # self.compiled_user_answer = None - # self.compiled_test_code = None - # self.user_output_path = "" - # self.ref_output_path = "" - def teardown(self): # Delete the created file. os.remove(self.submit_code_path) diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index a57afbe..76c132d 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -25,11 +25,6 @@ class CppStdioEvaluator(StdIOEvaluator): self.expected_output = test_case_data.get('expected_output') self.weight = test_case_data.get('weight') - # def setup(self): - # super(CppStdioEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = self.create_submit_code_file('main.c') - def teardown(self): os.remove(self.submit_code_path) if self.files: diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 8888ee6..142d7f0 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -34,13 +34,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) - # get_class = BashCodeEvaluator(self.in_dir) - # kwargs = {'user_answer': user_answer, - # 'partial_grading': True, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -60,13 +53,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def test_error(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") - # get_class = BashCodeEvaluator(self.in_dir) - # kwargs = {'user_answer': user_answer, - # 'partial_grading': True, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -86,13 +72,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") - # get_class = BashCodeEvaluator(self.in_dir) - # kwargs = {'user_answer': user_answer, - # 'partial_grading': True, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -118,13 +97,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): } ] user_answer = ("#!/bin/bash\ncat $1") - # get_class = BashCodeEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': True, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -162,12 +134,6 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - # get_class = BashStdioEvaluator() - # kwargs = {"user_answer": user_answer, - # "partial_grading": True, - # "test_case_data": test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -198,12 +164,6 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - # get_class = BashStdioEvaluator() - # kwargs = {"user_answer": user_answer, - # "partial_grading": True, - # "test_case_data": test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -232,12 +192,6 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - # get_class = BashStdioEvaluator() - # kwargs = {"user_answer": user_answer, - # "partial_grading": True, - # "test_case_data": test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -265,12 +219,6 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - # get_class = BashStdioEvaluator() - # kwargs = {"user_answer": user_answer, - # "partial_grading": True, - # "test_case_data": test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 9080e88..4dd6a2f 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -34,12 +34,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -58,12 +52,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a-b;}" - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -84,12 +72,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -108,13 +90,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" - # get_class = CppCodeEvaluator(self.in_dir) - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -149,13 +124,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase): return buff[0]; } """) - # get_class = CppCodeEvaluator(self.in_dir) - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data, - # 'file_paths': self.file_paths - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -193,12 +161,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%d%d",&a,&b); printf("%d",a+b); }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -230,12 +192,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): for(i=0;i<3;i++){ printf("%d",a[i]);} }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -265,12 +221,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%s",a); printf("%s",a); }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -294,12 +244,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a); }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -325,12 +269,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a) }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -354,12 +292,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): while(0==0){ printf("abc");} }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -388,12 +320,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=5,b=6; printf("%d",a+b); }""") - # get_class = CppStdioEvaluator() - # kwargs = {'user_answer': user_answer, - # 'partial_grading': False, - # 'test_case_data': self.test_case_data - # } - # result = get_class.evaluate(**kwargs) kwargs = { 'metadata': { 'user_answer': user_answer, @@ -419,12 +345,6 @@ class CppStdioEvaluationTestCases(unittest.TestCase): cin>>a>>b; cout<>a; cout<', mode='exec') @@ -68,7 +63,7 @@ class PythonStdioEvaluator(BaseEvaluator): self.output_value = output_buffer.getvalue().rstrip("\n") return self.output_value - def check_code(self): # user_answer, file_paths, partial_grading, expected_input, expected_output, weight): + def check_code(self): success = False test_case_weight = 0.0 -- cgit From 798d36aa12e22928e884668ae5c80a25d48393ea Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 19 Dec 2016 19:37:23 +0530 Subject: Change weight variable name to mark_fraction --- yaksh/bash_code_evaluator.py | 10 +++++----- yaksh/bash_stdio_evaluator.py | 6 +++--- yaksh/code_evaluator.py | 4 ++-- yaksh/cpp_code_evaluator.py | 6 +++--- yaksh/cpp_stdio_evaluator.py | 6 +++--- yaksh/java_code_evaluator.py | 6 +++--- yaksh/java_stdio_evaluator.py | 6 +++--- yaksh/python_assertion_evaluator.py | 6 +++--- yaksh/python_stdio_evaluator.py | 6 +++--- 9 files changed, 28 insertions(+), 28 deletions(-) (limited to 'yaksh') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 43d7be7..1e6fc9c 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -60,7 +60,7 @@ class BashCodeEvaluator(BaseEvaluator): """ ref_code_path = self.test_case success = False - test_case_weight = 0.0 + mark_fraction = 0.0 self.submit_code_path = self.create_submit_code_file('submit.sh') self._set_file_as_executable(self.submit_code_path) @@ -103,8 +103,8 @@ class BashCodeEvaluator(BaseEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - test_case_weight = float(self.weight) if self.partial_grading else 0.0 - return True, "Correct answer", test_case_weight + mark_fraction = float(self.weight) if self.partial_grading else 0.0 + return True, "Correct answer", mark_fraction else: err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr @@ -146,8 +146,8 @@ class BashCodeEvaluator(BaseEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - test_case_weight = float(self.weight) if self.partial_grading else 0.0 - return True, "Correct answer", test_case_weight + mark_fraction = float(self.weight) if self.partial_grading else 0.0 + return True, "Correct answer", mark_fraction else: err = ("Error:expected" " {0}, got {1}").format(inst_stdout+inst_stderr, diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 38b48e6..63bf3dc 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -42,7 +42,7 @@ class BashStdioEvaluator(StdIOEvaluator): def check_code(self): success = False - test_case_weight = 0.0 + mark_fraction = 0.0 self.expected_input = str(self.expected_input).replace('\r', '') proc = subprocess.Popen("bash ./Test.sh", @@ -55,5 +55,5 @@ class BashStdioEvaluator(StdIOEvaluator): self.expected_input, self.expected_output ) - test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 - return success, err, test_case_weight + mark_fraction = float(self.weight) if self.partial_grading and success else 0.0 + return success, err, mark_fraction diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index ae61752..095099b 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -144,9 +144,9 @@ class CodeEvaluator(object): for idx, test_case_instance in enumerate(test_case_instances): test_case_success = False test_case_instance.compile_code() - test_case_success, err, test_case_weight = test_case_instance.check_code() + test_case_success, err, mark_fraction = test_case_instance.check_code() if test_case_success: - weight += test_case_weight + weight += mark_fraction error += err + "\n" test_case_success_status[idx] = test_case_success diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index f97f274..f0c2029 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -114,7 +114,7 @@ class CppCodeEvaluator(BaseEvaluator): if the required permissions are not given to the file(s). """ success = False - test_case_weight = 0.0 + mark_fraction = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -134,7 +134,7 @@ class CppCodeEvaluator(BaseEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" - test_case_weight = float(self.weight) if self.partial_grading else 0.0 + mark_fraction = float(self.weight) if self.partial_grading else 0.0 else: err = "{0} \n {1}".format(stdout, stderr) else: @@ -160,4 +160,4 @@ class CppCodeEvaluator(BaseEvaluator): except: err = "{0} \n {1}".format(err, stdnt_stderr) - return success, err, test_case_weight + return success, err, mark_fraction diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 76c132d..6d4b55d 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -70,7 +70,7 @@ class CppStdioEvaluator(StdIOEvaluator): def check_code(self): success = False - test_case_weight = 0.0 + mark_fraction = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -112,5 +112,5 @@ class CppStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 - return success, err, test_case_weight + mark_fraction = float(self.weight) if self.partial_grading and success else 0.0 + return success, err, mark_fraction diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index f94321e..5d3fd28 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -124,7 +124,7 @@ class JavaCodeEvaluator(BaseEvaluator): """ success = False - test_case_weight = 0.0 + mark_fraction = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -143,7 +143,7 @@ class JavaCodeEvaluator(BaseEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" - test_case_weight = float(seelf.weight) if self.partial_grading else 0.0 + mark_fraction = float(seelf.weight) if self.partial_grading else 0.0 else: err = stdout + "\n" + stderr else: @@ -169,4 +169,4 @@ class JavaCodeEvaluator(BaseEvaluator): except: err = err + "\n" + stdnt_stderr - return success, err, test_case_weight + return success, err, mark_fraction diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index a3b528d..a5f5540 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -59,7 +59,7 @@ class JavaStdioEvaluator(StdIOEvaluator): def check_code(self): success = False - test_case_weight = 0.0 + mark_fraction = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) if stdnt_stderr == '' or "error" not in stdnt_stderr: @@ -85,5 +85,5 @@ class JavaStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - test_case_weight = float(self.weight) if self.partial_grading and success else 0.0 - return success, err, test_case_weight + mark_fraction = float(self.weight) if self.partial_grading and success else 0.0 + return success, err, mark_fraction diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 8e4f0b3..0b49cb0 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -66,7 +66,7 @@ class PythonAssertionEvaluator(BaseEvaluator): the required permissions are not given to the file(s). """ success = False - test_case_weight = 0.0 + mark_fraction = 0.0 try: tb = None _tests = compile(self.test_case, '', mode='exec') @@ -86,6 +86,6 @@ class PythonAssertionEvaluator(BaseEvaluator): else: success = True err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(self.test_case) - test_case_weight = float(self.weight) if self.partial_grading else 0.0 + mark_fraction = float(self.weight) if self.partial_grading else 0.0 del tb - return success, err, test_case_weight + return success, err, mark_fraction diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 0f24a27..07bd59c 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -65,13 +65,13 @@ class PythonStdioEvaluator(BaseEvaluator): def check_code(self): success = False - test_case_weight = 0.0 + mark_fraction = 0.0 tb = None if self.output_value == self.expected_output: success = True err = "Correct answer" - test_case_weight = self.weight + mark_fraction = self.weight else: success = False err = dedent(""" @@ -85,4 +85,4 @@ class PythonStdioEvaluator(BaseEvaluator): ) ) del tb - return success, err, test_case_weight + return success, err, mark_fraction -- cgit From bf5b4e7607bae0b81ceeb99e8bf5d750433e92e8 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 20 Dec 2016 12:42:44 +0530 Subject: Fix errors and rename resources - code_evaluator module and class renamed to grader - Test cases fixed - Comments removed - weight variable renamed to mark --- yaksh/admin.py | 4 +- yaksh/base_evaluator.py | 2 +- yaksh/bash_stdio_evaluator.py | 2 +- yaksh/code_evaluator.py | 178 ------------------------ yaksh/code_server.py | 12 +- yaksh/cpp_stdio_evaluator.py | 2 +- yaksh/evaluator_tests/test_bash_evaluation.py | 38 ++--- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 82 +++++------ yaksh/evaluator_tests/test_code_evaluation.py | 4 +- yaksh/evaluator_tests/test_java_evaluation.py | 83 +++++------ yaksh/evaluator_tests/test_python_evaluation.py | 80 +++++------ yaksh/evaluator_tests/test_scilab_evaluation.py | 30 ++-- yaksh/forms.py | 4 +- yaksh/grader.py | 178 ++++++++++++++++++++++++ yaksh/java_stdio_evaluator.py | 2 +- yaksh/language_registry.py | 7 +- yaksh/models.py | 2 +- yaksh/python_assertion_evaluator.py | 2 +- yaksh/python_stdio_evaluator.py | 2 +- yaksh/scilab_code_evaluator.py | 8 +- yaksh/settings.py | 30 +--- yaksh/test_models.py | 4 +- yaksh/test_views.py | 2 +- yaksh/views.py | 2 +- 24 files changed, 362 insertions(+), 398 deletions(-) delete mode 100644 yaksh/code_evaluator.py create mode 100644 yaksh/grader.py (limited to 'yaksh') diff --git a/yaksh/admin.py b/yaksh/admin.py index 58af7b2..c525ba3 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -1,11 +1,11 @@ from yaksh.models import Question, Quiz, QuestionPaper -from yaksh.models import TestCase, StandardTestCase, StdioBasedTestCase, Course, AnswerPaper +from yaksh.models import TestCase, StandardTestCase, StdIOBasedTestCase, Course, AnswerPaper from django.contrib import admin admin.site.register(Question) admin.site.register(TestCase) admin.site.register(StandardTestCase) -admin.site.register(StdioBasedTestCase) +admin.site.register(StdIOBasedTestCase) admin.site.register(Course) admin.site.register(Quiz) admin.site.register(QuestionPaper) diff --git a/yaksh/base_evaluator.py b/yaksh/base_evaluator.py index b290ba4..ce1647f 100644 --- a/yaksh/base_evaluator.py +++ b/yaksh/base_evaluator.py @@ -10,7 +10,7 @@ import stat # Local imports -from .code_evaluator import MY_DIR, TimeoutException +from .grader import MY_DIR, TimeoutException class BaseEvaluator(object): """Base Evaluator class containing generic attributes and callable methods""" diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 63bf3dc..50ee0d6 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class BashStdioEvaluator(StdIOEvaluator): +class BashStdIOEvaluator(StdIOEvaluator): """Evaluates Bash StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py deleted file mode 100644 index 095099b..0000000 --- a/yaksh/code_evaluator.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python -from __future__ import unicode_literals -import sys -import pwd -import os -import stat -import contextlib -from os.path import isdir, dirname, abspath, join, isfile, exists -import signal -import traceback -from multiprocessing import Process, Queue -import subprocess -import re - -try: - from SimpleXMLRPCServer import SimpleXMLRPCServer -except ImportError: - # The above import will not work on Python-3.x. - from xmlrpc.server import SimpleXMLRPCServer - -# Local imports -from .settings import SERVER_TIMEOUT -from .language_registry import create_evaluator_instance - - -MY_DIR = abspath(dirname(__file__)) - -registry = None - -# Raised when the code times-out. -# c.f. http://pguides.net/python/timeout-a-function -class TimeoutException(Exception): - pass - -@contextlib.contextmanager -def change_dir(path): - cur_dir = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(cur_dir) - - -def timeout_handler(signum, frame): - """A handler for the ALARM signal.""" - raise TimeoutException('Code took too long to run.') - - -def create_signal_handler(): - """Add a new signal handler for the execution of this code.""" - prev_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - return prev_handler - - -def set_original_signal_handler(old_handler=None): - """Set back any original signal handler.""" - if old_handler is not None: - signal.signal(signal.SIGALRM, old_handler) - return - else: - raise Exception("Signal Handler: object cannot be NoneType") - - -def delete_signal_handler(): - signal.alarm(0) - return - - -class CodeEvaluator(object): - """Tests the code obtained from Code Server""" - def __init__(self, in_dir=None): - msg = 'Code took more than %s seconds to run. You probably '\ - 'have an infinite loop in your code.' % SERVER_TIMEOUT - self.timeout_msg = msg - self.in_dir = in_dir if in_dir else MY_DIR - - - def evaluate(self, kwargs): #language, test_case_type, - """Evaluates given code with the test cases based on - given arguments in test_case_data. - - The ref_code_path is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message, weight). - """ - - self.setup() - test_case_instances = self.get_evaluator_objects(kwargs) - with change_dir(self.in_dir): - success, error, weight = self.safe_evaluate(test_case_instances) - self.teardown() - - result = {'success': success, 'error': error, 'weight': weight} - return result - - # Private Protocol ########## - def setup(self): - if self.in_dir: - if not os.path.exists(self.in_dir): - os.makedirs(self.in_dir) - - def get_evaluator_objects(self, kwargs): - metadata = kwargs.get('metadata') - test_case_data = kwargs.get('test_case_data') - test_case_instances = [] - - for test_case in test_case_data: - test_case_instance = create_evaluator_instance(metadata, test_case) - test_case_instances.append(test_case_instance) - - return test_case_instances - - - def safe_evaluate(self, test_case_instances): - """ - Handles code evaluation along with compilation, signal handling - and Exception handling - """ - # Add a new signal handler for the execution of this code. - prev_handler = create_signal_handler() - success = False - test_case_success_status = [False] * len(test_case_instances) - error = "" - weight = 0.0 - - # Do whatever testing needed. - try: - # Run evaluator selection registry here - for idx, test_case_instance in enumerate(test_case_instances): - test_case_success = False - test_case_instance.compile_code() - test_case_success, err, mark_fraction = test_case_instance.check_code() - if test_case_success: - weight += mark_fraction - - error += err + "\n" - test_case_success_status[idx] = test_case_success - - success = all(test_case_success_status) - - for test_case_instance in test_case_instances: - test_case_instance.teardown() - - except TimeoutException: - error = self.timeout_msg - except OSError: - msg = traceback.format_exc(limit=0) - error = "Error: {0}".format(msg) - except Exception as e: - exc_type, exc_value, exc_tb = sys.exc_info() - tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) - if len(tb_list) > 2: - del tb_list[1:3] - error = "Error: {0}".format("".join(tb_list)) - finally: - # Set back any original signal handler. - set_original_signal_handler(prev_handler) - - return success, error, weight - - def teardown(self): - # Cancel the signal - delete_signal_handler() diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 815eb55..4db5810 100644 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -54,7 +54,7 @@ from tornado.web import Application, RequestHandler # Local imports from .settings import SERVER_PORTS, SERVER_POOL_PORT from .language_registry import create_evaluator_instance -from .code_evaluator import CodeEvaluator +from .grader import Grader MY_DIR = abspath(dirname(__file__)) @@ -85,19 +85,15 @@ class CodeServer(object): """Calls relevant EvaluateCode class based on language to check the answer code """ - data = self.unpack_json_to_python_obj(json_data) - code_eval_instance = CodeEvaluator(in_dir) - result = code_eval_instance.evaluate(data) #language, test_case_type, + data = json.loads(json_data) + grader = Grader(in_dir) + result = grader.evaluate(data) # Put us back into the server pool queue since we are free now. self.queue.put(self.port) return json.dumps(result) - def unpack_json_to_python_obj(self, json_data): - data = json.loads(json_data) - return data - def run(self): """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("0.0.0.0", self.port)) diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 6d4b55d..c318a82 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class CppStdioEvaluator(StdIOEvaluator): +class CppStdIOEvaluator(StdIOEvaluator): """Evaluates C StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 142d7f0..06a56e4 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -3,9 +3,9 @@ import unittest import os import shutil import tempfile -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.bash_code_evaluator import BashCodeEvaluator -from yaksh.bash_stdio_evaluator import BashStdioEvaluator +from yaksh.bash_stdio_evaluator import BashStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent @@ -44,8 +44,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") @@ -63,8 +63,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) @@ -82,8 +82,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -107,13 +107,13 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class BashStdioEvaluationTestCases(unittest.TestCase): +class BashStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " @@ -144,8 +144,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -174,8 +174,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -202,8 +202,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -229,8 +229,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 4dd6a2f..dc6fdc9 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -6,9 +6,9 @@ import tempfile from textwrap import dedent # Local import -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.cpp_code_evaluator import CppCodeEvaluator -from yaksh.cpp_stdio_evaluator import CppStdioEvaluator +from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -44,8 +44,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") @@ -62,8 +62,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -82,8 +82,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -100,8 +100,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -134,13 +134,13 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") -class CppStdioEvaluationTestCases(unittest.TestCase): +class CppStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', @@ -171,8 +171,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -202,8 +202,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -231,8 +231,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -254,8 +254,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -279,8 +279,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -302,8 +302,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -330,8 +330,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -355,8 +355,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -387,8 +387,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -417,8 +417,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -441,8 +441,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -467,8 +467,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -491,8 +491,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -520,8 +520,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index f664200..cb783b0 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -14,7 +14,7 @@ class RegistryTestCase(unittest.TestCase): ".PythonAssertionEvaluator" ) stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdioEvaluator" + "PythonStdIOEvaluator" ) code_evaluators['python'] = \ {"standardtestcase": assertion_evaluator_path, @@ -29,7 +29,7 @@ class RegistryTestCase(unittest.TestCase): ".PythonAssertionEvaluator" ) stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdioEvaluator" + "PythonStdIOEvaluator" ) class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator' diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 246a3e5..36eb6a5 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -3,13 +3,14 @@ import unittest import os import shutil import tempfile -from yaksh import code_evaluator -from yaksh.code_evaluator import CodeEvaluator -from yaksh.java_code_evaluator import JavaCodeEvaluator -from yaksh.java_stdio_evaluator import JavaStdioEvaluator -from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent +# Local Import +from yaksh import grader as gd +from yaksh.grader import Grader +from yaksh.java_code_evaluator import JavaCodeEvaluator +from yaksh.java_stdio_evaluator import JavaStdIOEvaluator + class JavaAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -23,14 +24,15 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): } ] self.in_dir = tmp_in_dir_path - code_evaluator.SERVER_TIMEOUT = 9 + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(code_evaluator.SERVER_TIMEOUT) - self.file_paths = None + " your code.").format(gd.SERVER_TIMEOUT) + def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) @@ -46,8 +48,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -64,8 +66,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get('success')) lines_of_error = len(result.get('error').splitlines()) @@ -85,8 +87,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) @@ -103,8 +105,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -145,13 +147,13 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class JavaStdioEvaluationTestCases(unittest.TestCase): +class JavaStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) @@ -162,14 +164,14 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - code_evaluator.SERVER_TIMEOUT = 4 + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(code_evaluator.SERVER_TIMEOUT) - self.file_paths = None + " your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) @@ -193,8 +195,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -225,8 +227,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -251,8 +253,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -275,14 +277,13 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): - user_answer = dedent(""" class Test {public static void main(String[] args){ @@ -300,8 +301,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -329,8 +330,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -360,8 +361,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -401,8 +402,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 29336d6..e638049 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -6,9 +6,9 @@ import shutil from textwrap import dedent # Local import -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdio_evaluator import PythonStdioEvaluator +from yaksh.python_stdio_evaluator import PythonStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT class PythonAssertionEvaluationTestCases(unittest.TestCase): @@ -44,8 +44,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -65,8 +65,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -98,8 +98,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -125,8 +125,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -157,8 +157,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -191,8 +191,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -222,8 +222,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -254,8 +254,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -286,8 +286,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -321,8 +321,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -352,8 +352,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertIn("Correct answer", result.get('error')) @@ -389,8 +389,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -430,8 +430,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -471,8 +471,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -505,8 +505,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -537,8 +537,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -567,8 +567,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -599,8 +599,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), "Correct answer\n") @@ -628,8 +628,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), timeout_msg) diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index e879cda..0275ee8 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -4,10 +4,9 @@ import os import shutil import tempfile -from yaksh import code_evaluator -from yaksh.code_evaluator import CodeEvaluator +from yaksh import grader as gd +from yaksh.grader import Grader from yaksh.scilab_code_evaluator import ScilabCodeEvaluator -from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -17,14 +16,14 @@ class ScilabEvaluationTestCases(unittest.TestCase): "weight": 0.0 }] self.in_dir = tmp_in_dir_path + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) - code_evaluator.SERVER_TIMEOUT = 9 - self.file_paths = None + " in your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 shutil.rmtree(self.in_dir) def test_correct_answer(self): @@ -40,8 +39,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -59,8 +58,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue('error' in result.get("error")) @@ -79,8 +78,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -88,7 +87,6 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(lines_of_error > 1) def test_infinite_loop(self): - code_evaluator.SERVER_TIMEOUT = 4 user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") kwargs = { @@ -101,8 +99,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) diff --git a/yaksh/forms.py b/yaksh/forms.py index 1931fad..6fbaf5d 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,6 +1,6 @@ from django import forms from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course,\ - QuestionPaper, StandardTestCase, StdioBasedTestCase + QuestionPaper, StandardTestCase, StdIOBasedTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -37,7 +37,7 @@ question_types = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - ("stdiobasedtestcase", "Stdio Based Testcase"), + ("stdiobasedtestcase", "StdIO Based Testcase"), ("mcqtestcase", "MCQ Testcase"), ) diff --git a/yaksh/grader.py b/yaksh/grader.py new file mode 100644 index 0000000..ef349e0 --- /dev/null +++ b/yaksh/grader.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +from __future__ import unicode_literals +import sys +import pwd +import os +import stat +import contextlib +from os.path import isdir, dirname, abspath, join, isfile, exists +import signal +import traceback +from multiprocessing import Process, Queue +import subprocess +import re + +try: + from SimpleXMLRPCServer import SimpleXMLRPCServer +except ImportError: + # The above import will not work on Python-3.x. + from xmlrpc.server import SimpleXMLRPCServer + +# Local imports +from .settings import SERVER_TIMEOUT +from .language_registry import create_evaluator_instance + + +MY_DIR = abspath(dirname(__file__)) + +registry = None + +# Raised when the code times-out. +# c.f. http://pguides.net/python/timeout-a-function +class TimeoutException(Exception): + pass + +@contextlib.contextmanager +def change_dir(path): + cur_dir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(cur_dir) + + +def timeout_handler(signum, frame): + """A handler for the ALARM signal.""" + raise TimeoutException('Code took too long to run.') + + +def create_signal_handler(): + """Add a new signal handler for the execution of this code.""" + prev_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + return prev_handler + + +def set_original_signal_handler(old_handler=None): + """Set back any original signal handler.""" + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + return + else: + raise Exception("Signal Handler: object cannot be NoneType") + + +def delete_signal_handler(): + signal.alarm(0) + return + + +class Grader(object): + """Tests the code obtained from Code Server""" + def __init__(self, in_dir=None): + msg = 'Code took more than %s seconds to run. You probably '\ + 'have an infinite loop in your code.' % SERVER_TIMEOUT + self.timeout_msg = msg + self.in_dir = in_dir if in_dir else MY_DIR + + + def evaluate(self, kwargs): #language, test_case_type, + """Evaluates given code with the test cases based on + given arguments in test_case_data. + + The ref_code_path is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + If the optional `in_dir` keyword argument is supplied it changes the + directory to that directory (it does not change it back to the original + when done). + + Returns + ------- + + A tuple: (success, error message, weight). + """ + + self.setup() + test_case_instances = self.get_evaluator_objects(kwargs) + with change_dir(self.in_dir): + success, error, mark = self.safe_evaluate(test_case_instances) + self.teardown() + + result = {'success': success, 'error': error, 'weight': mark} + return result + + # Private Protocol ########## + def setup(self): + if self.in_dir: + if not os.path.exists(self.in_dir): + os.makedirs(self.in_dir) + + def get_evaluator_objects(self, kwargs): + metadata = kwargs.get('metadata') + test_case_data = kwargs.get('test_case_data') + test_case_instances = [] + + for test_case in test_case_data: + test_case_instance = create_evaluator_instance(metadata, test_case) + test_case_instances.append(test_case_instance) + + return test_case_instances + + + def safe_evaluate(self, test_case_instances): + """ + Handles code evaluation along with compilation, signal handling + and Exception handling + """ + # Add a new signal handler for the execution of this code. + prev_handler = create_signal_handler() + success = False + test_case_success_status = [False] * len(test_case_instances) + error = "" + weight = 0.0 + + # Do whatever testing needed. + try: + # Run evaluator selection registry here + for idx, test_case_instance in enumerate(test_case_instances): + test_case_success = False + test_case_instance.compile_code() + test_case_success, err, mark_fraction = test_case_instance.check_code() + if test_case_success: + weight += mark_fraction + + error += err + "\n" + test_case_success_status[idx] = test_case_success + + success = all(test_case_success_status) + + for test_case_instance in test_case_instances: + test_case_instance.teardown() + + except TimeoutException: + error = self.timeout_msg + except OSError: + msg = traceback.format_exc(limit=0) + error = "Error: {0}".format(msg) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) + if len(tb_list) > 2: + del tb_list[1:3] + error = "Error: {0}".format("".join(tb_list)) + finally: + # Set back any original signal handler. + set_original_signal_handler(prev_handler) + + return success, error, weight + + def teardown(self): + # Cancel the signal + delete_signal_handler() diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index a5f5540..a854847 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class JavaStdioEvaluator(StdIOEvaluator): +class JavaStdIOEvaluator(StdIOEvaluator): """Evaluates Java StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 8d3aad2..994e9ed 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -14,15 +14,10 @@ def get_registry(): registry = _LanguageRegistry() return registry -# def unpack_json_to_python_obj(json_data): -# data = json.loads(json_data) -# return data - -def create_evaluator_instance(metadata, test_case): #create_evaluator_instance +def create_evaluator_instance(metadata, test_case): """Create instance of relevant EvaluateCode class based on language""" registry = get_registry() cls = registry.get_class(metadata.get('language'), test_case.get('test_case_type')) - # instance = cls(in_dir) instance = cls(metadata, test_case) return instance diff --git a/yaksh/models.py b/yaksh/models.py index 9ba85ba..6e1744c 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1160,7 +1160,7 @@ class StandardTestCase(TestCase): ) -class StdioBasedTestCase(TestCase): +class StdIOBasedTestCase(TestCase): expected_input = models.CharField(max_length=100, blank=True) expected_output = models.CharField(max_length=100) weight = models.IntegerField(default=1.0) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 0b49cb0..4d44838 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -9,7 +9,7 @@ import importlib # Local imports from .file_utils import copy_files, delete_files from .base_evaluator import BaseEvaluator -from .code_evaluator import TimeoutException +from .grader import TimeoutException class PythonAssertionEvaluator(BaseEvaluator): diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 07bd59c..da0c954 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -28,7 +28,7 @@ def redirect_stdout(): sys.stdout = old_target # restore to the previous value -class PythonStdioEvaluator(BaseEvaluator): +class PythonStdIOEvaluator(BaseEvaluator): """Tests the Python code obtained from Code Server""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 97e40a8..cc3c401 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -26,12 +26,6 @@ class ScilabCodeEvaluator(BaseEvaluator): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - # def setup(self): - # super(ScilabCodeEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = \ - # self.create_submit_code_file('function.sci') - def teardown(self): # Delete the created file. os.remove(self.submit_code_path) @@ -61,7 +55,7 @@ class ScilabCodeEvaluator(BaseEvaluator): cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format( clean_ref_path ) - cmd += ' | timeout 8 scilab-cli -nb' + cmd += ' | scilab-cli -nb' ret = self._run_command(cmd, shell=True, stdout=subprocess.PIPE, diff --git a/yaksh/settings.py b/yaksh/settings.py index e18d310..0e432cf 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -19,40 +19,20 @@ SERVER_TIMEOUT = 4 # host.org/foo/exam set URL_ROOT='/foo' URL_ROOT = '' -# code_evaluators = { -# "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", -# "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" -# }, -# "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", -# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" -# }, -# "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", -# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" -# }, -# "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", -# "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, - -# "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", -# "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" -# }, - -# "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, -# } - code_evaluators = { "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" + "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator" }, "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" }, "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" }, "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", - "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, + "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator"}, "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", - "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" + "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator" }, "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/test_models.py b/yaksh/test_models.py index 0db1b82..317c832 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -1,7 +1,7 @@ import unittest from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdioBasedTestCase, FileUpload, McqTestCase + StdIOBasedTestCase, FileUpload, McqTestCase import json from datetime import datetime, timedelta from django.utils import timezone @@ -879,7 +879,7 @@ class TestCaseTestCases(unittest.TestCase): test_case='assert myfunc(12, 13) == 15', type='standardtestcase' ) - self.stdout_based_testcase = StdioBasedTestCase( + self.stdout_based_testcase = StdIOBasedTestCase( question=self.question2, expected_output='Hello World', type='standardtestcase' diff --git a/yaksh/test_views.py b/yaksh/test_views.py index 30ebcaa..2419591 100644 --- a/yaksh/test_views.py +++ b/yaksh/test_views.py @@ -9,7 +9,7 @@ from django.utils import timezone from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdioBasedTestCase, has_profile + StdIOBasedTestCase, has_profile class TestProfile(TestCase): diff --git a/yaksh/views.py b/yaksh/views.py index 0d77426..89274df 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -25,7 +25,7 @@ import six # Local imports. from yaksh.models import get_model_class, Quiz, Question, QuestionPaper, QuestionSet, Course from yaksh.models import Profile, Answer, AnswerPaper, User, TestCase, FileUpload,\ - has_profile, StandardTestCase, McqTestCase, StdioBasedTestCase, HookTestCase + has_profile, StandardTestCase, McqTestCase, StdIOBasedTestCase, HookTestCase from yaksh.forms import UserRegisterForm, UserLoginForm, QuizForm,\ QuestionForm, RandomQuestionForm,\ QuestionFilterForm, CourseForm, ProfileForm, UploadFileForm,\ -- cgit