diff options
| -rw-r--r-- | yaksh/error_messages.py | 3 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_bash_evaluation.py | 8 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 16 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_java_evaluation.py | 12 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 154 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_stdio_evaluator.py | 2 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_scilab_evaluation.py | 4 | ||||
| -rw-r--r-- | yaksh/tests/test_code_server.py | 6 |
8 files changed, 113 insertions, 92 deletions
diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py index 5453edd..25b690b 100644 --- a/yaksh/error_messages.py +++ b/yaksh/error_messages.py @@ -9,7 +9,8 @@ def prettify_exceptions(exception, message, traceback=None, testcase=None): "traceback": traceback, "message": message } - + if exception == "RecursionError": + err["traceback"] = None if exception == 'AssertionError': value = ("Expected answer from the" + " test case did not match the output") diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 2faa7bf..5542710 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -104,7 +104,9 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -533,7 +535,9 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 0898b3f..162d90c 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -151,7 +151,9 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -406,7 +408,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -616,7 +620,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_cpp_only_stdout(self): # Given @@ -976,7 +982,9 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 5ddf8cd..35b64d0 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -160,7 +160,9 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -405,7 +407,9 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -845,7 +849,9 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index a2faf77..71d7732 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -24,9 +24,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, + self.test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(add(1,2)==3)', + 'weight': 0.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,2)==1)', + 'weight': 0.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,-2)==-3)', + 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -76,23 +82,29 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output('AssertionError in:\n assert(add(1,2)==3)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,2)==1)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,-2)==-3)', - result.get('error') - ) + given_test_case_list = [tc["test_case"] for tc in self.test_case_data] + for error in result.get("error"): + self.assertEqual(error['exception'], 'AssertionError') + self.assertEqual(error['message'], + "Expected answer from the test case did not match the output" + ) + error_testcase_list = [tc['test_case'] for tc in result.get('error')] + self.assertEqual(error_testcase_list, given_test_case_list) + def test_partial_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" - test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} - ] + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,2)==1)', + 'weight': 1.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,-2)==-3)', + 'weight': 1.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(1,2)==3)', + 'weight': 2.0} + ] kwargs = { 'metadata': { 'user_answer': user_answer, @@ -110,13 +122,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('weight'), 2.0) - self.assert_correct_output('AssertionError in:\n assert(add(-1,2)==1)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,-2)==-3)', - result.get('error') - ) - + given_test_case_list = [tc["test_case"] for tc in self.test_case_data] + given_test_case_list.remove('assert(add(1,2)==3)') + for error in result.get("error"): + self.assertEqual(error['exception'], 'AssertionError') + self.assertEqual(error['message'], + "Expected answer from the test case did not match the output" + ) + error_testcase_list = [tc['test_case'] for tc in result.get('error')] + self.assertEqual(error_testcase_list, given_test_case_list) def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" @@ -136,7 +150,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_syntax_error(self): # Given @@ -165,14 +181,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) for msg in syntax_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_indent_error(self): # Given @@ -200,13 +214,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error")[0].splitlines() + err = result.get("error")[0]["traceback"].splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, + result.get("error")[0]['traceback'] + ) def test_name_error(self): # Given @@ -231,14 +247,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(25, len(err)) + err = result.get("error")[0]["traceback"] for msg in name_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assertIn(msg, err) def test_recursion_error(self): # Given @@ -246,10 +257,7 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): def add(a, b): return add(3, 3) """) - recursion_error_msg = ["Traceback", - "maximum recursion depth exceeded" - ] - + recursion_error_msg = "maximum recursion depth exceeded" kwargs = { 'metadata': { 'user_answer': user_answer, @@ -263,13 +271,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['message'] # Then self.assertFalse(result.get("success")) - for msg in recursion_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(recursion_error_msg, err) def test_type_error(self): # Given @@ -296,14 +302,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(25, len(err)) for msg in type_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_value_error(self): # Given @@ -332,18 +336,19 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(28, len(err)) for msg in value_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}] + self.test_case_data = [{"test_case_type": "standardtestcase", + "test_case": "assert(ans()=='2')", + "weight": 0.0} + ] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" def ans(): @@ -369,20 +374,17 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): self.assertTrue(result.get('success')) def test_single_testcase_error(self): - # Given """ Tests the user answer with just an incorrect test case """ + # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case_type": "standardtestcase", - "test_case": 's="abbb"\nasert palindrome(s)==False', - "weight": 0.0 - } + "test_case": 's="abbb"\nasert palindrome(s)==False', + "weight": 0.0 + } ] syntax_error_msg = ["Traceback", "call", - "File", - "line", - "<string>", "SyntaxError", "invalid syntax" ] @@ -399,14 +401,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(13, len(err)) for msg in syntax_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_multiple_testcase_error(self): @@ -415,13 +415,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case_type": "standardtestcase", - "test_case": 'assert(palindrome("abba")==True)', - "weight": 0.0 - }, + "test_case": 'assert(palindrome("abba")==True)', + "weight": 0.0}, {"test_case_type": "standardtestcase", - "test_case": 's="abbb"\nassert palindrome(S)==False', - "weight": 0.0 - } + "test_case": 's="abbb"\nassert palindrome(S)==False', + "weight": 0.0} ] name_error_msg = ["Traceback", "call", @@ -441,14 +439,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(11, len(err)) for msg in name_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assertIn(msg, err) def test_unicode_literal_bug(self): # Given @@ -674,7 +670,9 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): result = grader.evaluate(kwargs) # Then - self.assert_correct_output(timeout_msg, result.get('error')) + self.assert_correct_output(timeout_msg, + result.get("error")[0]["message"] + ) self.assertFalse(result.get('success')) @@ -915,7 +913,9 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_assignment_upload(self): # Given diff --git a/yaksh/evaluator_tests/test_python_stdio_evaluator.py b/yaksh/evaluator_tests/test_python_stdio_evaluator.py index 8877544..9b8d702 100644 --- a/yaksh/evaluator_tests/test_python_stdio_evaluator.py +++ b/yaksh/evaluator_tests/test_python_stdio_evaluator.py @@ -1,4 +1,4 @@ -from yaksh.compare_stdio import compare_outputs +from yaksh.error_messages import compare_outputs def test_compare_outputs(): exp = "5\n5\n" diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index c3a1c83..f7a9925 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -137,7 +137,9 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest): result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 5f80f2d..1309624 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -61,7 +61,7 @@ class TestCodeServer(unittest.TestCase): # Then data = json.loads(result.get('result')) self.assertFalse(data['success']) - self.assertTrue('infinite loop' in data['error'][0]) + self.assertTrue('infinite loop' in data['error'][0]['message']) def test_correct_answer(self): # Given @@ -104,7 +104,7 @@ class TestCodeServer(unittest.TestCase): # Then data = json.loads(result.get('result')) self.assertFalse(data['success']) - self.assertTrue('AssertionError' in data['error'][0]) + self.assertTrue('AssertionError' in data['error'][0]['exception']) def test_multiple_simultaneous_hits(self): # Given @@ -143,7 +143,7 @@ class TestCodeServer(unittest.TestCase): for i in range(N): data = results.get() self.assertFalse(data['success']) - self.assertTrue('infinite loop' in data['error'][0]) + self.assertTrue('infinite loop' in data['error'][0]['message']) def test_server_pool_status(self): # Given |