diff options
Diffstat (limited to 'yaksh/evaluator_tests/test_python_evaluation.py')
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 87 |
1 files changed, 47 insertions, 40 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index e638049..c58d7f1 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -11,7 +11,14 @@ from yaksh.python_assertion_evaluator import PythonAssertionEvaluator from yaksh.python_stdio_evaluator import PythonStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT -class PythonAssertionEvaluationTestCases(unittest.TestCase): + +class EvaluatorBaseTest(unittest.TestCase): + def assert_correct_output(self, expected_output, actual_output): + actual_output_as_string = ''.join(actual_output) + self.assertIn(expected_output, actual_output_as_string) + + +class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) @@ -49,7 +56,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) def test_incorrect_answer(self): # Given @@ -70,13 +76,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get('success')) - self.assertIn('AssertionError in: assert(add(1,2)==3)', + self.assert_correct_output('AssertionError in: assert(add(1,2)==3)', result.get('error') ) - self.assertIn('AssertionError in: assert(add(-1,2)==1)', + self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)', result.get('error') ) - self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)', result.get('error') ) @@ -104,10 +110,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('weight'), 2.0) - self.assertIn('AssertionError in: assert(add(-1,2)==1)', + self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)', result.get('error') ) - self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)', result.get('error') ) @@ -130,7 +136,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), self.timeout_msg) + self.assert_correct_output(self.timeout_msg, result.get('error')) def test_syntax_error(self): # Given @@ -159,13 +165,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_indent_error(self): # Given @@ -193,13 +200,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + err = result.get("error")[0].splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_name_error(self): # Given @@ -224,13 +231,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(6, len(err)) for msg in name_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_recursion_error(self): # Given @@ -256,12 +264,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) for msg in recursion_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_type_error(self): # Given @@ -288,13 +297,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(6, len(err)) for msg in type_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_value_error(self): # Given @@ -323,13 +333,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(6, len(err)) for msg in value_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_file_based_assert(self): # Given @@ -356,7 +367,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result = grader.evaluate(kwargs) # Then - self.assertIn("Correct answer", result.get('error')) self.assertTrue(result.get('success')) def test_single_testcase_error(self): @@ -378,8 +388,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid syntax" ] - kwargs = { - 'metadata': { + kwargs = {'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, @@ -391,13 +400,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() - + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() + # Then self.assertFalse(result.get("success")) - self.assertEqual(6, len(err)) + self.assertEqual(5, len(err)) for msg in syntax_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) def test_multiple_testcase_error(self): @@ -432,15 +442,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error").splitlines() + error_as_str = ''.join(result.get("error")) + err = error_as_str.splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(7, len(err)) + self.assertEqual(2, len(err)) for msg in name_error_msg: - self.assertIn(msg, result.get("error")) + self.assert_correct_output(msg, result.get("error")) -class PythonStdIOEvaluationTestCases(unittest.TestCase): +class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) @@ -476,7 +487,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): # Then self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) def test_correct_answer_list(self): # Given @@ -510,7 +520,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): # Then self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) def test_correct_answer_string(self): # Given @@ -542,7 +551,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): # Then self.assertTrue(result.get('success')) - self.assertIn("Correct answer", result.get('error')) def test_incorrect_answer_integer(self): # Given @@ -572,7 +580,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get('success')) - self.assertIn("Incorrect answer", result.get('error')) + self.assert_correct_output("Incorrect answer", result.get('error')) def test_file_based_answer(self): # Given @@ -603,7 +611,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): result = grader.evaluate(kwargs) # Then - self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_infinite_loop(self): @@ -632,7 +639,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): result = grader.evaluate(kwargs) # Then - self.assertEqual(result.get('error'), timeout_msg) + self.assert_correct_output(timeout_msg, result.get('error')) self.assertFalse(result.get('success')) |