diff options
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 1012 | ||||
| -rw-r--r-- | yaksh/python_assertion_evaluator.py | 8 | ||||
| -rw-r--r-- | yaksh/python_stdio_evaluator.py | 49 |
3 files changed, 411 insertions, 658 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 43cd0ea..688002f 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -88,6 +88,148 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result.get('error') ) + def test_partial_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" + test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': True + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('weight'), 2.0) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) + + def test_infinite_loop(self): + # Given + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) + + def test_syntax_error(self): + # Given + user_answer = dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) + + def test_indent_error(self): + # Given + user_answer = dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = ["Traceback", "call", + "File", + "line", + "<string>", + "IndentationError", + "indented block" + ] + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } + + # When + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in indent_error_msg: + self.assertIn(msg, result.get("error")) def test_name_error(self): # Given @@ -98,20 +240,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "name", "defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) @@ -125,15 +277,25 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "call", "maximum recursion depth exceeded" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -152,20 +314,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "TypeError", "argument" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(3, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error")) @@ -182,41 +354,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid literal", "base" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(9, len(err)) + self.assertEqual(4, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] + self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertIn("Correct answer", result.get('error')) @@ -227,7 +419,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nasert palindrome(s)==False', "weight": 0.0 } ] @@ -239,20 +432,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "SyntaxError", "invalid syntax" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(6, len(err)) + self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error")) @@ -262,45 +465,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(palindrome("abba")==True)', "weight": 0.0 }, - {"test_case": 's="abbb"\nassert palindrome(S)==False', + {"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nassert palindrome(S)==False', "weight": 0.0 } ] name_error_msg = ["Traceback", "call", + "File", + "line", + "<string>", "NameError", "name 'S' is not defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) - self.assertEqual(7, len(err)) + self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) - class PythonStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) self.file_paths = None + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path def test_correct_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -310,14 +529,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -325,7 +552,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given - self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]", "weight": 0.0 }] @@ -338,14 +566,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -353,7 +590,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given - self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2", "weight": 0.0 }] @@ -364,14 +602,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a.count(b)) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -379,7 +626,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_incorrect_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -389,14 +637,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a-b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -404,7 +660,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given - self.test_case_data = [{"expected_input": "", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "", "expected_output": "2", "weight": 0.0 }] @@ -416,15 +673,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a[0]) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths, + # 'partial_grading': False + # } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertEqual(result.get('error'), "Correct answer\n") @@ -432,7 +697,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): # Given - test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -440,552 +706,28 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'partial_grading': False - } + # kwargs = {'user_answer': user_answer, + # 'test_case_data': test_case_data, + # 'partial_grading': False + # } + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) # Then self.assertEqual(result.get('error'), timeout_msg) self.assertFalse(result.get('success')) -# class PythonAssertionEvaluationTestCases(unittest.TestCase): -# def setUp(self): -# with open('/tmp/test.txt', 'wb') as f: -# f.write('2'.encode('ascii')) -# tmp_in_dir_path = tempfile.mkdtemp() -# self.in_dir = tmp_in_dir_path -# self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, -# {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, -# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, -# ] -# self.timeout_msg = ("Code took more than {0} seconds to run. " -# "You probably have an infinite loop in" -# " your code.").format(SERVER_TIMEOUT) -# self.file_paths = None - -# def tearDown(self): -# os.remove('/tmp/test.txt') -# shutil.rmtree(self.in_dir) - -# def test_correct_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn a + b" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_incorrect_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn a - b" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertIn('AssertionError in: assert(add(1,2)==3)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,2)==1)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', -# result.get('error') -# ) - -# def test_partial_incorrect_answer(self): -# # Given -# user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" -# test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, -# {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, -# {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': True -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertEqual(result.get('weight'), 2.0) -# self.assertIn('AssertionError in: assert(add(-1,2)==1)', -# result.get('error') -# ) -# self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', -# result.get('error') -# ) - -# def test_infinite_loop(self): -# # Given -# user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertEqual(result.get('error'), self.timeout_msg) - -# def test_syntax_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b); -# return a + b -# """) -# syntax_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "<string>", -# "SyntaxError", -# "invalid syntax" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in syntax_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_indent_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# return a + b -# """) -# indent_error_msg = ["Traceback", "call", -# "File", -# "line", -# "<string>", -# "IndentationError", -# "indented block" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False - -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in indent_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_name_error(self): -# # Given -# user_answer = "" -# name_error_msg = ["Traceback", -# "call", -# "NameError", -# "name", -# "defined" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in name_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_recursion_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# return add(3, 3) -# """) -# recursion_error_msg = ["Traceback", -# "call", -# "maximum recursion depth exceeded" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# for msg in recursion_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_type_error(self): -# # Given -# user_answer = dedent(""" -# def add(a): -# return a + b -# """) -# type_error_msg = ["Traceback", -# "call", -# "TypeError", -# "argument" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in type_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_value_error(self): -# # Given -# user_answer = dedent(""" -# def add(a, b): -# c = 'a' -# return int(a) + int(b) + int(c) -# """) -# value_error_msg = ["Traceback", -# "call", -# "ValueError", -# "invalid literal", -# "base" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(4, len(err)) -# for msg in value_error_msg: -# self.assertIn(msg, result.get("error")) - -# def test_file_based_assert(self): -# # Given -# self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] -# self.file_paths = [('/tmp/test.txt', False)] -# user_answer = dedent(""" -# def ans(): -# with open("test.txt") as f: -# return f.read()[0] -# """) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertIn("Correct answer", result.get('error')) -# self.assertTrue(result.get('success')) - -# def test_single_testcase_error(self): -# # Given -# """ Tests the user answer with just an incorrect test case """ - -# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" -# test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', -# "weight": 0.0 -# } -# ] -# syntax_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "<string>", -# "SyntaxError", -# "invalid syntax" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(5, len(err)) -# for msg in syntax_error_msg: -# self.assertIn(msg, result.get("error")) - - -# def test_multiple_testcase_error(self): -# """ Tests the user answer with an correct test case -# first and then with an incorrect test case """ -# # Given -# user_answer = "def palindrome(a):\n\treturn a == a[::-1]" -# test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', -# "weight": 0.0 -# }, -# {"test_case": 's="abbb"\nassert palindrome(S)==False', -# "weight": 0.0 -# } -# ] -# name_error_msg = ["Traceback", -# "call", -# "File", -# "line", -# "<string>", -# "NameError", -# "name 'S' is not defined" -# ] -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonAssertionEvaluator() -# result = evaluator.evaluate(**kwargs) -# err = result.get("error").splitlines() - -# # Then -# self.assertFalse(result.get("success")) -# self.assertEqual(3, len(err)) -# for msg in name_error_msg: -# self.assertIn(msg, result.get("error")) - - -# class PythonStdIOEvaluationTestCases(unittest.TestCase): -# def setUp(self): -# with open('/tmp/test.txt', 'wb') as f: -# f.write('2'.encode('ascii')) -# self.file_paths = None - -# def test_correct_answer_integer(self): -# # Given -# self.test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# a = int(input()) -# b = int(input()) -# print(a+b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_correct_answer_list(self): -# # Given -# self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", -# "expected_output": "[1, 2, 3, 5, 6, 7]", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# from six.moves import input -# input_a = input() -# input_b = input() -# a = [int(i) for i in input_a.split(',')] -# b = [int(i) for i in input_b.split(',')] -# print(a+b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_correct_answer_string(self): -# # Given -# self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), -# "expected_output": "2", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# from six.moves import input -# a = str(input()) -# b = str(input()) -# print(a.count(b)) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertTrue(result.get('success')) -# self.assertIn("Correct answer", result.get('error')) - -# def test_incorrect_answer_integer(self): -# # Given -# self.test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# user_answer = dedent(""" -# a = int(input()) -# b = int(input()) -# print(a-b) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertFalse(result.get('success')) -# self.assertIn("Incorrect answer", result.get('error')) - -# def test_file_based_answer(self): -# # Given -# self.test_case_data = [{"expected_input": "", -# "expected_output": "2", -# "weight": 0.0 -# }] -# self.file_paths = [('/tmp/test.txt', False)] - -# user_answer = dedent(""" -# with open("test.txt") as f: -# a = f.read() -# print(a[0]) -# """ -# ) -# kwargs = {'user_answer': user_answer, -# 'test_case_data': self.test_case_data, -# 'file_paths': self.file_paths, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertEqual(result.get('error'), "Correct answer\n") -# self.assertTrue(result.get('success')) - -# def test_infinite_loop(self): -# # Given -# test_case_data = [{"expected_input": "1\n2", -# "expected_output": "3", -# "weight": 0.0 -# }] -# timeout_msg = ("Code took more than {0} seconds to run. " -# "You probably have an infinite loop in" -# " your code.").format(SERVER_TIMEOUT) -# user_answer = "while True:\n\tpass" -# kwargs = {'user_answer': user_answer, -# 'test_case_data': test_case_data, -# 'partial_grading': False -# } - -# # When -# evaluator = PythonStdioEvaluator() -# result = evaluator.evaluate(**kwargs) - -# # Then -# self.assertEqual(result.get('error'), timeout_msg) -# self.assertFalse(result.get('success')) if __name__ == '__main__': unittest.main() diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 5f1b29e..3e172ec 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -27,10 +27,10 @@ class PythonAssertionEvaluator(object): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - def __del__(self): - # Delete the created file. - if self.files: - delete_files(self.files) + # def __del__(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) # def setup(self): diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 1506685..b618a0b 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -31,24 +31,36 @@ def redirect_stdout(): class PythonStdioEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def setup(self): - super(PythonStdioEvaluator, self).setup() + # def setup(self): + # super(PythonStdioEvaluator, self).setup() + # self.files = [] + + # def teardown(self): + # # Delete the created file. + # if self.files: + # delete_files(self.files) + # super(PythonStdioEvaluator, self).teardown() + def __init__(self, metadata, test_case_data): self.files = [] - def teardown(self): - # Delete the created file. - if self.files: - delete_files(self.files) - super(PythonStdioEvaluator, self).teardown() + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.expected_input = test_case_data.get('expected_input') + self.expected_output = test_case_data.get('expected_output') + self.weight = test_case_data.get('weight') - def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): - if file_paths: - self.files = copy_files(file_paths) - submitted = compile(user_answer, '<string>', mode='exec') - if expected_input: + def compile_code(self): # user_answer, file_paths, expected_input, expected_output, weight): + if self.file_paths: + self.files = copy_files(self.file_paths) + submitted = compile(self.user_answer, '<string>', mode='exec') + if self.expected_input: input_buffer = StringIO() - input_buffer.write(expected_input) + input_buffer.write(self.expected_input) input_buffer.seek(0) sys.stdin = input_buffer with redirect_stdout() as output_buffer: @@ -57,16 +69,15 @@ class PythonStdioEvaluator(CodeEvaluator): self.output_value = output_buffer.getvalue().rstrip("\n") return self.output_value - def check_code(self, user_answer, file_paths, partial_grading, expected_input, - expected_output, weight): + def check_code(self): # user_answer, file_paths, partial_grading, expected_input, expected_output, weight): success = False test_case_weight = 0.0 tb = None - if self.output_value == expected_output: + if self.output_value == self.expected_output: success = True err = "Correct answer" - test_case_weight = weight + test_case_weight = self.weight else: success = False err = dedent(""" @@ -74,8 +85,8 @@ class PythonStdioEvaluator(CodeEvaluator): Given input - {0} Expected output - {1} Your output - {2} - """.format(expected_input, - expected_output, + """.format(self.expected_input, + self.expected_output, self.output_value ) ) |