diff options
Diffstat (limited to 'yaksh')
| -rw-r--r-- | yaksh/code_evaluator.py | 3 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 391 | ||||
| -rw-r--r-- | yaksh/python_assertion_evaluator.py | 4 |
3 files changed, 271 insertions, 127 deletions
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index e139b55..8a9b7a6 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -101,6 +101,7 @@ class CodeEvaluator(object): # Do whatever testing needed. try: for test_case in test_case_data: + success = False self.compile_code(user_answer, file_paths, **test_case) success, err = self.check_code(user_answer, file_paths, **test_case) if not success: @@ -116,8 +117,6 @@ class CodeEvaluator(object): tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) if len(tb_list) > 2: del tb_list[1:3] - else: - pass err = "Error: {0}".format("".join(tb_list)) finally: # Set back any original signal handler. diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 3c07907..2a109d5 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -9,164 +9,205 @@ from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, - ] + {"test_case": 'assert(add(-1,2)==1)'}, + {"test_case": 'assert(add(-1,-2)==-3)'}, + ] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) self.file_paths = None def test_correct_answer(self): + # Given user_answer = "def add(a,b):\n\treturn a + b" - get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer") def test_incorrect_answer(self): + # Given user_answer = "def add(a,b):\n\treturn a - b" - get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), - "AssertionError in: assert(add(1,2)==3)" - ) + "AssertionError in: assert(add(1,2)==3)" + ) def test_infinite_loop(self): + # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg) - + def test_syntax_error(self): + # Given user_answer = dedent(""" def add(a, b); return a + b """) - syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "<string>", - "SyntaxError", - "invalid syntax" - ] - get_class = PythonAssertionEvaluator() + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error")) def test_indent_error(self): + # Given user_answer = dedent(""" def add(a, b): return a + b """) - indent_error_msg = ["Traceback", "call", - "File", - "line", - "<string>", - "IndentationError", - "indented block" - ] - get_class = PythonAssertionEvaluator() + indent_error_msg = ["Traceback", "call", + "File", + "line", + "<string>", + "IndentationError", + "indented block" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assertIn(msg, result.get("error")) def test_name_error(self): + # Given user_answer = "" - name_error_msg = ["Traceback", - "call", - "NameError", - "name", - "defined" - ] - get_class = PythonAssertionEvaluator() + name_error_msg = ["Traceback", + "call", + "NameError", + "name", + "defined" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) def test_recursion_error(self): + # Given user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = ["Traceback", - "call", - "RuntimeError", - "maximum recursion depth exceeded" - ] - get_class = PythonAssertionEvaluator() + "call", + "RuntimeError", + "maximum recursion depth exceeded" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(969, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error")) def test_type_error(self): + # Given user_answer = dedent(""" def add(a): return a + b """) - type_error_msg = ["Traceback", - "call", - "TypeError", - "exactly", - "argument" - ] - get_class = PythonAssertionEvaluator() + type_error_msg = ["Traceback", + "call", + "TypeError", + "exactly", + "argument" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error")) def test_value_error(self): + # Given user_answer = dedent(""" def add(a, b): c = 'a' @@ -178,19 +219,24 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid literal", "base" ] - get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(4, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) def test_file_based_assert(self): + # Given self.test_case_data = [{"test_case": "assert(ans()=='2')"}] self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] user_answer = dedent(""" @@ -198,15 +244,84 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): with open("test.txt") as f: return f.read()[0] """) - get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) + def test_single_testcase_error(self): + # Given + """ Tests the user answer with just an incorrect test case """ + + user_answer = "def palindrome(a):\n\treturn a == a[::-1]" + test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'} + ] + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) + + + def test_multiple_testcase_error(self): + """ Tests the user answer with an correct test case + first and then with an incorrect test case """ + # Given + user_answer = "def palindrome(a):\n\treturn a == a[::-1]" + test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'}, + {"test_case": 's="abbb"\nassert palindrome(S)==False'} + ] + name_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "NameError", + "name 'S' is not defined" + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + err = result.get("error").splitlines() + + # Then + self.assertFalse(result.get("success")) + self.assertEqual(3, len(err)) + for msg in name_error_msg: + self.assertIn(msg, result.get("error")) + + class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"expected_input": None, @@ -216,86 +331,106 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) - self.file_paths = None def test_correct_answer(self): + # Given user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" - get_class = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) def test_incorrect_answer(self): + # Given user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" - get_class = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertIn("Incorrect Answer", result.get('error')) def test_infinite_loop(self): + # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)" - get_class = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } - -class PythonStdIOEvaluator(unittest.TestCase): + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertEqual(result.get('error'), self.timeout_msg) + self.assertFalse(result.get('success')) + + +class PythonStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) + self.file_paths = None - def test_add_two_integers_correct(self): + def test_correct_answer_integer(self): + # Given self.test_case_data = [{"expected_input": "1\n2", "expected_output": "3" }] user_answer = dedent(""" - a = int(raw_input()) - b = int(raw_input()) + a = input() + b = input() print a+b """ ) - - get_class = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_add_two_lists(self): + def test_correct_answer_list(self): + # Given self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]", "expected_output": "[1, 2, 3, 5, 6, 7]" }] user_answer = dedent(""" - from ast import literal_eval - a = literal_eval(raw_input()) - b = literal_eval(raw_input()) + a = input() + b = input() print a+b """ ) - - get_class = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_find_string_in_sentence(self): + def test_correct_answer_string(self): + # Given self.test_case_data = [{"expected_input": """the quick brown fox jumps\ over the lazy dog\nthe""", "expected_output": "2" @@ -306,37 +441,43 @@ class PythonStdIOEvaluator(unittest.TestCase): print (a.count(b)) """ ) - - get_class = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_add_two_integers_incorrect(self): + def test_incorrect_answer_integer(self): + # Given self.test_case_data = [{"expected_input": "1\n2", "expected_output": "3" }] user_answer = dedent(""" - a = int(raw_input()) - b = int(raw_input()) + a = input() + b = input() print a-b """ ) - - get_class = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertIn("Incorrect Answer", result.get('error')) def test_file_based_answer(self): + # Given self.test_case_data = [{"expected_input": "", "expected_output": "2"}] self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] @@ -346,12 +487,16 @@ class PythonStdIOEvaluator(unittest.TestCase): print a[0] """ ) - get_class = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } - result = get_class.evaluate(**kwargs) + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 04a4e69..1b66fd2 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -47,8 +47,8 @@ class PythonAssertionEvaluator(CodeEvaluator): fname, lineno, func, text = info[-1] text = str(test_case).splitlines()[lineno-1] err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - except TimeoutException: - raise + except Exception: + raise # Exception will be caught in CodeEvaluator. else: success = True err = 'Correct answer' |