diff options
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 315 |
1 files changed, 200 insertions, 115 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 1087c5d..2a109d5 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -9,164 +9,205 @@ from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, - ] + {"test_case": 'assert(add(-1,2)==1)'}, + {"test_case": 'assert(add(-1,-2)==-3)'}, + ] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) self.file_paths = None def test_correct_answer(self): + # Given user_answer = "def add(a,b):\n\treturn a + b" - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer") def test_incorrect_answer(self): + # Given user_answer = "def add(a,b):\n\treturn a - b" - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), - "AssertionError in: assert(add(1,2)==3)" - ) + "AssertionError in: assert(add(1,2)==3)" + ) def test_infinite_loop(self): + # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg) def test_syntax_error(self): + # Given user_answer = dedent(""" def add(a, b); return a + b """) - syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "<string>", - "SyntaxError", - "invalid syntax" - ] - evaluator = PythonAssertionEvaluator() + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error")) def test_indent_error(self): + # Given user_answer = dedent(""" def add(a, b): return a + b """) - indent_error_msg = ["Traceback", "call", - "File", - "line", - "<string>", - "IndentationError", - "indented block" - ] - evaluator = PythonAssertionEvaluator() + indent_error_msg = ["Traceback", "call", + "File", + "line", + "<string>", + "IndentationError", + "indented block" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assertIn(msg, result.get("error")) def test_name_error(self): + # Given user_answer = "" - name_error_msg = ["Traceback", - "call", - "NameError", - "name", - "defined" - ] - evaluator = PythonAssertionEvaluator() + name_error_msg = ["Traceback", + "call", + "NameError", + "name", + "defined" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error")) def test_recursion_error(self): + # Given user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = ["Traceback", - "call", - "RuntimeError", - "maximum recursion depth exceeded" - ] - evaluator = PythonAssertionEvaluator() + "call", + "RuntimeError", + "maximum recursion depth exceeded" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(969, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error")) def test_type_error(self): + # Given user_answer = dedent(""" def add(a): return a + b """) - type_error_msg = ["Traceback", - "call", - "TypeError", - "exactly", - "argument" - ] - evaluator = PythonAssertionEvaluator() + type_error_msg = ["Traceback", + "call", + "TypeError", + "exactly", + "argument" + ] kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error")) def test_value_error(self): + # Given user_answer = dedent(""" def add(a, b): c = 'a' @@ -178,19 +219,24 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid literal", "base" ] - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(4, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) def test_file_based_assert(self): + # Given self.test_case_data = [{"test_case": "assert(ans()=='2')"}] self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] user_answer = dedent(""" @@ -198,36 +244,45 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): with open("test.txt") as f: return f.read()[0] """) - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_single_testcase_error(self): + # Given """ Tests the user answer with just an incorrect test case """ - + user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'} ] syntax_error_msg = ["Traceback", - "call", - "File", - "line", - "<string>", - "SyntaxError", - "invalid syntax" - ] - evaluator = PythonAssertionEvaluator() + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: @@ -237,7 +292,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_multiple_testcase_error(self): """ Tests the user answer with an correct test case first and then with an incorrect test case """ - + # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'}, {"test_case": 's="abbb"\nassert palindrome(S)==False'} @@ -250,13 +305,17 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "NameError", "name 'S' is not defined" ] - evaluator = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() + + # Then self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in name_error_msg: @@ -272,90 +331,106 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) - self.file_paths = None def test_correct_answer(self): + # Given user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" - evaluator = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) def test_incorrect_answer(self): + # Given user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" - evaluator = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertIn("Incorrect Answer", result.get('error')) def test_infinite_loop(self): + # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)" - evaluator = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), self.timeout_msg) self.assertFalse(result.get('success')) -class PythonStdIOEvaluator(unittest.TestCase): +class PythonStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) + self.file_paths = None - def test_add_two_integers_correct(self): + def test_correct_answer_integer(self): + # Given self.test_case_data = [{"expected_input": "1\n2", "expected_output": "3" }] user_answer = dedent(""" - a = int(raw_input()) - b = int(raw_input()) + a = input() + b = input() print a+b """ ) - - evaluator = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_add_two_lists(self): + def test_correct_answer_list(self): + # Given self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]", "expected_output": "[1, 2, 3, 5, 6, 7]" }] user_answer = dedent(""" - from ast import literal_eval - a = literal_eval(raw_input()) - b = literal_eval(raw_input()) + a = input() + b = input() print a+b """ ) - - evaluator = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_find_string_in_sentence(self): + def test_correct_answer_string(self): + # Given self.test_case_data = [{"expected_input": """the quick brown fox jumps\ over the lazy dog\nthe""", "expected_output": "2" @@ -366,37 +441,43 @@ class PythonStdIOEvaluator(unittest.TestCase): print (a.count(b)) """ ) - - evaluator = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertTrue(result.get('success')) self.assertIn("Correct Answer", result.get('error')) - def test_add_two_integers_incorrect(self): + def test_incorrect_answer_integer(self): + # Given self.test_case_data = [{"expected_input": "1\n2", "expected_output": "3" }] user_answer = dedent(""" - a = int(raw_input()) - b = int(raw_input()) + a = input() + b = input() print a-b """ ) - - evaluator = PythonStdioEvaluator() - kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertFalse(result.get('success')) self.assertIn("Incorrect Answer", result.get('error')) def test_file_based_answer(self): + # Given self.test_case_data = [{"expected_input": "", "expected_output": "2"}] self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] @@ -406,12 +487,16 @@ class PythonStdIOEvaluator(unittest.TestCase): print a[0] """ ) - evaluator = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } + + # When + evaluator = PythonStdioEvaluator() result = evaluator.evaluate(**kwargs) + + # Then self.assertEqual(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) |