diff options
Diffstat (limited to 'yaksh/evaluator_tests')
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 101 |
1 files changed, 65 insertions, 36 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 8a3ba0d..ec46b5e 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -2,6 +2,7 @@ import unittest import os from yaksh.python_code_evaluator import PythonCodeEvaluator from yaksh.settings import SERVER_TIMEOUT +import textwrap class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -15,21 +16,31 @@ class PythonEvaluationTestCases(unittest.TestCase): }] def test_correct_answer(self): - user_answer = "def add(a, b):\n\treturn a + b" + user_answer = textwrap.dedent(""" + def add(a, b): + return a + b + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): - user_answer = "def add(a, b):\n\treturn a - b" + user_answer = textwrap.dedent(""" + def add(a, b): + return a - b + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") def test_infinite_loop(self): - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + user_answer = textwrap.dedent(""" + def add(a, b): + while True: + pass + """) timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) @@ -38,21 +49,34 @@ class PythonEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get("error"), timeout_msg) def test_syntax_error(self): - user_answer = "def add(a, b);\n\treturn a+b" - syntax_error_msg = ('Traceback (most recent call last):\n File ' - '"<string>", line 1\n def add(a, b);\n ' - ' ^\nSyntaxError: invalid syntax\n') + # Do not touch any error messages as it can fail test case + user_answer = textwrap.dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + File "<string>", line 2 + def add(a, b); + ^ + SyntaxError: invalid syntax\n""") evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), syntax_error_msg) def test_indent_error(self): - user_answer = "def add(a, b):\nreturn a+b" - indent_error_msg = ('Traceback (most recent call last):\n ' - 'File "<string>", line 2\n ' - 'return a+b\n ^\nIndentationError: ' - 'expected an indented block\n') + user_answer = textwrap.dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + File "<string>", line 3 + return a + b + ^ + IndentationError: expected an indented block + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) @@ -60,48 +84,53 @@ class PythonEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" - name_error_msg = ("Traceback (most recent call last):\nNameError: " - "name 'add' is not defined\n") + name_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + NameError: name 'add' is not defined + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), name_error_msg) def test_recursion_error(self): - user_answer = ("def fact(a):\n\tif a == 0:\n\t\treturn fact(1)" - "\n\telse:\n\t\treturn a * fact(a-1)") - self.test_case_data = [{"func_name": "fact", - "expected_answer": "24", - "test_id": u'null', - "pos_args": ["4"], - "kw_args": {} - }] - recursion_error_msg = ('Traceback (most recent call last):\nRuntimeError: ' - 'maximum recursion depth exceeded\n') + user_answer = textwrap.dedent(""" + def add(a, b): + return add(3, 3) + """) + recursion_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + RuntimeError: maximum recursion depth exceeded + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), recursion_error_msg) def test_type_error(self): - user_answer = "def add(a):\n\treturn a+b" - type_error_msg = ("Traceback (most recent call last):\nTypeError: " - "add() takes exactly 1 argument (2 given)\n") + user_answer = textwrap.dedent(""" + def add(a): + return a + b + """) + type_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + TypeError: add() takes exactly 1 argument (2 given) + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), type_error_msg) def test_value_error(self): - user_answer = "def split(line):\n\t[ key, value ] = line.split()" - value_error_msg = ("Traceback (most recent call last):\nValueError: " - "need more than 1 value to unpack\n") - self.test_case_data = [{"func_name": "split", - "expected_answer": "Hello", - "test_id": u'null', - "pos_args": ["'Hello'"], - "kw_args": {} - }] + user_answer = textwrap.dedent(""" + def add(a, b): + c = 'a' + return int(a) + int(b) + int(c) + """) + value_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + ValueError: invalid literal for int() with base 10: 'a' + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) |