From 1ddc29e1cb703557ab43d14318c279fce8fc609e Mon Sep 17 00:00:00 2001 From: adityacp Date: Fri, 19 Feb 2016 16:19:14 +0530 Subject: Frontend design changes remove older change Remove further changes interface changes interface changes small views.py change change in code evaluator code evaluator changes python code evaluator change exception changes in code evaluator and test cases for python Update test_python_evaluation.py changed variable name in test case Views.py conflict resolve --- yaksh/evaluator_tests/test_python_evaluation.py | 112 ++++++++++++++++++------ 1 file changed, 85 insertions(+), 27 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 0478353..8a3ba0d 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -7,47 +7,105 @@ class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "Python" self.test = None - self.test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} + self.test_case_data = [{"func_name": "add", + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} }] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "def add(a, b):\n\treturn a + b""" - get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = get_class.evaluate() + user_answer = "def add(a, b):\n\treturn a + b" + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): user_answer = "def add(a, b):\n\treturn a - b" - test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} - }] - get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = get_class.evaluate() + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") def test_infinite_loop(self): - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass""" - test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), timeout_msg) + + def test_syntax_error(self): + user_answer = "def add(a, b);\n\treturn a+b" + syntax_error_msg = ('Traceback (most recent call last):\n File ' + '"", line 1\n def add(a, b);\n ' + ' ^\nSyntaxError: invalid syntax\n') + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), syntax_error_msg) + + def test_indent_error(self): + user_answer = "def add(a, b):\nreturn a+b" + indent_error_msg = ('Traceback (most recent call last):\n ' + 'File "", line 2\n ' + 'return a+b\n ^\nIndentationError: ' + 'expected an indented block\n') + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), indent_error_msg) + + def test_name_error(self): + user_answer = "" + name_error_msg = ("Traceback (most recent call last):\nNameError: " + "name 'add' is not defined\n") + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), name_error_msg) + + def test_recursion_error(self): + user_answer = ("def fact(a):\n\tif a == 0:\n\t\treturn fact(1)" + "\n\telse:\n\t\treturn a * fact(a-1)") + self.test_case_data = [{"func_name": "fact", + "expected_answer": "24", + "test_id": u'null', + "pos_args": ["4"], + "kw_args": {} + }] + recursion_error_msg = ('Traceback (most recent call last):\nRuntimeError: ' + 'maximum recursion depth exceeded\n') + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), recursion_error_msg) + + def test_type_error(self): + user_answer = "def add(a):\n\treturn a+b" + type_error_msg = ("Traceback (most recent call last):\nTypeError: " + "add() takes exactly 1 argument (2 given)\n") + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), type_error_msg) + + def test_value_error(self): + user_answer = "def split(line):\n\t[ key, value ] = line.split()" + value_error_msg = ("Traceback (most recent call last):\nValueError: " + "need more than 1 value to unpack\n") + self.test_case_data = [{"func_name": "split", + "expected_answer": "Hello", + "test_id": u'null', + "pos_args": ["'Hello'"], + "kw_args": {} }] - get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = get_class.evaluate() + evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), self.timeout_msg) + self.assertEquals(result.get("error"), value_error_msg) if __name__ == '__main__': unittest.main() -- cgit From 5e7a49b729010b8c5c7cb12ea725dc3538f27a79 Mon Sep 17 00:00:00 2001 From: adityacp Date: Thu, 10 Mar 2016 14:19:53 +0530 Subject: python test case changes --- yaksh/evaluator_tests/test_python_evaluation.py | 101 +++++++++++++++--------- 1 file changed, 65 insertions(+), 36 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 8a3ba0d..ec46b5e 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -2,6 +2,7 @@ import unittest import os from yaksh.python_code_evaluator import PythonCodeEvaluator from yaksh.settings import SERVER_TIMEOUT +import textwrap class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -15,21 +16,31 @@ class PythonEvaluationTestCases(unittest.TestCase): }] def test_correct_answer(self): - user_answer = "def add(a, b):\n\treturn a + b" + user_answer = textwrap.dedent(""" + def add(a, b): + return a + b + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): - user_answer = "def add(a, b):\n\treturn a - b" + user_answer = textwrap.dedent(""" + def add(a, b): + return a - b + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") def test_infinite_loop(self): - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + user_answer = textwrap.dedent(""" + def add(a, b): + while True: + pass + """) timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) @@ -38,21 +49,34 @@ class PythonEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get("error"), timeout_msg) def test_syntax_error(self): - user_answer = "def add(a, b);\n\treturn a+b" - syntax_error_msg = ('Traceback (most recent call last):\n File ' - '"", line 1\n def add(a, b);\n ' - ' ^\nSyntaxError: invalid syntax\n') + # Do not touch any error messages as it can fail test case + user_answer = textwrap.dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + File "", line 2 + def add(a, b); + ^ + SyntaxError: invalid syntax\n""") evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), syntax_error_msg) def test_indent_error(self): - user_answer = "def add(a, b):\nreturn a+b" - indent_error_msg = ('Traceback (most recent call last):\n ' - 'File "", line 2\n ' - 'return a+b\n ^\nIndentationError: ' - 'expected an indented block\n') + user_answer = textwrap.dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + File "", line 3 + return a + b + ^ + IndentationError: expected an indented block + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) @@ -60,48 +84,53 @@ class PythonEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" - name_error_msg = ("Traceback (most recent call last):\nNameError: " - "name 'add' is not defined\n") + name_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + NameError: name 'add' is not defined + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), name_error_msg) def test_recursion_error(self): - user_answer = ("def fact(a):\n\tif a == 0:\n\t\treturn fact(1)" - "\n\telse:\n\t\treturn a * fact(a-1)") - self.test_case_data = [{"func_name": "fact", - "expected_answer": "24", - "test_id": u'null', - "pos_args": ["4"], - "kw_args": {} - }] - recursion_error_msg = ('Traceback (most recent call last):\nRuntimeError: ' - 'maximum recursion depth exceeded\n') + user_answer = textwrap.dedent(""" + def add(a, b): + return add(3, 3) + """) + recursion_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + RuntimeError: maximum recursion depth exceeded + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), recursion_error_msg) def test_type_error(self): - user_answer = "def add(a):\n\treturn a+b" - type_error_msg = ("Traceback (most recent call last):\nTypeError: " - "add() takes exactly 1 argument (2 given)\n") + user_answer = textwrap.dedent(""" + def add(a): + return a + b + """) + type_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + TypeError: add() takes exactly 1 argument (2 given) + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), type_error_msg) def test_value_error(self): - user_answer = "def split(line):\n\t[ key, value ] = line.split()" - value_error_msg = ("Traceback (most recent call last):\nValueError: " - "need more than 1 value to unpack\n") - self.test_case_data = [{"func_name": "split", - "expected_answer": "Hello", - "test_id": u'null', - "pos_args": ["'Hello'"], - "kw_args": {} - }] + user_answer = textwrap.dedent(""" + def add(a, b): + c = 'a' + return int(a) + int(b) + int(c) + """) + value_error_msg = textwrap.dedent("""\ + Traceback (most recent call last): + ValueError: invalid literal for int() with base 10: 'a' + """) evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = evaluator.evaluate() self.assertFalse(result.get("success")) -- cgit From 994c2a2c175d70bac1a01f582a4ad9c37f569690 Mon Sep 17 00:00:00 2001 From: adityacp Date: Fri, 11 Mar 2016 16:06:30 +0530 Subject: changes in python test cases --- yaksh/evaluator_tests/test_python_evaluation.py | 162 ++++++++++-------------- 1 file changed, 69 insertions(+), 93 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index ec46b5e..b71154a 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -2,139 +2,115 @@ import unittest import os from yaksh.python_code_evaluator import PythonCodeEvaluator from yaksh.settings import SERVER_TIMEOUT -import textwrap +from textwrap import dedent class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "Python" self.test = None self.test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} }] def test_correct_answer(self): - user_answer = textwrap.dedent(""" - def add(a, b): - return a + b - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b): + return a + b + """) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): - user_answer = textwrap.dedent(""" - def add(a, b): - return a - b - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b): + return a - b + """) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") def test_infinite_loop(self): - user_answer = textwrap.dedent(""" - def add(a, b): - while True: - pass - """) + user_answer = dedent(""" + def add(a, b): + while True: + pass + """) timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), timeout_msg) def test_syntax_error(self): - # Do not touch any error messages as it can fail test case - user_answer = textwrap.dedent(""" - def add(a, b); - return a + b - """) - syntax_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - File "", line 2 - def add(a, b); - ^ - SyntaxError: invalid syntax\n""") - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b); + return a + b + """) + syntax_error_msg = "SyntaxError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), syntax_error_msg) + self.assertIn(syntax_error_msg, result.get("error")) def test_indent_error(self): - user_answer = textwrap.dedent(""" - def add(a, b): - return a + b - """) - indent_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - File "", line 3 - return a + b - ^ - IndentationError: expected an indented block - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b): + return a + b + """) + indent_error_msg = "IndentationError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), indent_error_msg) + self.assertIn(indent_error_msg, result.get("error")) def test_name_error(self): user_answer = "" - name_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - NameError: name 'add' is not defined - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + name_error_msg = "NameError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), name_error_msg) + self.assertIn(name_error_msg, result.get("error")) def test_recursion_error(self): - user_answer = textwrap.dedent(""" - def add(a, b): - return add(3, 3) - """) - recursion_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - RuntimeError: maximum recursion depth exceeded - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b): + return add(3, 3) + """) + recursion_error_msg = "RuntimeError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), recursion_error_msg) + self.assertIn(recursion_error_msg, result.get("error")) def test_type_error(self): - user_answer = textwrap.dedent(""" - def add(a): - return a + b - """) - type_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - TypeError: add() takes exactly 1 argument (2 given) - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a): + return a + b + """) + type_error_msg = "TypeError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), type_error_msg) + self.assertIn(type_error_msg, result.get("error")) def test_value_error(self): - user_answer = textwrap.dedent(""" - def add(a, b): - c = 'a' - return int(a) + int(b) + int(c) - """) - value_error_msg = textwrap.dedent("""\ - Traceback (most recent call last): - ValueError: invalid literal for int() with base 10: 'a' - """) - evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) - result = evaluator.evaluate() + user_answer = dedent(""" + def add(a, b): + c = 'a' + return int(a) + int(b) + int(c) + """) + value_error_msg = "ValueError" + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_evaluator.evaluate() self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), value_error_msg) + self.assertIn(value_error_msg, result.get("error")) if __name__ == '__main__': unittest.main() -- cgit From d0c8e1467491f389f5a639ed32e81f4b3148c7d5 Mon Sep 17 00:00:00 2001 From: adityacp Date: Mon, 21 Mar 2016 15:15:51 +0530 Subject: python test cases changes --- yaksh/evaluator_tests/test_python_evaluation.py | 36 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index b71154a..7cd4352 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -4,15 +4,16 @@ from yaksh.python_code_evaluator import PythonCodeEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent + class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "Python" self.test = None self.test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} }] def test_correct_answer(self): @@ -20,7 +21,8 @@ class PythonEvaluationTestCases(unittest.TestCase): def add(a, b): return a + b """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") @@ -30,7 +32,8 @@ class PythonEvaluationTestCases(unittest.TestCase): def add(a, b): return a - b """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") @@ -43,7 +46,8 @@ class PythonEvaluationTestCases(unittest.TestCase): """) timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), timeout_msg) @@ -54,7 +58,8 @@ class PythonEvaluationTestCases(unittest.TestCase): return a + b """) syntax_error_msg = "SyntaxError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(syntax_error_msg, result.get("error")) @@ -65,7 +70,8 @@ class PythonEvaluationTestCases(unittest.TestCase): return a + b """) indent_error_msg = "IndentationError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(indent_error_msg, result.get("error")) @@ -73,7 +79,8 @@ class PythonEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" name_error_msg = "NameError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(name_error_msg, result.get("error")) @@ -84,7 +91,8 @@ class PythonEvaluationTestCases(unittest.TestCase): return add(3, 3) """) recursion_error_msg = "RuntimeError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(recursion_error_msg, result.get("error")) @@ -95,7 +103,8 @@ class PythonEvaluationTestCases(unittest.TestCase): return a + b """) type_error_msg = "TypeError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(type_error_msg, result.get("error")) @@ -107,7 +116,8 @@ class PythonEvaluationTestCases(unittest.TestCase): return int(a) + int(b) + int(c) """) value_error_msg = "ValueError" - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, + self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertIn(value_error_msg, result.get("error")) -- cgit From 17a1eacda10beea98b4bbddf3d3dd06378a74eee Mon Sep 17 00:00:00 2001 From: adityacp Date: Fri, 1 Apr 2016 15:09:01 +0530 Subject: python error based test case changes --- yaksh/evaluator_tests/test_python_evaluation.py | 47 ++++++++++++++++++------- 1 file changed, 34 insertions(+), 13 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 7cd4352..c55f04f 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -57,57 +57,75 @@ class PythonEvaluationTestCases(unittest.TestCase): def add(a, b); return a + b """) - syntax_error_msg = "SyntaxError" + syntax_error_msg = ["Traceback", "call", "File", "line", "", + "SyntaxError", "invalid syntax"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(syntax_error_msg, result.get("error")) + self.assertEqual(5, len(err)) + for msg in syntax_error_msg: + self.assertIn(msg, result.get("error")) def test_indent_error(self): user_answer = dedent(""" def add(a, b): return a + b """) - indent_error_msg = "IndentationError" + indent_error_msg = ["Traceback", "call", "File", "line", "", + "IndentationError", "indented block"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(indent_error_msg, result.get("error")) + self.assertEqual(5, len(err)) + for msg in indent_error_msg: + self.assertIn(msg, result.get("error")) def test_name_error(self): user_answer = "" - name_error_msg = "NameError" + name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(name_error_msg, result.get("error")) + self.assertEqual(2, len(err)) + for msg in name_error_msg: + self.assertIn(msg, result.get("error")) def test_recursion_error(self): user_answer = dedent(""" def add(a, b): return add(3, 3) """) - recursion_error_msg = "RuntimeError" + recursion_error_msg = ["Traceback", "call", "RuntimeError", + "maximum recursion depth exceeded"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(recursion_error_msg, result.get("error")) + self.assertEqual(2, len(err)) + for msg in recursion_error_msg: + self.assertIn(msg, result.get("error")) def test_type_error(self): user_answer = dedent(""" def add(a): return a + b """) - type_error_msg = "TypeError" + type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(type_error_msg, result.get("error")) + self.assertEqual(2, len(err)) + for msg in type_error_msg: + self.assertIn(msg, result.get("error")) def test_value_error(self): user_answer = dedent(""" @@ -115,12 +133,15 @@ class PythonEvaluationTestCases(unittest.TestCase): c = 'a' return int(a) + int(b) + int(c) """) - value_error_msg = "ValueError" + value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() + err = result.get("error").splitlines() self.assertFalse(result.get("success")) - self.assertIn(value_error_msg, result.get("error")) + self.assertEqual(2, len(err)) + for msg in value_error_msg: + self.assertIn(msg, result.get("error")) if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file -- cgit