1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
import unittest
import os
from yaksh.python_code_evaluator import PythonCodeEvaluator
from yaksh.settings import SERVER_TIMEOUT
import textwrap
class PythonEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.language = "Python"
self.test = None
self.test_case_data = [{"func_name": "add",
"expected_answer": "5",
"test_id": u'null',
"pos_args": ["3", "2"],
"kw_args": {}
}]
def test_correct_answer(self):
user_answer = textwrap.dedent("""
def add(a, b):
return a + b
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertTrue(result.get("success"))
self.assertEqual(result.get("error"), "Correct answer")
def test_incorrect_answer(self):
user_answer = textwrap.dedent("""
def add(a, b):
return a - b
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5")
def test_infinite_loop(self):
user_answer = textwrap.dedent("""
def add(a, b):
while True:
pass
""")
timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), timeout_msg)
def test_syntax_error(self):
# Do not touch any error messages as it can fail test case
user_answer = textwrap.dedent("""
def add(a, b);
return a + b
""")
syntax_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
File "<string>", line 2
def add(a, b);
^
SyntaxError: invalid syntax\n""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), syntax_error_msg)
def test_indent_error(self):
user_answer = textwrap.dedent("""
def add(a, b):
return a + b
""")
indent_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
File "<string>", line 3
return a + b
^
IndentationError: expected an indented block
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), indent_error_msg)
def test_name_error(self):
user_answer = ""
name_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
NameError: name 'add' is not defined
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), name_error_msg)
def test_recursion_error(self):
user_answer = textwrap.dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
RuntimeError: maximum recursion depth exceeded
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), recursion_error_msg)
def test_type_error(self):
user_answer = textwrap.dedent("""
def add(a):
return a + b
""")
type_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
TypeError: add() takes exactly 1 argument (2 given)
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), type_error_msg)
def test_value_error(self):
user_answer = textwrap.dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = textwrap.dedent("""\
Traceback (most recent call last):
ValueError: invalid literal for int() with base 10: 'a'
""")
evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
result = evaluator.evaluate()
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), value_error_msg)
if __name__ == '__main__':
unittest.main()
|
