summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/code_evaluator.py3
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py391
-rw-r--r--yaksh/python_assertion_evaluator.py4
3 files changed, 271 insertions, 127 deletions
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index e139b55..8a9b7a6 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -101,6 +101,7 @@ class CodeEvaluator(object):
# Do whatever testing needed.
try:
for test_case in test_case_data:
+ success = False
self.compile_code(user_answer, file_paths, **test_case)
success, err = self.check_code(user_answer, file_paths, **test_case)
if not success:
@@ -116,8 +117,6 @@ class CodeEvaluator(object):
tb_list = traceback.format_exception(exc_type, exc_value, exc_tb)
if len(tb_list) > 2:
del tb_list[1:3]
- else:
- pass
err = "Error: {0}".format("".join(tb_list))
finally:
# Set back any original signal handler.
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 3c07907..2a109d5 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -9,164 +9,205 @@ from textwrap import dedent
class PythonAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
- {"test_case": 'assert(add(-1,2)==1)'},
- {"test_case": 'assert(add(-1,-2)==-3)'},
- ]
+ {"test_case": 'assert(add(-1,2)==1)'},
+ {"test_case": 'assert(add(-1,-2)==-3)'},
+ ]
self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop in"
- " your code.").format(SERVER_TIMEOUT)
+ "You probably have an infinite loop in"
+ " your code.").format(SERVER_TIMEOUT)
self.file_paths = None
def test_correct_answer(self):
+ # Given
user_answer = "def add(a,b):\n\treturn a + b"
- get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer")
def test_incorrect_answer(self):
+ # Given
user_answer = "def add(a,b):\n\treturn a - b"
- get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'),
- "AssertionError in: assert(add(1,2)==3)"
- )
+ "AssertionError in: assert(add(1,2)==3)"
+ )
def test_infinite_loop(self):
+ # Given
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
- get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
-
+ def test_syntax_error(self):
+ # Given
user_answer = dedent("""
def add(a, b);
return a + b
""")
- syntax_error_msg = ["Traceback",
- "call",
- "File",
- "line",
- "<string>",
- "SyntaxError",
- "invalid syntax"
- ]
- get_class = PythonAssertionEvaluator()
+ syntax_error_msg = ["Traceback",
+ "call",
+ "File",
+ "line",
+ "<string>",
+ "SyntaxError",
+ "invalid syntax"
+ ]
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in syntax_error_msg:
self.assertIn(msg, result.get("error"))
def test_indent_error(self):
+ # Given
user_answer = dedent("""
def add(a, b):
return a + b
""")
- indent_error_msg = ["Traceback", "call",
- "File",
- "line",
- "<string>",
- "IndentationError",
- "indented block"
- ]
- get_class = PythonAssertionEvaluator()
+ indent_error_msg = ["Traceback", "call",
+ "File",
+ "line",
+ "<string>",
+ "IndentationError",
+ "indented block"
+ ]
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
self.assertIn(msg, result.get("error"))
def test_name_error(self):
+ # Given
user_answer = ""
- name_error_msg = ["Traceback",
- "call",
- "NameError",
- "name",
- "defined"
- ]
- get_class = PythonAssertionEvaluator()
+ name_error_msg = ["Traceback",
+ "call",
+ "NameError",
+ "name",
+ "defined"
+ ]
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(3, len(err))
for msg in name_error_msg:
self.assertIn(msg, result.get("error"))
def test_recursion_error(self):
+ # Given
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = ["Traceback",
- "call",
- "RuntimeError",
- "maximum recursion depth exceeded"
- ]
- get_class = PythonAssertionEvaluator()
+ "call",
+ "RuntimeError",
+ "maximum recursion depth exceeded"
+ ]
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(969, len(err))
for msg in recursion_error_msg:
self.assertIn(msg, result.get("error"))
def test_type_error(self):
+ # Given
user_answer = dedent("""
def add(a):
return a + b
""")
- type_error_msg = ["Traceback",
- "call",
- "TypeError",
- "exactly",
- "argument"
- ]
- get_class = PythonAssertionEvaluator()
+ type_error_msg = ["Traceback",
+ "call",
+ "TypeError",
+ "exactly",
+ "argument"
+ ]
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(3, len(err))
for msg in type_error_msg:
self.assertIn(msg, result.get("error"))
def test_value_error(self):
+ # Given
user_answer = dedent("""
def add(a, b):
c = 'a'
@@ -178,19 +219,24 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
"invalid literal",
"base"
]
- get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
+
+ # Then
self.assertFalse(result.get("success"))
self.assertEqual(4, len(err))
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
def test_file_based_assert(self):
+ # Given
self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
user_answer = dedent("""
@@ -198,15 +244,84 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
with open("test.txt") as f:
return f.read()[0]
""")
- get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
+ def test_single_testcase_error(self):
+ # Given
+ """ Tests the user answer with just an incorrect test case """
+
+ user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
+ test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'}
+ ]
+ syntax_error_msg = ["Traceback",
+ "call",
+ "File",
+ "line",
+ "<string>",
+ "SyntaxError",
+ "invalid syntax"
+ ]
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+ err = result.get("error").splitlines()
+
+ # Then
+ self.assertFalse(result.get("success"))
+ self.assertEqual(5, len(err))
+ for msg in syntax_error_msg:
+ self.assertIn(msg, result.get("error"))
+
+
+ def test_multiple_testcase_error(self):
+ """ Tests the user answer with an correct test case
+ first and then with an incorrect test case """
+ # Given
+ user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
+ test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'},
+ {"test_case": 's="abbb"\nassert palindrome(S)==False'}
+ ]
+ name_error_msg = ["Traceback",
+ "call",
+ "File",
+ "line",
+ "<string>",
+ "NameError",
+ "name 'S' is not defined"
+ ]
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': test_case_data,
+ 'file_paths': self.file_paths
+ }
+
+ # When
+ evaluator = PythonAssertionEvaluator()
+ result = evaluator.evaluate(**kwargs)
+ err = result.get("error").splitlines()
+
+ # Then
+ self.assertFalse(result.get("success"))
+ self.assertEqual(3, len(err))
+ for msg in name_error_msg:
+ self.assertIn(msg, result.get("error"))
+
+
class PythonStdoutEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{"expected_input": None,
@@ -216,86 +331,106 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop"
" in your code.").format(SERVER_TIMEOUT)
- self.file_paths = None
def test_correct_answer(self):
+ # Given
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
- get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertEqual(result.get('error'), "Correct Answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
+ # Given
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
- get_class = PythonStdioEvaluator()
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertFalse(result.get('success'))
self.assertIn("Incorrect Answer", result.get('error'))
def test_infinite_loop(self):
+ # Given
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
- get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
-
-class PythonStdIOEvaluator(unittest.TestCase):
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
+ self.assertEqual(result.get('error'), self.timeout_msg)
+ self.assertFalse(result.get('success'))
+
+
+class PythonStdIOEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop"
- " in your code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
- def test_add_two_integers_correct(self):
+ def test_correct_answer_integer(self):
+ # Given
self.test_case_data = [{"expected_input": "1\n2",
"expected_output": "3"
}]
user_answer = dedent("""
- a = int(raw_input())
- b = int(raw_input())
+ a = input()
+ b = input()
print a+b
"""
)
-
- get_class = PythonStdioEvaluator()
-
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertTrue(result.get('success'))
self.assertIn("Correct Answer", result.get('error'))
- def test_add_two_lists(self):
+ def test_correct_answer_list(self):
+ # Given
self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
"expected_output": "[1, 2, 3, 5, 6, 7]"
}]
user_answer = dedent("""
- from ast import literal_eval
- a = literal_eval(raw_input())
- b = literal_eval(raw_input())
+ a = input()
+ b = input()
print a+b
"""
)
-
- get_class = PythonStdioEvaluator()
-
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertTrue(result.get('success'))
self.assertIn("Correct Answer", result.get('error'))
- def test_find_string_in_sentence(self):
+ def test_correct_answer_string(self):
+ # Given
self.test_case_data = [{"expected_input": """the quick brown fox jumps\
over the lazy dog\nthe""",
"expected_output": "2"
@@ -306,37 +441,43 @@ class PythonStdIOEvaluator(unittest.TestCase):
print (a.count(b))
"""
)
-
- get_class = PythonStdioEvaluator()
-
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertTrue(result.get('success'))
self.assertIn("Correct Answer", result.get('error'))
- def test_add_two_integers_incorrect(self):
+ def test_incorrect_answer_integer(self):
+ # Given
self.test_case_data = [{"expected_input": "1\n2",
"expected_output": "3"
}]
user_answer = dedent("""
- a = int(raw_input())
- b = int(raw_input())
+ a = input()
+ b = input()
print a-b
"""
)
-
- get_class = PythonStdioEvaluator()
-
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertFalse(result.get('success'))
self.assertIn("Incorrect Answer", result.get('error'))
def test_file_based_answer(self):
+ # Given
self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
@@ -346,12 +487,16 @@ class PythonStdIOEvaluator(unittest.TestCase):
print a[0]
"""
)
- get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
- result = get_class.evaluate(**kwargs)
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
self.assertEqual(result.get('error'), "Correct Answer")
self.assertTrue(result.get('success'))
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
index 04a4e69..1b66fd2 100644
--- a/yaksh/python_assertion_evaluator.py
+++ b/yaksh/python_assertion_evaluator.py
@@ -47,8 +47,8 @@ class PythonAssertionEvaluator(CodeEvaluator):
fname, lineno, func, text = info[-1]
text = str(test_case).splitlines()[lineno-1]
err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
- except TimeoutException:
- raise
+ except Exception:
+ raise # Exception will be caught in CodeEvaluator.
else:
success = True
err = 'Correct answer'
generated by cgit (git 2.34.1) at 2024-12-21 05:09:06 +0000