summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py16
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py40
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py34
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py90
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py13
-rw-r--r--yaksh/python_assertion_evaluator.py2
6 files changed, 96 insertions, 99 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 06a56e4..abadf26 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -6,11 +6,12 @@ import tempfile
from yaksh.grader import Grader
from yaksh.bash_code_evaluator import BashCodeEvaluator
from yaksh.bash_stdio_evaluator import BashStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
-class BashAssertionEvaluationTestCases(unittest.TestCase):
+class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -48,7 +49,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
def test_error(self):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] "
@@ -67,7 +67,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Error" in result.get("error"))
+ self.assert_correct_output("Error", result.get("error"))
def test_infinite_loop(self):
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
@@ -86,7 +86,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
self.file_paths = [('/tmp/test.txt', False)]
@@ -111,9 +111,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
-class BashStdIOEvaluationTestCases(unittest.TestCase):
+class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.in_dir = tempfile.mkdtemp()
self.timeout_msg = ("Code took more than {0} seconds to run. "
@@ -147,7 +146,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -177,7 +175,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -204,7 +201,7 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertFalse(result.get('success'))
def test_stdout_only(self):
@@ -231,7 +228,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index dc6fdc9..ec59a6b 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -9,11 +9,11 @@ from textwrap import dedent
from yaksh.grader import Grader
from yaksh.cpp_code_evaluator import CppCodeEvaluator
from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
from yaksh.settings import SERVER_TIMEOUT
-
-class CAssertionEvaluationTestCases(unittest.TestCase):
+class CAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -48,7 +48,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
def test_incorrect_answer(self):
user_answer = "int add(int a, int b)\n{return a-b;}"
@@ -65,9 +64,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect:", result.get('error'))
+ self.assert_correct_output("Incorrect:", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_compilation_error(self):
@@ -86,7 +85,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_infinite_loop(self):
user_answer = "int add(int a, int b)\n{while(1>0){}}"
@@ -104,7 +103,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
self.file_paths = [('/tmp/test.txt', False)]
@@ -138,9 +137,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
-class CppStdIOEvaluationTestCases(unittest.TestCase):
+class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
@@ -174,7 +172,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -205,7 +202,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -234,7 +230,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -257,9 +252,9 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -283,7 +278,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_infinite_loop(self):
user_answer = dedent("""
@@ -306,7 +301,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -333,7 +328,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_correct_answer(self):
@@ -358,7 +352,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_array_input(self):
@@ -390,7 +383,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_string_input(self):
@@ -420,7 +412,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_incorrect_answer(self):
@@ -444,9 +435,9 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_cpp_error(self):
@@ -471,7 +462,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_cpp_infinite_loop(self):
user_answer = dedent("""
@@ -495,7 +486,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_cpp_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -523,7 +514,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index 36eb6a5..503372e 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -10,9 +10,10 @@ from yaksh import grader as gd
from yaksh.grader import Grader
from yaksh.java_code_evaluator import JavaCodeEvaluator
from yaksh.java_stdio_evaluator import JavaStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
-class JavaAssertionEvaluationTestCases(unittest.TestCase):
+class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -51,7 +52,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -70,9 +71,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get('success'))
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -91,7 +92,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Error" in result.get("error"))
+ self.assert_correct_output("Error", result.get("error"))
def test_infinite_loop(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
@@ -109,7 +110,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
self.file_paths = [("/tmp/test.txt", False)]
@@ -151,9 +152,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get("error"))
-class JavaStdIOEvaluationTestCases(unittest.TestCase):
+class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -198,7 +199,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -230,7 +231,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -256,9 +256,9 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -281,7 +281,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
def test_infinite_loop(self):
user_answer = dedent("""
@@ -305,7 +305,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -333,7 +333,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -364,7 +364,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_file_based_stdout(self):
@@ -406,7 +406,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get("error"))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index e638049..291e639 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -11,7 +11,14 @@ from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
from yaksh.python_stdio_evaluator import PythonStdIOEvaluator
from yaksh.settings import SERVER_TIMEOUT
-class PythonAssertionEvaluationTestCases(unittest.TestCase):
+
+class EvaluatorBaseTest(unittest.TestCase):
+ def assert_correct_output(self, expected_output, actual_output):
+ actual_output_as_string = ''.join(actual_output)
+ self.assertIn(expected_output, actual_output_as_string)
+
+
+class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -49,7 +56,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
+ self.assert_correct_output("Correct answer", result.get('error'))
def test_incorrect_answer(self):
# Given
@@ -70,13 +77,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertIn('AssertionError in: assert(add(1,2)==3)',
+ self.assert_correct_output('AssertionError in: assert(add(1,2)==3)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error')
)
@@ -104,10 +111,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('weight'), 2.0)
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error')
)
@@ -130,7 +137,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
def test_syntax_error(self):
# Given
@@ -159,13 +166,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_indent_error(self):
# Given
@@ -193,13 +201,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ err = result.get("error")[0].splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_name_error(self):
# Given
@@ -224,13 +232,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_recursion_error(self):
# Given
@@ -256,12 +265,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
for msg in recursion_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_type_error(self):
# Given
@@ -288,13 +298,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in type_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_value_error(self):
# Given
@@ -323,13 +334,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in value_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_file_based_assert(self):
# Given
@@ -356,7 +368,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertIn("Correct answer", result.get('error'))
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_single_testcase_error(self):
@@ -378,8 +390,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
"invalid syntax"
]
- kwargs = {
- 'metadata': {
+ kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
@@ -391,13 +402,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
-
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
+
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(6, len(err))
+ self.assertEqual(5, len(err))
for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_multiple_testcase_error(self):
@@ -432,15 +444,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(7, len(err))
+ self.assertEqual(5, len(err))
for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
-class PythonStdIOEvaluationTestCases(unittest.TestCase):
+class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -476,7 +489,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_list(self):
# Given
@@ -510,7 +522,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_string(self):
# Given
@@ -542,7 +553,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_incorrect_answer_integer(self):
# Given
@@ -572,7 +582,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect answer", result.get('error'))
+ self.assert_correct_output("Incorrect answer", result.get('error'))
def test_file_based_answer(self):
# Given
@@ -603,7 +613,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
+ self.assert_correct_output("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
@@ -632,7 +642,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), timeout_msg)
+ self.assert_correct_output(timeout_msg, result.get('error'))
self.assertFalse(result.get('success'))
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 0275ee8..938d0e5 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -7,8 +7,10 @@ import tempfile
from yaksh import grader as gd
from yaksh.grader import Grader
from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
-class ScilabEvaluationTestCases(unittest.TestCase):
+
+class ScilabEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [{"test_case": "scilab_files/test_add.sce",
@@ -42,7 +44,6 @@ class ScilabEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_error(self):
@@ -62,7 +63,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue('error' in result.get("error"))
+ self.assert_correct_output('error', result.get("error"))
def test_incorrect_answer(self):
@@ -81,9 +82,9 @@ class ScilabEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Message", result.get('error'))
+ self.assert_correct_output("Message", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_infinite_loop(self):
@@ -103,7 +104,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
index 4d44838..41ebeb6 100644
--- a/yaksh/python_assertion_evaluator.py
+++ b/yaksh/python_assertion_evaluator.py
@@ -75,7 +75,7 @@ class PythonAssertionEvaluator(BaseEvaluator):
type, value, tb = sys.exc_info()
info = traceback.extract_tb(tb)
fname, lineno, func, text = info[-1]
- text = str(self.test_case).splitlines()[lineno-1]
+ text = str(self.test_case)
err = ("-----\nExpected Test Case:\n{0}\n"
"Error - {1} {2} in: {3}\n-----").format(self.test_case, type.__name__, str(value), text)
except TimeoutException:
generated by cgit (git 2.34.1) at 2024-12-21 14:40:42 +0000