summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py16
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py43
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py29
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py87
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py13
5 files changed, 86 insertions, 102 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 5022c1d..4b551d7 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -6,11 +6,12 @@ import tempfile
from yaksh.grader import Grader
from yaksh.bash_code_evaluator import BashCodeEvaluator
from yaksh.bash_stdio_evaluator import BashStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
-class BashAssertionEvaluationTestCases(unittest.TestCase):
+class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
with open(self.f_path, 'wb') as f:
@@ -58,7 +59,6 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
def test_error(self):
# Given
@@ -80,7 +80,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Error" in result.get("error"))
+ self.assert_correct_output("Error", result.get("error"))
def test_infinite_loop(self):
# Given
@@ -102,7 +102,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
# Given
@@ -136,9 +136,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
-class BashStdIOEvaluationTestCases(unittest.TestCase):
+class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.in_dir = tempfile.mkdtemp()
self.timeout_msg = ("Code took more than {0} seconds to run. "
@@ -175,7 +174,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -208,7 +206,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -239,7 +236,7 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertFalse(result.get('success'))
def test_stdout_only(self):
@@ -270,7 +267,6 @@ class BashStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index ac56142..d734cf2 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -9,11 +9,11 @@ from textwrap import dedent
from yaksh.grader import Grader
from yaksh.cpp_code_evaluator import CppCodeEvaluator
from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
from yaksh.settings import SERVER_TIMEOUT
-
-class CAssertionEvaluationTestCases(unittest.TestCase):
+class CAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
with open(self.f_path, 'wb') as f:
@@ -86,7 +86,6 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
def test_incorrect_answer(self):
# Given
@@ -105,11 +104,10 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
-
# Then
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect:", result.get('error'))
+ self.assert_correct_output("Incorrect:", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_compilation_error(self):
@@ -131,7 +129,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_infinite_loop(self):
# Given
@@ -152,7 +150,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
# Given
@@ -216,10 +214,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer\n")
-class CppStdIOEvaluationTestCases(unittest.TestCase):
+class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
@@ -259,7 +256,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -293,7 +289,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -325,7 +320,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -350,11 +344,10 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
-
# Then
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -381,7 +374,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_infinite_loop(self):
# Given
@@ -407,7 +400,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_only_stdout(self):
# Given
@@ -437,7 +430,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_correct_answer(self):
@@ -465,7 +457,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_array_input(self):
@@ -500,7 +491,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_string_input(self):
@@ -533,7 +523,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_cpp_incorrect_answer(self):
@@ -559,11 +548,10 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
-
# Then
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_cpp_error(self):
@@ -591,7 +579,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assert_correct_output("Compilation Error", result.get("error"))
def test_cpp_infinite_loop(self):
# Given
@@ -618,7 +606,7 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_cpp_only_stdout(self):
# Given
@@ -649,7 +637,6 @@ class CppStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index 58d5432..b53d8aa 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -10,9 +10,10 @@ from yaksh import grader as gd
from yaksh.grader import Grader
from yaksh.java_code_evaluator import JavaCodeEvaluator
from yaksh.java_stdio_evaluator import JavaStdIOEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
-class JavaAssertionEvaluationTestCases(unittest.TestCase):
+class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
with open(self.f_path, 'wb') as f:
@@ -90,7 +91,6 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -112,9 +112,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -136,7 +136,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Error" in result.get("error"))
+ self.assert_correct_output("Error", result.get("error"))
def test_infinite_loop(self):
# Given
@@ -157,7 +157,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_file_based_assert(self):
# Given
@@ -230,9 +230,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
-class JavaStdIOEvaluationTestCases(unittest.TestCase):
+class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
with open(self.f_path, 'wb') as f:
@@ -281,7 +280,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -316,7 +314,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -344,11 +341,10 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
-
# Then
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect", result.get('error'))
+ self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -374,7 +370,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in result.get("error"))
+ self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
def test_infinite_loop(self):
# Given
@@ -401,7 +397,7 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_only_stdout(self):
# Given
@@ -432,7 +428,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -466,7 +461,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_file_based_stdout(self):
@@ -511,7 +505,6 @@ class JavaStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer\n")
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index e638049..c58d7f1 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -11,7 +11,14 @@ from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
from yaksh.python_stdio_evaluator import PythonStdIOEvaluator
from yaksh.settings import SERVER_TIMEOUT
-class PythonAssertionEvaluationTestCases(unittest.TestCase):
+
+class EvaluatorBaseTest(unittest.TestCase):
+ def assert_correct_output(self, expected_output, actual_output):
+ actual_output_as_string = ''.join(actual_output)
+ self.assertIn(expected_output, actual_output_as_string)
+
+
+class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -49,7 +56,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_incorrect_answer(self):
# Given
@@ -70,13 +76,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertIn('AssertionError in: assert(add(1,2)==3)',
+ self.assert_correct_output('AssertionError in: assert(add(1,2)==3)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error')
)
@@ -104,10 +110,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('weight'), 2.0)
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,2)==1)',
result.get('error')
)
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
+ self.assert_correct_output('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error')
)
@@ -130,7 +136,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
def test_syntax_error(self):
# Given
@@ -159,13 +165,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_indent_error(self):
# Given
@@ -193,13 +200,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ err = result.get("error")[0].splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_name_error(self):
# Given
@@ -224,13 +231,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_recursion_error(self):
# Given
@@ -256,12 +264,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
for msg in recursion_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_type_error(self):
# Given
@@ -288,13 +297,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in type_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_value_error(self):
# Given
@@ -323,13 +333,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(9, len(err))
+ self.assertEqual(6, len(err))
for msg in value_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_file_based_assert(self):
# Given
@@ -356,7 +367,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertIn("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_single_testcase_error(self):
@@ -378,8 +388,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
"invalid syntax"
]
- kwargs = {
- 'metadata': {
+ kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
@@ -391,13 +400,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
-
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
+
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(6, len(err))
+ self.assertEqual(5, len(err))
for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
def test_multiple_testcase_error(self):
@@ -432,15 +442,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- err = result.get("error").splitlines()
+ error_as_str = ''.join(result.get("error"))
+ err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(7, len(err))
+ self.assertEqual(2, len(err))
for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
+ self.assert_correct_output(msg, result.get("error"))
-class PythonStdIOEvaluationTestCases(unittest.TestCase):
+class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
@@ -476,7 +487,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_list(self):
# Given
@@ -510,7 +520,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_string(self):
# Given
@@ -542,7 +551,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
def test_incorrect_answer_integer(self):
# Given
@@ -572,7 +580,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect answer", result.get('error'))
+ self.assert_correct_output("Incorrect answer", result.get('error'))
def test_file_based_answer(self):
# Given
@@ -603,7 +611,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
@@ -632,7 +639,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
# Then
- self.assertEqual(result.get('error'), timeout_msg)
+ self.assert_correct_output(timeout_msg, result.get('error'))
self.assertFalse(result.get('success'))
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index c6b3b54..5a452a3 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -7,8 +7,10 @@ from textwrap import dedent
from yaksh import grader as gd
from yaksh.grader import Grader
from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
+from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest
-class ScilabEvaluationTestCases(unittest.TestCase):
+
+class ScilabEvaluationTestCases(EvaluatorBaseTest):
def setUp(self):
tmp_in_dir_path = tempfile.mkdtemp()
self.tc_data = dedent("""
@@ -73,7 +75,6 @@ class ScilabEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
def test_error(self):
@@ -93,7 +94,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue('error' in result.get("error"))
+ self.assert_correct_output('error', result.get("error"))
def test_incorrect_answer(self):
@@ -112,9 +113,9 @@ class ScilabEvaluationTestCases(unittest.TestCase):
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
- lines_of_error = len(result.get('error').splitlines())
+ lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assertIn("Message", result.get('error'))
+ self.assert_correct_output("Message", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_infinite_loop(self):
@@ -134,7 +135,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
- self.assertEqual(result.get("error"), self.timeout_msg)
+ self.assert_correct_output(self.timeout_msg, result.get("error"))
if __name__ == '__main__':
unittest.main()
generated by cgit (git 2.34.1) at 2024-12-22 07:28:38 +0000