Fix conflict

1 files changed, 47 insertions, 40 deletions

diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index e638049..c58d7f1 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -11,7 +11,14 @@ from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
 from yaksh.python_stdio_evaluator import PythonStdIOEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 
-class PythonAssertionEvaluationTestCases(unittest.TestCase):
+
+class EvaluatorBaseTest(unittest.TestCase):
+    def assert_correct_output(self, expected_output, actual_output):
+        actual_output_as_string = ''.join(actual_output)
+        self.assertIn(expected_output, actual_output_as_string)
+
+
+class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
     def setUp(self):
         with open('/tmp/test.txt', 'wb') as f:
             f.write('2'.encode('ascii'))
@@ -49,7 +56,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertTrue(result.get('success'))
-        self.assertIn("Correct answer", result.get('error'))
 
     def test_incorrect_answer(self):
         # Given
@@ -70,13 +76,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assertIn('AssertionError  in: assert(add(1,2)==3)',
+        self.assert_correct_output('AssertionError  in: assert(add(1,2)==3)',
                         result.get('error')
                       )
-        self.assertIn('AssertionError  in: assert(add(-1,2)==1)',
+        self.assert_correct_output('AssertionError  in: assert(add(-1,2)==1)',
                         result.get('error')
                       )
-        self.assertIn('AssertionError  in: assert(add(-1,-2)==-3)',
+        self.assert_correct_output('AssertionError  in: assert(add(-1,-2)==-3)',
                         result.get('error')
                       )
 
@@ -104,10 +110,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # Then
         self.assertFalse(result.get('success'))
         self.assertEqual(result.get('weight'), 2.0)
-        self.assertIn('AssertionError  in: assert(add(-1,2)==1)',
+        self.assert_correct_output('AssertionError  in: assert(add(-1,2)==1)',
                         result.get('error')
                       )
-        self.assertIn('AssertionError  in: assert(add(-1,-2)==-3)',
+        self.assert_correct_output('AssertionError  in: assert(add(-1,-2)==-3)',
                         result.get('error')
                       )
 
@@ -130,7 +136,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assertEqual(result.get('error'), self.timeout_msg)
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
 
     def test_syntax_error(self):
         # Given
@@ -159,13 +165,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
         for msg in syntax_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_indent_error(self):
         # Given
@@ -193,13 +200,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        err = result.get("error")[0].splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
         for msg in indent_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_name_error(self):
         # Given
@@ -224,13 +231,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(6, len(err))
         for msg in name_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_recursion_error(self):
         # Given
@@ -256,12 +264,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
         for msg in recursion_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_type_error(self):
         # Given
@@ -288,13 +297,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(6, len(err))
         for msg in type_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_value_error(self):
         # Given
@@ -323,13 +333,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(6, len(err))
         for msg in value_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
     def test_file_based_assert(self):
         # Given
@@ -356,7 +367,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         result = grader.evaluate(kwargs)
 
         # Then
-        self.assertIn("Correct answer", result.get('error'))
         self.assertTrue(result.get('success'))
 
     def test_single_testcase_error(self):
@@ -378,8 +388,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                             "invalid syntax"
                             ]
 
-        kwargs = {
-                  'metadata': {
+        kwargs = {'metadata': {
                     'user_answer': user_answer,
                     'file_paths': self.file_paths,
                     'partial_grading': False,
@@ -391,13 +400,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
-
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
+        
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(6, len(err))
+        self.assertEqual(5, len(err))
         for msg in syntax_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
 
     def test_multiple_testcase_error(self):
@@ -432,15 +442,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error").splitlines()
+        error_as_str = ''.join(result.get("error"))
+        err = error_as_str.splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(7, len(err))
+        self.assertEqual(2, len(err))
         for msg in name_error_msg:
-            self.assertIn(msg, result.get("error"))
+            self.assert_correct_output(msg, result.get("error"))
 
-class PythonStdIOEvaluationTestCases(unittest.TestCase):
+class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
     def setUp(self):
         with open('/tmp/test.txt', 'wb') as f:
             f.write('2'.encode('ascii'))
@@ -476,7 +487,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertTrue(result.get('success'))
-        self.assertIn("Correct answer", result.get('error'))
 
     def test_correct_answer_list(self):
         # Given
@@ -510,7 +520,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertTrue(result.get('success'))
-        self.assertIn("Correct answer", result.get('error'))
 
     def test_correct_answer_string(self):
         # Given
@@ -542,7 +551,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertTrue(result.get('success'))
-        self.assertIn("Correct answer", result.get('error'))
 
     def test_incorrect_answer_integer(self):
         # Given
@@ -572,7 +580,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assertIn("Incorrect answer", result.get('error'))
+        self.assert_correct_output("Incorrect answer", result.get('error'))
 
     def test_file_based_answer(self):
         # Given
@@ -603,7 +611,6 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
         result = grader.evaluate(kwargs)
 
         # Then
-        self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_infinite_loop(self):
@@ -632,7 +639,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
         result = grader.evaluate(kwargs)
 
         # Then
-        self.assertEqual(result.get('error'), timeout_msg)
+        self.assert_correct_output(timeout_msg, result.get('error'))
         self.assertFalse(result.get('success'))