Modify testcases wrt changes in assertion error output

6 files changed, 108 insertions, 88 deletions

diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 2faa7bf..5542710 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -104,7 +104,9 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
@@ -533,7 +535,9 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assert_correct_output(self.timeout_msg, result.get('error'))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 0898b3f..162d90c 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -151,7 +151,9 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
@@ -406,7 +408,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
@@ -616,7 +620,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
 
     def test_cpp_only_stdout(self):
         # Given
@@ -976,7 +982,9 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assert_correct_output(self.timeout_msg, result.get('error'))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index 5ddf8cd..35b64d0 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -160,7 +160,9 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
@@ -405,7 +407,9 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
@@ -845,7 +849,9 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest):
         # Then
 
         self.assertFalse(result.get('success'))
-        self.assert_correct_output(self.timeout_msg, result.get('error'))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index a2faf77..71d7732 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -24,9 +24,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
             f.write('2'.encode('ascii'))
         tmp_in_dir_path = tempfile.mkdtemp()
         self.in_dir = tmp_in_dir_path
-        self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 0.0},
-                               {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 0.0},
-                               {"test_case_type": "standardtestcase", "test_case":  'assert(add(-1,-2)==-3)', 'weight': 0.0},
+        self.test_case_data = [{"test_case_type": "standardtestcase",
+                                "test_case": 'assert(add(1,2)==3)',
+                                 'weight': 0.0},
+                               {"test_case_type": "standardtestcase",
+                                "test_case": 'assert(add(-1,2)==1)',
+                                 'weight': 0.0},
+                               {"test_case_type": "standardtestcase",
+                                "test_case":  'assert(add(-1,-2)==-3)',
+                                 'weight': 0.0},
                                ]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in"
@@ -76,23 +82,29 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assert_correct_output('AssertionError  in:\n assert(add(1,2)==3)',
-                        result.get('error')
-                      )
-        self.assert_correct_output('AssertionError  in:\n assert(add(-1,2)==1)',
-                        result.get('error')
-                      )
-        self.assert_correct_output('AssertionError  in:\n assert(add(-1,-2)==-3)',
-                        result.get('error')
-                      )
+        given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
+        for error in result.get("error"):
+            self.assertEqual(error['exception'], 'AssertionError')
+            self.assertEqual(error['message'],
+                 "Expected answer from the test case did not match the output"
+                 )
+        error_testcase_list = [tc['test_case'] for tc in result.get('error')]
+        self.assertEqual(error_testcase_list, given_test_case_list)
+
 
     def test_partial_incorrect_answer(self):
         # Given
         user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
-        test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0},
-                            {"test_case_type": "standardtestcase", "test_case":  'assert(add(-1,-2)==-3)', 'weight': 1.0},
-                            {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0}
-                               ]
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": 'assert(add(-1,2)==1)',
+                           'weight': 1.0},
+                          {"test_case_type": "standardtestcase",
+                           "test_case":  'assert(add(-1,-2)==-3)',
+                            'weight': 1.0},
+                          {"test_case_type": "standardtestcase",
+                           "test_case": 'assert(add(1,2)==3)',
+                           'weight': 2.0}
+                          ]
         kwargs = {
                   'metadata': {
                     'user_answer': user_answer,
@@ -110,13 +122,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # Then
         self.assertFalse(result.get('success'))
         self.assertEqual(result.get('weight'), 2.0)
-        self.assert_correct_output('AssertionError  in:\n assert(add(-1,2)==1)',
-                        result.get('error')
-                      )
-        self.assert_correct_output('AssertionError  in:\n assert(add(-1,-2)==-3)',
-                        result.get('error')
-                      )
-
+        given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
+        given_test_case_list.remove('assert(add(1,2)==3)')
+        for error in result.get("error"):
+            self.assertEqual(error['exception'], 'AssertionError')
+            self.assertEqual(error['message'],
+                 "Expected answer from the test case did not match the output"
+                 )
+        error_testcase_list = [tc['test_case'] for tc in result.get('error')]
+        self.assertEqual(error_testcase_list, given_test_case_list)
     def test_infinite_loop(self):
         # Given
         user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
@@ -136,7 +150,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assert_correct_output(self.timeout_msg, result.get('error'))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
 
     def test_syntax_error(self):
         # Given
@@ -165,14 +181,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err = result.get("error")[0]['traceback']
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(5, len(err))
         for msg in syntax_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assert_correct_output(msg, err)
 
     def test_indent_error(self):
         # Given
@@ -200,13 +214,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        err = result.get("error")[0].splitlines()
+        err = result.get("error")[0]["traceback"].splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
         for msg in indent_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assert_correct_output(msg, 
+                                      result.get("error")[0]['traceback']
+                                      )
 
     def test_name_error(self):
         # Given
@@ -231,14 +247,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
-
-        # Then
-        self.assertFalse(result.get("success"))
-        self.assertEqual(25, len(err))
+        err = result.get("error")[0]["traceback"]
         for msg in name_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assertIn(msg, err)
 
     def test_recursion_error(self):
         # Given
@@ -246,10 +257,7 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         def add(a, b):
             return add(3, 3)
         """)
-        recursion_error_msg = ["Traceback", 
-                               "maximum recursion depth exceeded"
-                               ]
-
+        recursion_error_msg = "maximum recursion depth exceeded"
         kwargs = {
                   'metadata': {
                     'user_answer': user_answer,
@@ -263,13 +271,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err = result.get("error")[0]['message']
 
         # Then
         self.assertFalse(result.get("success"))
-        for msg in recursion_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+        self.assert_correct_output(recursion_error_msg, err)
 
     def test_type_error(self):
         # Given
@@ -296,14 +302,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err = result.get("error")[0]['traceback']
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(25, len(err))
         for msg in type_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assert_correct_output(msg, err)
 
     def test_value_error(self):
         # Given
@@ -332,18 +336,19 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err = result.get("error")[0]['traceback']
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(28, len(err))
         for msg in value_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assert_correct_output(msg, err)
 
     def test_file_based_assert(self):
         # Given
-        self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}]
+        self.test_case_data = [{"test_case_type": "standardtestcase",
+                                "test_case": "assert(ans()=='2')",
+                                "weight": 0.0}
+                               ]
         self.file_paths = [(self.tmp_file, False)]
         user_answer = dedent("""
             def ans():
@@ -369,20 +374,17 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         self.assertTrue(result.get('success'))
 
     def test_single_testcase_error(self):
-        # Given
         """ Tests the user answer with just an incorrect test case """
 
+        # Given
         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
         test_case_data = [{"test_case_type": "standardtestcase",
-                             "test_case": 's="abbb"\nasert palindrome(s)==False',
-                             "weight": 0.0
-                            }
+                           "test_case": 's="abbb"\nasert palindrome(s)==False',
+                           "weight": 0.0
+                           }
                           ]
         syntax_error_msg = ["Traceback",
                             "call",
-                            "File",
-                            "line",
-                            "<string>",
                             "SyntaxError",
                             "invalid syntax"
                             ]
@@ -399,14 +401,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err =  result.get("error")[0]['traceback']
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(13, len(err))
         for msg in syntax_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assert_correct_output(msg, err)
 
 
     def test_multiple_testcase_error(self):
@@ -415,13 +415,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # Given
         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
         test_case_data = [{"test_case_type": "standardtestcase",
-                             "test_case": 'assert(palindrome("abba")==True)',
-                             "weight": 0.0
-                            },
+                           "test_case": 'assert(palindrome("abba")==True)',
+                           "weight": 0.0},
                           {"test_case_type": "standardtestcase",
-                             "test_case": 's="abbb"\nassert palindrome(S)==False',
-                             "weight": 0.0
-                            }
+                           "test_case": 's="abbb"\nassert palindrome(S)==False',
+                           "weight": 0.0}
                           ]
         name_error_msg = ["Traceback",
                           "call",
@@ -441,14 +439,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-        error_as_str = ''.join(result.get("error"))
-        err = error_as_str.splitlines()
+        err = result.get("error")[0]['traceback']
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(11, len(err))
         for msg in name_error_msg:
-            self.assert_correct_output(msg, result.get("error"))
+            self.assertIn(msg, err)
 
     def test_unicode_literal_bug(self):
         # Given
@@ -674,7 +670,9 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
 
         # Then
-        self.assert_correct_output(timeout_msg, result.get('error'))
+        self.assert_correct_output(timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         self.assertFalse(result.get('success'))
 
 
@@ -915,7 +913,9 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
-        self.assert_correct_output(self.timeout_msg, result.get('error'))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
 
     def test_assignment_upload(self):
         # Given
diff --git a/yaksh/evaluator_tests/test_python_stdio_evaluator.py b/yaksh/evaluator_tests/test_python_stdio_evaluator.py
index 8877544..9b8d702 100644
--- a/yaksh/evaluator_tests/test_python_stdio_evaluator.py
+++ b/yaksh/evaluator_tests/test_python_stdio_evaluator.py
@@ -1,4 +1,4 @@
-from yaksh.compare_stdio import compare_outputs
+from yaksh.error_messages import compare_outputs
 
 def test_compare_outputs():
     exp = "5\n5\n"
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index c3a1c83..f7a9925 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -137,7 +137,9 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
 
         self.assertFalse(result.get("success"))
-        self.assert_correct_output(self.timeout_msg, result.get("error"))
+        self.assert_correct_output(self.timeout_msg,
+                                   result.get("error")[0]["message"]
+                                   )
         parent_proc = Process(os.getpid()).children()
         if parent_proc:
             children_procs = Process(parent_proc[0].pid)