From 99f0c944fafec51b9327fe5dea01096842788e03 Mon Sep 17 00:00:00 2001
From: maheshgudi
Date: Wed, 4 Jul 2018 16:34:56 +0530
Subject: Raise appropriate exceptions for C and Java language to the Grader

- Raise AssertionError,CompilationError and TestCaseError for assertion, user answer compilation
  and testcase compilation errors respectively.
- Add test case for C/C++ and Java
---
 yaksh/cpp_code_evaluator.py                    |  4 ++++
 yaksh/cpp_stdio_evaluator.py                   |  3 ++-
 yaksh/error_messages.py                        |  5 ++++-
 yaksh/evaluator_tests/test_c_cpp_evaluation.py | 29 +++++++++++++++-----------
 yaksh/evaluator_tests/test_java_evaluation.py  | 23 ++++++++++----------
 yaksh/grader.py                                |  7 +++++++
 yaksh/java_code_evaluator.py                   |  4 ++++
 yaksh/java_stdio_evaluator.py                  |  2 ++
 8 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py
index 8bd3beb..d249c66 100644
--- a/yaksh/cpp_code_evaluator.py
+++ b/yaksh/cpp_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
 # Local imports
 from .file_utils import copy_files, delete_files
 from .base_evaluator import BaseEvaluator
+from .grader import CompilationError, TestCaseError
 
 
 class CppCodeEvaluator(BaseEvaluator):
@@ -141,6 +142,7 @@ class CppCodeEvaluator(BaseEvaluator):
                     mark_fraction = 1.0 if self.partial_grading else 0.0
                 else:
                     err = "{0} \n {1}".format(stdout, stderr)
+                    raise AssertionError(err)
             else:
                 err = "Test case Error:"
                 try:
@@ -152,6 +154,7 @@ class CppCodeEvaluator(BaseEvaluator):
                             err = "{0} \n {1}".format(err, e)
                 except Exception:
                         err = "{0} \n {1}".format(err, main_err)
+                raise TestCaseError(err)
         else:
             err = "Compilation Error:"
             try:
@@ -163,5 +166,6 @@ class CppCodeEvaluator(BaseEvaluator):
                         err = "{0} \n {1}".format(err, e)
             except Exception:
                 err = "{0} \n {1}".format(err, stdnt_stderr)
+            raise CompilationError(err)
 
         return success, err, mark_fraction
diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py
index 4e8f8df..26d655f 100644
--- a/yaksh/cpp_stdio_evaluator.py
+++ b/yaksh/cpp_stdio_evaluator.py
@@ -7,7 +7,7 @@ from os.path import isfile
 # Local imports
 from .stdio_evaluator import StdIOEvaluator
 from .file_utils import copy_files, delete_files
-
+from .grader import CompilationError
 
 class CppStdIOEvaluator(StdIOEvaluator):
     """Evaluates C StdIO based code"""
@@ -104,5 +104,6 @@ class CppStdIOEvaluator(StdIOEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
         mark_fraction = 1.0 if self.partial_grading and success else 0.0
         return success, err, mark_fraction
diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py
index 512d664..86375bf 100644
--- a/yaksh/error_messages.py
+++ b/yaksh/error_messages.py
@@ -11,7 +11,10 @@ def prettify_exceptions(exception, message, traceback=None,
            "traceback": traceback,
            "message": message
            }
-    if exception == 'RuntimeError' or exception == 'RecursionError':
+    ignore_traceback = ['RuntimeError','RecursionError',
+                        "CompilationError", "TestCaseError"
+                        ]
+    if exception in ignore_traceback:
         err["traceback"] = None
 
     if exception == 'AssertionError':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 5b49671..17f7c82 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -100,12 +100,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-
+        errors = result.get('error')
         # Then
-        lines_of_error = len(result.get('error')[0].splitlines())
         self.assertFalse(result.get('success'))
-        self.assert_correct_output("Incorrect:", result.get('error'))
-        self.assertTrue(lines_of_error > 1)
+        for error in errors:
+            self.assertEqual(error['exception'], 'AssertionError')
+
 
     def test_compilation_error(self):
         # Given
@@ -122,10 +122,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
@@ -272,14 +274,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get("error")
 
         # Then
-        err = result.get('error')[0]
-        lines_of_error = len(err.splitlines())
         self.assertFalse(result.get('success'))
-        self.assertTrue(lines_of_error > 1)
-        self.assertIn("Test case Error", err)
-
+        for error in errors:
+            self.assertEqual(error['exception'], 'TestCaseError')
 
 class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
     def setUp(self):
@@ -432,10 +432,13 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
+
 
     def test_infinite_loop(self):
         # Given
@@ -636,10 +639,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
 
     def test_cpp_infinite_loop(self):
         # Given
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index ab86dec..eb09f2f 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -113,13 +113,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get('success'))
-        lines_of_error = len(result.get('error')[0].splitlines())
-        self.assertFalse(result.get('success'))
-        self.assert_correct_output("Incorrect", result.get('error'))
-        self.assertTrue(lines_of_error > 1)
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'AssertionError')
 
     def test_error(self):
         # Given
@@ -136,10 +135,11 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-
+        errors = result.get('error')
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
@@ -290,13 +290,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
-        err = result.get('error')[0]
-        lines_of_error = len(err.splitlines())
         self.assertFalse(result.get('success'))
-        self.assertTrue(lines_of_error > 1)
-        self.assertIn("Test case Error", err)
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'TestCaseError')
 
 
 class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
@@ -432,10 +431,12 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
diff --git a/yaksh/grader.py b/yaksh/grader.py
index 320e7e7..de8f897 100644
--- a/yaksh/grader.py
+++ b/yaksh/grader.py
@@ -23,6 +23,13 @@ class TimeoutException(Exception):
     pass
 
 
+class CompilationError(Exception):
+    pass
+
+
+class TestCaseError(Exception):
+    pass
+
 @contextlib.contextmanager
 def change_dir(path):
     cur_dir = abspath(dirname(MY_DIR))
diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py
index 5f2288d..35573c0 100644
--- a/yaksh/java_code_evaluator.py
+++ b/yaksh/java_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
 # Local imports
 from .base_evaluator import BaseEvaluator
 from .file_utils import copy_files, delete_files
+from .grader import CompilationError, TestCaseError
 
 
 class JavaCodeEvaluator(BaseEvaluator):
@@ -149,6 +150,7 @@ class JavaCodeEvaluator(BaseEvaluator):
                     mark_fraction = 1.0 if self.partial_grading else 0.0
                 else:
                     err = stdout + "\n" + stderr
+                    raise AssertionError(err)
             else:
                 err = "Test case Error:"
                 try:
@@ -160,6 +162,7 @@ class JavaCodeEvaluator(BaseEvaluator):
                             err = err + "\n" + e
                 except Exception:
                         err = err + "\n" + main_err
+                raise TestCaseError(err)
         else:
             err = "Compilation Error:"
             try:
@@ -171,5 +174,6 @@ class JavaCodeEvaluator(BaseEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
 
         return success, err, mark_fraction
diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py
index 0504177..89f9fc4 100644
--- a/yaksh/java_stdio_evaluator.py
+++ b/yaksh/java_stdio_evaluator.py
@@ -7,6 +7,7 @@ from os.path import isfile
 # Local imports
 from .stdio_evaluator import StdIOEvaluator
 from .file_utils import copy_files, delete_files
+from .grader import CompilationError
 
 
 class JavaStdIOEvaluator(StdIOEvaluator):
@@ -86,5 +87,6 @@ class JavaStdIOEvaluator(StdIOEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
         mark_fraction = 1.0 if self.partial_grading and success else 0.0
         return success, err, mark_fraction
-- 
cgit