Merge pull request #491 from maheshgudi/catch_compilation_error

Raise appropriate exceptions for C and Java language to the Grader

10 files changed, 83 insertions, 29 deletions

diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py
index 8bd3beb..d249c66 100644
--- a/yaksh/cpp_code_evaluator.py
+++ b/yaksh/cpp_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
 # Local imports
 from .file_utils import copy_files, delete_files
 from .base_evaluator import BaseEvaluator
+from .grader import CompilationError, TestCaseError
 
 
 class CppCodeEvaluator(BaseEvaluator):
@@ -141,6 +142,7 @@ class CppCodeEvaluator(BaseEvaluator):
                     mark_fraction = 1.0 if self.partial_grading else 0.0
                 else:
                     err = "{0} \n {1}".format(stdout, stderr)
+                    raise AssertionError(err)
             else:
                 err = "Test case Error:"
                 try:
@@ -152,6 +154,7 @@ class CppCodeEvaluator(BaseEvaluator):
                             err = "{0} \n {1}".format(err, e)
                 except Exception:
                         err = "{0} \n {1}".format(err, main_err)
+                raise TestCaseError(err)
         else:
             err = "Compilation Error:"
             try:
@@ -163,5 +166,6 @@ class CppCodeEvaluator(BaseEvaluator):
                         err = "{0} \n {1}".format(err, e)
             except Exception:
                 err = "{0} \n {1}".format(err, stdnt_stderr)
+            raise CompilationError(err)
 
         return success, err, mark_fraction
diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py
index 4e8f8df..3f44cb2 100644
--- a/yaksh/cpp_stdio_evaluator.py
+++ b/yaksh/cpp_stdio_evaluator.py
@@ -7,6 +7,7 @@ from os.path import isfile
 # Local imports
 from .stdio_evaluator import StdIOEvaluator
 from .file_utils import copy_files, delete_files
+from .grader import CompilationError
 
 
 class CppStdIOEvaluator(StdIOEvaluator):
@@ -104,5 +105,6 @@ class CppStdIOEvaluator(StdIOEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
         mark_fraction = 1.0 if self.partial_grading and success else 0.0
         return success, err, mark_fraction
diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py
index 512d664..f34bf28 100644
--- a/yaksh/error_messages.py
+++ b/yaksh/error_messages.py
@@ -11,7 +11,10 @@ def prettify_exceptions(exception, message, traceback=None,
            "traceback": traceback,
            "message": message
            }
-    if exception == 'RuntimeError' or exception == 'RecursionError':
+    ignore_traceback = ['RuntimeError', 'RecursionError',
+                        "CompilationError", "TestCaseError"
+                        ]
+    if exception in ignore_traceback:
         err["traceback"] = None
 
     if exception == 'AssertionError':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 5b49671..14ed808 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -100,12 +100,11 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-
+        errors = result.get('error')
         # Then
-        lines_of_error = len(result.get('error')[0].splitlines())
         self.assertFalse(result.get('success'))
-        self.assert_correct_output("Incorrect:", result.get('error'))
-        self.assertTrue(lines_of_error > 1)
+        for error in errors:
+            self.assertEqual(error['exception'], 'AssertionError')
 
     def test_compilation_error(self):
         # Given
@@ -122,10 +121,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
@@ -272,13 +273,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get("error")
 
         # Then
-        err = result.get('error')[0]
-        lines_of_error = len(err.splitlines())
         self.assertFalse(result.get('success'))
-        self.assertTrue(lines_of_error > 1)
-        self.assertIn("Test case Error", err)
+        for error in errors:
+            self.assertEqual(error['exception'], 'TestCaseError')
 
 
 class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
@@ -432,10 +432,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
@@ -636,10 +638,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Compilation Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error['exception'], 'CompilationError')
 
     def test_cpp_infinite_loop(self):
         # Given
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index ab86dec..eb09f2f 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -113,13 +113,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get('success'))
-        lines_of_error = len(result.get('error')[0].splitlines())
-        self.assertFalse(result.get('success'))
-        self.assert_correct_output("Incorrect", result.get('error'))
-        self.assertTrue(lines_of_error > 1)
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'AssertionError')
 
     def test_error(self):
         # Given
@@ -136,10 +135,11 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
-
+        errors = result.get('error')
         # Then
         self.assertFalse(result.get("success"))
-        self.assert_correct_output("Error", result.get("error"))
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
@@ -290,13 +290,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
-        err = result.get('error')[0]
-        lines_of_error = len(err.splitlines())
         self.assertFalse(result.get('success'))
-        self.assertTrue(lines_of_error > 1)
-        self.assertIn("Test case Error", err)
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'TestCaseError')
 
 
 class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
@@ -432,10 +431,12 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         # When
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
+        errors = result.get('error')
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
+        for error in errors:
+            self.assertEqual(error.get('exception'), 'CompilationError')
 
     def test_infinite_loop(self):
         # Given
diff --git a/yaksh/grader.py b/yaksh/grader.py
index 320e7e7..c1be493 100644
--- a/yaksh/grader.py
+++ b/yaksh/grader.py
@@ -23,6 +23,14 @@ class TimeoutException(Exception):
     pass
 
 
+class CompilationError(Exception):
+    pass
+
+
+class TestCaseError(Exception):
+    pass
+
+
 @contextlib.contextmanager
 def change_dir(path):
     cur_dir = abspath(dirname(MY_DIR))
@@ -159,10 +167,14 @@ class Grader(object):
                 line_no = traceback.extract_tb(exc_tb)[-1][1]
             if len(tb_list) > 2:
                 del tb_list[1:3]
+            try:
+                exc_value = str(exc_value)
+            except UnicodeEncodeError:
+                exc_value = unicode(exc_value)
             error.append(
                 prettify_exceptions(
-                    exc_type.__name__, str(exc_value), "".join(tb_list),
-                    line_no=line_no
+                    exc_type.__name__, exc_value,
+                    "".join(tb_list), line_no=line_no
                     )
                 )
         finally:
diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py
index 5f2288d..35573c0 100644
--- a/yaksh/java_code_evaluator.py
+++ b/yaksh/java_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
 # Local imports
 from .base_evaluator import BaseEvaluator
 from .file_utils import copy_files, delete_files
+from .grader import CompilationError, TestCaseError
 
 
 class JavaCodeEvaluator(BaseEvaluator):
@@ -149,6 +150,7 @@ class JavaCodeEvaluator(BaseEvaluator):
                     mark_fraction = 1.0 if self.partial_grading else 0.0
                 else:
                     err = stdout + "\n" + stderr
+                    raise AssertionError(err)
             else:
                 err = "Test case Error:"
                 try:
@@ -160,6 +162,7 @@ class JavaCodeEvaluator(BaseEvaluator):
                             err = err + "\n" + e
                 except Exception:
                         err = err + "\n" + main_err
+                raise TestCaseError(err)
         else:
             err = "Compilation Error:"
             try:
@@ -171,5 +174,6 @@ class JavaCodeEvaluator(BaseEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
 
         return success, err, mark_fraction
diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py
index 0504177..89f9fc4 100644
--- a/yaksh/java_stdio_evaluator.py
+++ b/yaksh/java_stdio_evaluator.py
@@ -7,6 +7,7 @@ from os.path import isfile
 # Local imports
 from .stdio_evaluator import StdIOEvaluator
 from .file_utils import copy_files, delete_files
+from .grader import CompilationError
 
 
 class JavaStdIOEvaluator(StdIOEvaluator):
@@ -86,5 +87,6 @@ class JavaStdIOEvaluator(StdIOEvaluator):
                         err = err + "\n" + e
             except Exception:
                 err = err + "\n" + stdnt_stderr
+            raise CompilationError(err)
         mark_fraction = 1.0 if self.partial_grading and success else 0.0
         return success, err, mark_fraction
diff --git a/yaksh/models.py b/yaksh/models.py
index e25fd81..1eca721 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -1457,9 +1457,12 @@ class QuestionPaper(models.Model):
                     random.shuffle(testcases)
                     testcases_ids = ",".join([str(tc.id) for tc in testcases]
                                              )
-                    TestCaseOrder.objects.create(
-                        answer_paper=ans_paper, question=question,
-                        order=testcases_ids)
+                    if not TestCaseOrder.objects.filter(
+                        answer_paper=ans_paper, question=question
+                        ).exists():
+                        TestCaseOrder.objects.create(
+                            answer_paper=ans_paper, question=question,
+                            order=testcases_ids)
 
             ans_paper.questions_order = ",".join(question_ids)
             ans_paper.save()
@@ -2297,4 +2300,7 @@ class TestCaseOrder(models.Model):
     order = models.TextField()
 
 
+    class Meta:
+        unique_together = ("answer_paper", "question", "order")
+
 ##############################################################################
diff --git a/yaksh/test_models.py b/yaksh/test_models.py
index e2795bb..403dfb4 100644
--- a/yaksh/test_models.py
+++ b/yaksh/test_models.py
@@ -2,7 +2,8 @@ import unittest
 from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\
     QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\
     StdIOBasedTestCase, FileUpload, McqTestCase, AssignmentUpload,\
-    LearningModule, LearningUnit, Lesson, LessonFile, CourseStatus
+    LearningModule, LearningUnit, Lesson, LessonFile, CourseStatus, \
+    TestCaseOrder
 from yaksh.code_server import (
     ServerPool, get_result as get_result_from_code_server
     )
@@ -1272,6 +1273,21 @@ class AnswerPaperTestCases(unittest.TestCase):
         self.assertEqual(self.answer.marks, 0)
         self.assertFalse(self.answer.correct)
 
+    def test_testcase_order(self):
+        testcase_ids = ",".join([str(ids) for ids in
+                                 self.question2.get_test_cases()
+                                 ])
+        testcase_order = TestCaseOrder.objects.create(
+                                       answer_paper=self.answerpaper,
+                                       question=self.question2,
+                                       order=testcase_ids)
+        with self.assertRaises(IntegrityError):
+            TestCaseOrder.objects.create(answer_paper=self.answerpaper,
+                                         question=self.question2,
+                                         order=testcase_ids
+                                         )
+        testcase_order.delete()
+
     def test_validate_and_regrade_mcq_correct_answer(self):
         # Given
         mcq_answer = str(self.mcq_based_testcase.id)