summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/cpp_code_evaluator.py4
-rw-r--r--yaksh/cpp_stdio_evaluator.py2
-rw-r--r--yaksh/error_messages.py5
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py26
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py23
-rw-r--r--yaksh/grader.py16
-rw-r--r--yaksh/java_code_evaluator.py4
-rw-r--r--yaksh/java_stdio_evaluator.py2
-rw-r--r--yaksh/models.py12
-rw-r--r--yaksh/test_models.py18
10 files changed, 83 insertions, 29 deletions
diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py
index 8bd3beb..d249c66 100644
--- a/yaksh/cpp_code_evaluator.py
+++ b/yaksh/cpp_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
# Local imports
from .file_utils import copy_files, delete_files
from .base_evaluator import BaseEvaluator
+from .grader import CompilationError, TestCaseError
class CppCodeEvaluator(BaseEvaluator):
@@ -141,6 +142,7 @@ class CppCodeEvaluator(BaseEvaluator):
mark_fraction = 1.0 if self.partial_grading else 0.0
else:
err = "{0} \n {1}".format(stdout, stderr)
+ raise AssertionError(err)
else:
err = "Test case Error:"
try:
@@ -152,6 +154,7 @@ class CppCodeEvaluator(BaseEvaluator):
err = "{0} \n {1}".format(err, e)
except Exception:
err = "{0} \n {1}".format(err, main_err)
+ raise TestCaseError(err)
else:
err = "Compilation Error:"
try:
@@ -163,5 +166,6 @@ class CppCodeEvaluator(BaseEvaluator):
err = "{0} \n {1}".format(err, e)
except Exception:
err = "{0} \n {1}".format(err, stdnt_stderr)
+ raise CompilationError(err)
return success, err, mark_fraction
diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py
index 4e8f8df..3f44cb2 100644
--- a/yaksh/cpp_stdio_evaluator.py
+++ b/yaksh/cpp_stdio_evaluator.py
@@ -7,6 +7,7 @@ from os.path import isfile
# Local imports
from .stdio_evaluator import StdIOEvaluator
from .file_utils import copy_files, delete_files
+from .grader import CompilationError
class CppStdIOEvaluator(StdIOEvaluator):
@@ -104,5 +105,6 @@ class CppStdIOEvaluator(StdIOEvaluator):
err = err + "\n" + e
except Exception:
err = err + "\n" + stdnt_stderr
+ raise CompilationError(err)
mark_fraction = 1.0 if self.partial_grading and success else 0.0
return success, err, mark_fraction
diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py
index 512d664..f34bf28 100644
--- a/yaksh/error_messages.py
+++ b/yaksh/error_messages.py
@@ -11,7 +11,10 @@ def prettify_exceptions(exception, message, traceback=None,
"traceback": traceback,
"message": message
}
- if exception == 'RuntimeError' or exception == 'RecursionError':
+ ignore_traceback = ['RuntimeError', 'RecursionError',
+ "CompilationError", "TestCaseError"
+ ]
+ if exception in ignore_traceback:
err["traceback"] = None
if exception == 'AssertionError':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 5b49671..14ed808 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -100,12 +100,11 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
-
+ errors = result.get('error')
# Then
- lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
- self.assert_correct_output("Incorrect:", result.get('error'))
- self.assertTrue(lines_of_error > 1)
+ for error in errors:
+ self.assertEqual(error['exception'], 'AssertionError')
def test_compilation_error(self):
# Given
@@ -122,10 +121,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
- self.assert_correct_output("Compilation Error", result.get("error"))
+ for error in errors:
+ self.assertEqual(error['exception'], 'CompilationError')
def test_infinite_loop(self):
# Given
@@ -272,13 +273,12 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get("error")
# Then
- err = result.get('error')[0]
- lines_of_error = len(err.splitlines())
self.assertFalse(result.get('success'))
- self.assertTrue(lines_of_error > 1)
- self.assertIn("Test case Error", err)
+ for error in errors:
+ self.assertEqual(error['exception'], 'TestCaseError')
class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
@@ -432,10 +432,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
- self.assert_correct_output("Compilation Error", result.get("error"))
+ for error in errors:
+ self.assertEqual(error['exception'], 'CompilationError')
def test_infinite_loop(self):
# Given
@@ -636,10 +638,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
- self.assert_correct_output("Compilation Error", result.get("error"))
+ for error in errors:
+ self.assertEqual(error['exception'], 'CompilationError')
def test_cpp_infinite_loop(self):
# Given
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index ab86dec..eb09f2f 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -113,13 +113,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
self.assertFalse(result.get('success'))
- lines_of_error = len(result.get('error')[0].splitlines())
- self.assertFalse(result.get('success'))
- self.assert_correct_output("Incorrect", result.get('error'))
- self.assertTrue(lines_of_error > 1)
+ for error in errors:
+ self.assertEqual(error.get('exception'), 'AssertionError')
def test_error(self):
# Given
@@ -136,10 +135,11 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
-
+ errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
- self.assert_correct_output("Error", result.get("error"))
+ for error in errors:
+ self.assertEqual(error.get('exception'), 'CompilationError')
def test_infinite_loop(self):
# Given
@@ -290,13 +290,12 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
- err = result.get('error')[0]
- lines_of_error = len(err.splitlines())
self.assertFalse(result.get('success'))
- self.assertTrue(lines_of_error > 1)
- self.assertIn("Test case Error", err)
+ for error in errors:
+ self.assertEqual(error.get('exception'), 'TestCaseError')
class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
@@ -432,10 +431,12 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
+ errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
- self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
+ for error in errors:
+ self.assertEqual(error.get('exception'), 'CompilationError')
def test_infinite_loop(self):
# Given
diff --git a/yaksh/grader.py b/yaksh/grader.py
index 320e7e7..c1be493 100644
--- a/yaksh/grader.py
+++ b/yaksh/grader.py
@@ -23,6 +23,14 @@ class TimeoutException(Exception):
pass
+class CompilationError(Exception):
+ pass
+
+
+class TestCaseError(Exception):
+ pass
+
+
@contextlib.contextmanager
def change_dir(path):
cur_dir = abspath(dirname(MY_DIR))
@@ -159,10 +167,14 @@ class Grader(object):
line_no = traceback.extract_tb(exc_tb)[-1][1]
if len(tb_list) > 2:
del tb_list[1:3]
+ try:
+ exc_value = str(exc_value)
+ except UnicodeEncodeError:
+ exc_value = unicode(exc_value)
error.append(
prettify_exceptions(
- exc_type.__name__, str(exc_value), "".join(tb_list),
- line_no=line_no
+ exc_type.__name__, exc_value,
+ "".join(tb_list), line_no=line_no
)
)
finally:
diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py
index 5f2288d..35573c0 100644
--- a/yaksh/java_code_evaluator.py
+++ b/yaksh/java_code_evaluator.py
@@ -7,6 +7,7 @@ import subprocess
# Local imports
from .base_evaluator import BaseEvaluator
from .file_utils import copy_files, delete_files
+from .grader import CompilationError, TestCaseError
class JavaCodeEvaluator(BaseEvaluator):
@@ -149,6 +150,7 @@ class JavaCodeEvaluator(BaseEvaluator):
mark_fraction = 1.0 if self.partial_grading else 0.0
else:
err = stdout + "\n" + stderr
+ raise AssertionError(err)
else:
err = "Test case Error:"
try:
@@ -160,6 +162,7 @@ class JavaCodeEvaluator(BaseEvaluator):
err = err + "\n" + e
except Exception:
err = err + "\n" + main_err
+ raise TestCaseError(err)
else:
err = "Compilation Error:"
try:
@@ -171,5 +174,6 @@ class JavaCodeEvaluator(BaseEvaluator):
err = err + "\n" + e
except Exception:
err = err + "\n" + stdnt_stderr
+ raise CompilationError(err)
return success, err, mark_fraction
diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py
index 0504177..89f9fc4 100644
--- a/yaksh/java_stdio_evaluator.py
+++ b/yaksh/java_stdio_evaluator.py
@@ -7,6 +7,7 @@ from os.path import isfile
# Local imports
from .stdio_evaluator import StdIOEvaluator
from .file_utils import copy_files, delete_files
+from .grader import CompilationError
class JavaStdIOEvaluator(StdIOEvaluator):
@@ -86,5 +87,6 @@ class JavaStdIOEvaluator(StdIOEvaluator):
err = err + "\n" + e
except Exception:
err = err + "\n" + stdnt_stderr
+ raise CompilationError(err)
mark_fraction = 1.0 if self.partial_grading and success else 0.0
return success, err, mark_fraction
diff --git a/yaksh/models.py b/yaksh/models.py
index e25fd81..1eca721 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -1457,9 +1457,12 @@ class QuestionPaper(models.Model):
random.shuffle(testcases)
testcases_ids = ",".join([str(tc.id) for tc in testcases]
)
- TestCaseOrder.objects.create(
- answer_paper=ans_paper, question=question,
- order=testcases_ids)
+ if not TestCaseOrder.objects.filter(
+ answer_paper=ans_paper, question=question
+ ).exists():
+ TestCaseOrder.objects.create(
+ answer_paper=ans_paper, question=question,
+ order=testcases_ids)
ans_paper.questions_order = ",".join(question_ids)
ans_paper.save()
@@ -2297,4 +2300,7 @@ class TestCaseOrder(models.Model):
order = models.TextField()
+ class Meta:
+ unique_together = ("answer_paper", "question", "order")
+
##############################################################################
diff --git a/yaksh/test_models.py b/yaksh/test_models.py
index e2795bb..403dfb4 100644
--- a/yaksh/test_models.py
+++ b/yaksh/test_models.py
@@ -2,7 +2,8 @@ import unittest
from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\
QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\
StdIOBasedTestCase, FileUpload, McqTestCase, AssignmentUpload,\
- LearningModule, LearningUnit, Lesson, LessonFile, CourseStatus
+ LearningModule, LearningUnit, Lesson, LessonFile, CourseStatus, \
+ TestCaseOrder
from yaksh.code_server import (
ServerPool, get_result as get_result_from_code_server
)
@@ -1272,6 +1273,21 @@ class AnswerPaperTestCases(unittest.TestCase):
self.assertEqual(self.answer.marks, 0)
self.assertFalse(self.answer.correct)
+ def test_testcase_order(self):
+ testcase_ids = ",".join([str(ids) for ids in
+ self.question2.get_test_cases()
+ ])
+ testcase_order = TestCaseOrder.objects.create(
+ answer_paper=self.answerpaper,
+ question=self.question2,
+ order=testcase_ids)
+ with self.assertRaises(IntegrityError):
+ TestCaseOrder.objects.create(answer_paper=self.answerpaper,
+ question=self.question2,
+ order=testcase_ids
+ )
+ testcase_order.delete()
+
def test_validate_and_regrade_mcq_correct_answer(self):
# Given
mcq_answer = str(self.mcq_based_testcase.id)
generated by cgit (git 2.34.1) at 2024-12-21 10:08:32 +0000