From 16e8d4b3c096e6034c0066adffc8f5a520f272c7 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 26 Oct 2017 18:34:36 +0530 Subject: Beautiful error outputs --- yaksh/compare_stdio.py | 3 ++- yaksh/python_assertion_evaluator.py | 40 +++++++++++++++++-------------------- yaksh/templates/exam.html | 32 ++++++++++++++++++++--------- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/yaksh/compare_stdio.py b/yaksh/compare_stdio.py index c4076de..9c13a98 100644 --- a/yaksh/compare_stdio.py +++ b/yaksh/compare_stdio.py @@ -18,7 +18,8 @@ def compare_outputs(expected_output, user_output, given_input=None): exp_lines = expected_output.splitlines() msg = {"given_input":given_input, "expected_output": exp_lines, - "user_output":given_lines + "user_output":given_lines, + "type": "stdio" } ng = len(given_lines) ne = len(exp_lines) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index c8f2dd0..35a08ec 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -4,6 +4,7 @@ import traceback import os import re from os.path import join +from textwrap import dedent import importlib # Local imports @@ -73,31 +74,26 @@ class PythonAssertionEvaluator(BaseEvaluator): exec(_tests, self.exec_scope) except TimeoutException: raise + except AssertionError: + type, _, tb = sys.exc_info() + tb_info = traceback.extract_tb(tb) + filename, line, func, text = tb_info[-1] + value = "Expected answer from the test case didnt match the output" + err = {"type": "assertion", + "test_case": self.test_case, + "exception": type.__name__, + "message": value + } except Exception: type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(self.test_case) + tb_info = traceback.extract_tb(tb) + filename, line, func, text = tb_info[-1] + err = {"type": "assertion", + "test_case": self.test_case, + "exception": type.__name__, + "message": str(value) + } - # Get truncated traceback - err_tb_lines = traceback.format_exc().splitlines() - stripped_tb_lines = [] - for line in err_tb_lines: - line = re.sub(r'File\s+".*?",\s+line', - 'File , line', - line - ) - stripped_tb_lines.append(line) - stripped_tb = '\n'.join(stripped_tb_lines[-10::]) - - err = "Expected Test Case:\n{0}\n" \ - "Error Traceback - {1} {2} in:\n {3}\n{4}".format( - self.test_case, - type.__name__, - str(value), - text, - stripped_tb - ) else: success = True err = None diff --git a/yaksh/templates/exam.html b/yaksh/templates/exam.html index 9596c1c..7b6d54e 100644 --- a/yaksh/templates/exam.html +++ b/yaksh/templates/exam.html @@ -88,17 +88,31 @@
Testcase No. {{ forloop.counter }}
- {% if not error.expected_output %} + {% if not error.type %} 
 {{error|safe}}
- {% else %} - {% if error.given_input %} - - - - - + {% elif error.type == 'assertion' %} + We tried the calling your function with the following test case:
+ 
{{error.test_case}}
+
For given Input value(s):{{error.given_input}}
+ But the following error took place: + + + + + + -
Exception Name{{error.exception}} +
Exception Message{{error.message}}
+ + {% elif error.type == 'stdio' %} + {% if error.given_input %} + + + + + + +
For given Input value(s):{{error.given_input}}
{% endif %} -- cgit From 72b3d672735f624064431cbb0751d3cc3b08b6ba Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 27 Oct 2017 02:22:08 +0530 Subject: Add traceback for exceptions --- yaksh/python_assertion_evaluator.py | 22 +++++++++++----------- yaksh/templates/exam.html | 28 +++++++++++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 35a08ec..af89fc3 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -75,23 +75,23 @@ class PythonAssertionEvaluator(BaseEvaluator): except TimeoutException: raise except AssertionError: - type, _, tb = sys.exc_info() - tb_info = traceback.extract_tb(tb) - filename, line, func, text = tb_info[-1] + exc_type, exc_value, exc_tb = sys.exc_info() value = "Expected answer from the test case didnt match the output" err = {"type": "assertion", "test_case": self.test_case, - "exception": type.__name__, - "message": value + "exception": exc_type.__name__, + "message": value, } except Exception: - type, value, tb = sys.exc_info() - tb_info = traceback.extract_tb(tb) - filename, line, func, text = tb_info[-1] + exc_type, exc_value, exc_tb = sys.exc_info() + tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) + if len(tb_list) > 2: + del tb_list[1:3] + err = {"type": "assertion", - "test_case": self.test_case, - "exception": type.__name__, - "message": str(value) + "traceback": "".join(tb_list), + "exception": exc_type.__name__, + "message": str(exc_value) } else: diff --git a/yaksh/templates/exam.html b/yaksh/templates/exam.html index 7b6d54e..8b573d4 100644 --- a/yaksh/templates/exam.html +++ b/yaksh/templates/exam.html @@ -91,17 +91,31 @@ {% if not error.type %} 
 {{error|safe}}
{% elif error.type == 'assertion' %} - We tried the calling your function with the following test case:
- 
{{error.test_case}}
-
- But the following error took place: - - + {% if error.test_case %} +
Exception Name
+ + + + + +
We tried calling your function with the following test case:{{error.test_case}}
+ {% endif %} +

The following error took place:

+ + + + - + + + + {% if error.traceback %} + + + {% endif %}
Exception Name: {{error.exception}}
Exception Message{{error.message}}Exception Message: {{error.message}}
Full exception: 
{{error.traceback}}
{% elif error.type == 'stdio' %} -- cgit From 70a35ac2a001bd9638d9db5ed645d00f94ae4666 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Mon, 6 Nov 2017 16:34:37 +0530 Subject: Change module compare_stdio to error_messages --- yaksh/compare_stdio.py | 44 --------------------------- yaksh/error_messages.py | 60 +++++++++++++++++++++++++++++++++++++ yaksh/grader.py | 16 +++++++--- yaksh/python_assertion_evaluator.py | 21 ++++--------- yaksh/python_stdio_evaluator.py | 2 +- yaksh/stdio_evaluator.py | 2 +- yaksh/templates/exam.html | 15 ++++------ 7 files changed, 85 insertions(+), 75 deletions(-) delete mode 100644 yaksh/compare_stdio.py create mode 100644 yaksh/error_messages.py diff --git a/yaksh/compare_stdio.py b/yaksh/compare_stdio.py deleted file mode 100644 index 9c13a98..0000000 --- a/yaksh/compare_stdio.py +++ /dev/null @@ -1,44 +0,0 @@ -try: - from itertools import zip_longest -except ImportError: - from itertools import izip_longest as zip_longest - - -def _get_incorrect_user_lines(exp_lines, user_lines): - err_line_numbers = [] - for line_no, (expected_line, user_line) in \ - enumerate(zip_longest(exp_lines, user_lines)): - if not user_line or not expected_line or \ - user_line.strip() != expected_line.strip(): - err_line_numbers.append(line_no) - return err_line_numbers - -def compare_outputs(expected_output, user_output, given_input=None): - given_lines = user_output.splitlines() - exp_lines = expected_output.splitlines() - msg = {"given_input":given_input, - "expected_output": exp_lines, - "user_output":given_lines, - "type": "stdio" - } - ng = len(given_lines) - ne = len(exp_lines) - err_line_numbers = _get_incorrect_user_lines(exp_lines, given_lines) - msg["error_line_numbers"] = err_line_numbers - if ng != ne: - msg["error_msg"] = ("Incorrect Answer: " - + "We had expected {} number of lines. ".format(ne) - + "We got {} number of lines.".format(ng) - ) - return False, msg - else: - if err_line_numbers: - msg["error_msg"] = ("Incorrect Answer: " - + "Line number(s) {0} did not match." - .format(", ".join(map( - str,[x+1 for x in err_line_numbers] - )))) - return False, msg - else: - msg["error_msg"] = "Correct Answer" - return True, msg diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py new file mode 100644 index 0000000..5453edd --- /dev/null +++ b/yaksh/error_messages.py @@ -0,0 +1,60 @@ +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest + +def prettify_exceptions(exception, message, traceback=None, testcase=None): + err = {"type": "assertion", + "exception": exception, + "traceback": traceback, + "message": message + } + + if exception == 'AssertionError': + value = ("Expected answer from the" + + " test case did not match the output") + err["message"] = value + err["traceback"] = None + if testcase: + err["test_case"] = testcase + return err + +def _get_incorrect_user_lines(exp_lines, user_lines): + err_line_numbers = [] + for line_no, (expected_line, user_line) in \ + enumerate(zip_longest(exp_lines, user_lines)): + if not user_line or not expected_line or \ + user_line.strip() != expected_line.strip(): + err_line_numbers.append(line_no) + return err_line_numbers + +def compare_outputs(expected_output, user_output, given_input=None): + given_lines = user_output.splitlines() + exp_lines = expected_output.splitlines() + msg = {"given_input":given_input, + "expected_output": exp_lines, + "user_output":given_lines, + "type": "stdio" + } + ng = len(given_lines) + ne = len(exp_lines) + err_line_numbers = _get_incorrect_user_lines(exp_lines, given_lines) + msg["error_line_numbers"] = err_line_numbers + if ng != ne: + msg["error_msg"] = ("Incorrect Answer: " + + "We had expected {} number of lines. "\ + .format(ne) + + "We got {} number of lines.".format(ng) + ) + return False, msg + else: + if err_line_numbers: + msg["error_msg"] = ("Incorrect Answer: " + + "Line number(s) {0} did not match." + .format(", ".join(map( + str,[x+1 for x in err_line_numbers] + )))) + return False, msg + else: + msg["error_msg"] = "Correct Answer" + return True, msg diff --git a/yaksh/grader.py b/yaksh/grader.py index a9a3738..4b4c892 100644 --- a/yaksh/grader.py +++ b/yaksh/grader.py @@ -21,7 +21,7 @@ except ImportError: # Local imports from .settings import SERVER_TIMEOUT from .language_registry import create_evaluator_instance - +from .error_messages import prettify_exceptions MY_DIR = abspath(dirname(__file__)) registry = None @@ -141,7 +141,8 @@ class Grader(object): for idx, test_case_instance in enumerate(test_case_instances): test_case_success = False test_case_instance.compile_code() - test_case_success, err, mark_fraction = test_case_instance.check_code() + eval_result = test_case_instance.check_code() + test_case_success, err, mark_fraction = eval_result if test_case_success: weight += mark_fraction * test_case_instance.weight else: @@ -154,7 +155,10 @@ class Grader(object): test_case_instance.teardown() except TimeoutException: - error.append(self.timeout_msg) + error.append(prettify_exceptions("TimeoutException", + self.timeout_msg + ) + ) except OSError: msg = traceback.format_exc(limit=0) error.append("Error: {0}".format(msg)) @@ -163,7 +167,11 @@ class Grader(object): tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) if len(tb_list) > 2: del tb_list[1:3] - error.append("Error: {0}".format("".join(tb_list))) + error.append(prettify_exceptions(exc_type.__name__, + str(exc_value), + "".join(tb_list), + ) + ) finally: # Set back any original signal handler. set_original_signal_handler(prev_handler) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index af89fc3..8c24e24 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -11,6 +11,7 @@ import importlib from .file_utils import copy_files, delete_files from .base_evaluator import BaseEvaluator from .grader import TimeoutException +from .error_messages import prettify_exceptions class PythonAssertionEvaluator(BaseEvaluator): @@ -74,26 +75,16 @@ class PythonAssertionEvaluator(BaseEvaluator): exec(_tests, self.exec_scope) except TimeoutException: raise - except AssertionError: - exc_type, exc_value, exc_tb = sys.exc_info() - value = "Expected answer from the test case didnt match the output" - err = {"type": "assertion", - "test_case": self.test_case, - "exception": exc_type.__name__, - "message": value, - } except Exception: exc_type, exc_value, exc_tb = sys.exc_info() tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) if len(tb_list) > 2: del tb_list[1:3] - - err = {"type": "assertion", - "traceback": "".join(tb_list), - "exception": exc_type.__name__, - "message": str(exc_value) - } - + err = prettify_exceptions(exc_type.__name__, + str(exc_value), + "".join(tb_list), + self.test_case + ) else: success = True err = None diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 2b443a7..b08103a 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -9,7 +9,7 @@ except ImportError: # Local imports from .file_utils import copy_files, delete_files from .base_evaluator import BaseEvaluator -from .compare_stdio import compare_outputs +from .error_messages import compare_outputs @contextmanager diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py index 5e4ce18..55adb5c 100644 --- a/yaksh/stdio_evaluator.py +++ b/yaksh/stdio_evaluator.py @@ -5,7 +5,7 @@ import signal # Local imports from .base_evaluator import BaseEvaluator from .grader import TimeoutException -from .compare_stdio import compare_outputs +from .error_messages import compare_outputs class StdIOEvaluator(BaseEvaluator): diff --git a/yaksh/templates/exam.html b/yaksh/templates/exam.html index 8b573d4..1d7af9c 100644 --- a/yaksh/templates/exam.html +++ b/yaksh/templates/exam.html @@ -80,6 +80,7 @@ {% block main %} {% endblock %}
+
{% if question.type == 'code' or question.type == 'upload' %} {% if error_message %}
@@ -92,28 +93,22 @@ 
 {{error|safe}}
{% elif error.type == 'assertion' %} {% if error.test_case %} - - - - - - -
We tried calling your function with the following test case:{{error.test_case}}
+ We tried you code with the following test case:

+ 
{{error.test_case}}
{% endif %}

The following error took place:

- + {% if error.traceback %} - + {% endif %} -- cgit From 08c076b7945dec8257efc10db4c9b141a8cf7f8c Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Mon, 6 Nov 2017 19:35:13 +0530 Subject: Modify testcases wrt changes in assertion error output --- yaksh/error_messages.py | 3 +- yaksh/evaluator_tests/test_bash_evaluation.py | 8 +- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 16 ++- yaksh/evaluator_tests/test_java_evaluation.py | 12 +- yaksh/evaluator_tests/test_python_evaluation.py | 154 ++++++++++----------- .../evaluator_tests/test_python_stdio_evaluator.py | 2 +- yaksh/evaluator_tests/test_scilab_evaluation.py | 4 +- yaksh/tests/test_code_server.py | 6 +- 8 files changed, 113 insertions(+), 92 deletions(-) diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py index 5453edd..25b690b 100644 --- a/yaksh/error_messages.py +++ b/yaksh/error_messages.py @@ -9,7 +9,8 @@ def prettify_exceptions(exception, message, traceback=None, testcase=None): "traceback": traceback, "message": message } - + if exception == "RecursionError": + err["traceback"] = None if exception == 'AssertionError': value = ("Expected answer from the" + " test case did not match the output") diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 2faa7bf..5542710 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -104,7 +104,9 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -533,7 +535,9 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 0898b3f..162d90c 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -151,7 +151,9 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -406,7 +408,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -616,7 +620,9 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_cpp_only_stdout(self): # Given @@ -976,7 +982,9 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 5ddf8cd..35b64d0 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -160,7 +160,9 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -405,7 +407,9 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) @@ -845,7 +849,9 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index a2faf77..71d7732 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -24,9 +24,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, + self.test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(add(1,2)==3)', + 'weight': 0.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,2)==1)', + 'weight': 0.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,-2)==-3)', + 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -76,23 +82,29 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output('AssertionError in:\n assert(add(1,2)==3)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,2)==1)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,-2)==-3)', - result.get('error') - ) + given_test_case_list = [tc["test_case"] for tc in self.test_case_data] + for error in result.get("error"): + self.assertEqual(error['exception'], 'AssertionError') + self.assertEqual(error['message'], + "Expected answer from the test case did not match the output" + ) + error_testcase_list = [tc['test_case'] for tc in result.get('error')] + self.assertEqual(error_testcase_list, given_test_case_list) + def test_partial_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" - test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, - {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} - ] + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,2)==1)', + 'weight': 1.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(-1,-2)==-3)', + 'weight': 1.0}, + {"test_case_type": "standardtestcase", + "test_case": 'assert(add(1,2)==3)', + 'weight': 2.0} + ] kwargs = { 'metadata': { 'user_answer': user_answer, @@ -110,13 +122,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('weight'), 2.0) - self.assert_correct_output('AssertionError in:\n assert(add(-1,2)==1)', - result.get('error') - ) - self.assert_correct_output('AssertionError in:\n assert(add(-1,-2)==-3)', - result.get('error') - ) - + given_test_case_list = [tc["test_case"] for tc in self.test_case_data] + given_test_case_list.remove('assert(add(1,2)==3)') + for error in result.get("error"): + self.assertEqual(error['exception'], 'AssertionError') + self.assertEqual(error['message'], + "Expected answer from the test case did not match the output" + ) + error_testcase_list = [tc['test_case'] for tc in result.get('error')] + self.assertEqual(error_testcase_list, given_test_case_list) def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" @@ -136,7 +150,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_syntax_error(self): # Given @@ -165,14 +181,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(5, len(err)) for msg in syntax_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_indent_error(self): # Given @@ -200,13 +214,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error")[0].splitlines() + err = result.get("error")[0]["traceback"].splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, + result.get("error")[0]['traceback'] + ) def test_name_error(self): # Given @@ -231,14 +247,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() - - # Then - self.assertFalse(result.get("success")) - self.assertEqual(25, len(err)) + err = result.get("error")[0]["traceback"] for msg in name_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assertIn(msg, err) def test_recursion_error(self): # Given @@ -246,10 +257,7 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): def add(a, b): return add(3, 3) """) - recursion_error_msg = ["Traceback", - "maximum recursion depth exceeded" - ] - + recursion_error_msg = "maximum recursion depth exceeded" kwargs = { 'metadata': { 'user_answer': user_answer, @@ -263,13 +271,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['message'] # Then self.assertFalse(result.get("success")) - for msg in recursion_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(recursion_error_msg, err) def test_type_error(self): # Given @@ -296,14 +302,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(25, len(err)) for msg in type_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_value_error(self): # Given @@ -332,18 +336,19 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(28, len(err)) for msg in value_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}] + self.test_case_data = [{"test_case_type": "standardtestcase", + "test_case": "assert(ans()=='2')", + "weight": 0.0} + ] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" def ans(): @@ -369,20 +374,17 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): self.assertTrue(result.get('success')) def test_single_testcase_error(self): - # Given """ Tests the user answer with just an incorrect test case """ + # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case_type": "standardtestcase", - "test_case": 's="abbb"\nasert palindrome(s)==False', - "weight": 0.0 - } + "test_case": 's="abbb"\nasert palindrome(s)==False', + "weight": 0.0 + } ] syntax_error_msg = ["Traceback", "call", - "File", - "line", - "", "SyntaxError", "invalid syntax" ] @@ -399,14 +401,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(13, len(err)) for msg in syntax_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assert_correct_output(msg, err) def test_multiple_testcase_error(self): @@ -415,13 +415,11 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{"test_case_type": "standardtestcase", - "test_case": 'assert(palindrome("abba")==True)', - "weight": 0.0 - }, + "test_case": 'assert(palindrome("abba")==True)', + "weight": 0.0}, {"test_case_type": "standardtestcase", - "test_case": 's="abbb"\nassert palindrome(S)==False', - "weight": 0.0 - } + "test_case": 's="abbb"\nassert palindrome(S)==False', + "weight": 0.0} ] name_error_msg = ["Traceback", "call", @@ -441,14 +439,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - error_as_str = ''.join(result.get("error")) - err = error_as_str.splitlines() + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) - self.assertEqual(11, len(err)) for msg in name_error_msg: - self.assert_correct_output(msg, result.get("error")) + self.assertIn(msg, err) def test_unicode_literal_bug(self): # Given @@ -674,7 +670,9 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): result = grader.evaluate(kwargs) # Then - self.assert_correct_output(timeout_msg, result.get('error')) + self.assert_correct_output(timeout_msg, + result.get("error")[0]["message"] + ) self.assertFalse(result.get('success')) @@ -915,7 +913,9 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertFalse(result.get('success')) - self.assert_correct_output(self.timeout_msg, result.get('error')) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) def test_assignment_upload(self): # Given diff --git a/yaksh/evaluator_tests/test_python_stdio_evaluator.py b/yaksh/evaluator_tests/test_python_stdio_evaluator.py index 8877544..9b8d702 100644 --- a/yaksh/evaluator_tests/test_python_stdio_evaluator.py +++ b/yaksh/evaluator_tests/test_python_stdio_evaluator.py @@ -1,4 +1,4 @@ -from yaksh.compare_stdio import compare_outputs +from yaksh.error_messages import compare_outputs def test_compare_outputs(): exp = "5\n5\n" diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index c3a1c83..f7a9925 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -137,7 +137,9 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest): result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) - self.assert_correct_output(self.timeout_msg, result.get("error")) + self.assert_correct_output(self.timeout_msg, + result.get("error")[0]["message"] + ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 5f80f2d..1309624 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -61,7 +61,7 @@ class TestCodeServer(unittest.TestCase): # Then data = json.loads(result.get('result')) self.assertFalse(data['success']) - self.assertTrue('infinite loop' in data['error'][0]) + self.assertTrue('infinite loop' in data['error'][0]['message']) def test_correct_answer(self): # Given @@ -104,7 +104,7 @@ class TestCodeServer(unittest.TestCase): # Then data = json.loads(result.get('result')) self.assertFalse(data['success']) - self.assertTrue('AssertionError' in data['error'][0]) + self.assertTrue('AssertionError' in data['error'][0]['exception']) def test_multiple_simultaneous_hits(self): # Given @@ -143,7 +143,7 @@ class TestCodeServer(unittest.TestCase): for i in range(N): data = results.get() self.assertFalse(data['success']) - self.assertTrue('infinite loop' in data['error'][0]) + self.assertTrue('infinite loop' in data['error'][0]['message']) def test_server_pool_status(self): # Given -- cgit From caf37373ec546d53db7caaf8aca9d5550d0ed4ad Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Mon, 6 Nov 2017 20:05:40 +0530 Subject: Modify answerpaper wrt to changes in assertion error output --- yaksh/templates/yaksh/grade_user.html | 31 +++++++++++++++++++++++++--- yaksh/templates/yaksh/user_data.html | 32 +++++++++++++++++++++++++---- yaksh/templates/yaksh/view_answerpaper.html | 31 +++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/yaksh/templates/yaksh/grade_user.html b/yaksh/templates/yaksh/grade_user.html index 37bc788..3339177 100644 --- a/yaksh/templates/yaksh/grade_user.html +++ b/yaksh/templates/yaksh/grade_user.html @@ -218,9 +218,8 @@ Status : Passed
{% endif %} {% with ans.error_list as err %} {% for error in err %} - {% if not error.expected_output %} - 
 {{error|safe}}
- {% else %} + + {% if error.type == 'stdio' %}
{% if error.given_input %}
Exception Name: {{error.exception}} - {{error.exception}}
Exception Message: {{error.message}}
Full exception: Full Traceback: 
{{error.traceback}}
@@ -262,6 +261,32 @@ Status : Passed
+ {% elif error.type == 'assertion' %} + {% if error.test_case %} + We tried you code with the following test case:

+ 
{{error.test_case}}
+ {% endif %} +

The following error took place:

+
+ + + + + + + + + + + {% if error.traceback %} + + + {% endif %} + +
Exception Name: {{error.exception}}
Exception Message: {{error.message}}
Full Traceback: 
{{error.traceback}}
+
+ {% else %} + 
 {{error|safe}}
{% endif %} {% endfor %} {% endwith %} diff --git a/yaksh/templates/yaksh/user_data.html b/yaksh/templates/yaksh/user_data.html index 6dfaac3..a0219dd 100644 --- a/yaksh/templates/yaksh/user_data.html +++ b/yaksh/templates/yaksh/user_data.html @@ -136,12 +136,10 @@ User IP address: {{ paper.user_ip }}
Correct answer {% else %}
-
Error +
Error
{% with answer.error_list as err %} {% for error in err %} - {% if not error.expected_output %} - 
 {{error|safe}}
- {% else %} + {% if error.type == 'stdio' %}
{% if error.given_input %} @@ -183,6 +181,32 @@ User IP address: {{ paper.user_ip }}
+ {% elif error.type == 'assertion' %} + {% if error.test_case %} + We tried you code with the following test case:

+ 
{{error.test_case}}
+ {% endif %} +

The following error took place:

+
+ + + + + + + + + + + {% if error.traceback %} + + + {% endif %} + +
Exception Name: {{error.exception}}
Exception Message: {{error.message}}
Full Traceback: 
{{error.traceback}}
+
+ {% else %} + 
 {{error|safe}}
{% endif %} {% endfor %} {% endwith %} diff --git a/yaksh/templates/yaksh/view_answerpaper.html b/yaksh/templates/yaksh/view_answerpaper.html index 79987b1..fa16a08 100644 --- a/yaksh/templates/yaksh/view_answerpaper.html +++ b/yaksh/templates/yaksh/view_answerpaper.html @@ -131,9 +131,8 @@ {% with answer.error_list as err %} {% for error in err %} - {% if not error.expected_output %} - 
 {{error|safe}}
- {% else %} + + {% if error.type == 'stdio' %}
{% if error.given_input %} @@ -175,6 +174,32 @@
+ {% elif error.type == 'assertion' %} + {% if error.test_case %} + We tried you code with the following test case:

+ 
{{error.test_case}}
+ {% endif %} +

The following error took place:

+
+ + + + + + + + + + + {% if error.traceback %} + + + {% endif %} + +
Exception Name: {{error.exception}}
Exception Message: {{error.message}}
Full Traceback: 
{{error.traceback}}
+
+ {% else %} + 
 {{error|safe}}
{% endif %} {% endfor %} {% endwith %} -- cgit From 8f3cd30a5fe553bc8aa701fc76ba8d2a55c4418f Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 10 Nov 2017 03:00:24 +0530 Subject: No tracebacks if longer than 5 lines. - Tracebacks with more than 5 lines will not be shown. - Remove unnecessary imports. - PEP8 change. --- yaksh/error_messages.py | 12 ++++++------ yaksh/python_assertion_evaluator.py | 3 --- yaksh/templates/exam.html | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py index 25b690b..77bb1c9 100644 --- a/yaksh/error_messages.py +++ b/yaksh/error_messages.py @@ -9,8 +9,8 @@ def prettify_exceptions(exception, message, traceback=None, testcase=None): "traceback": traceback, "message": message } - if exception == "RecursionError": - err["traceback"] = None + if traceback and traceback.count('\n') > 6: + err["traceback"] = None if exception == 'AssertionError': value = ("Expected answer from the" + " test case did not match the output") @@ -32,11 +32,11 @@ def _get_incorrect_user_lines(exp_lines, user_lines): def compare_outputs(expected_output, user_output, given_input=None): given_lines = user_output.splitlines() exp_lines = expected_output.splitlines() - msg = {"given_input":given_input, + msg = {"type": "stdio", + "given_input": given_input, "expected_output": exp_lines, - "user_output":given_lines, - "type": "stdio" - } + "user_output": given_lines + } ng = len(given_lines) ne = len(exp_lines) err_line_numbers = _get_incorrect_user_lines(exp_lines, given_lines) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 8c24e24..440f422 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -4,7 +4,6 @@ import traceback import os import re from os.path import join -from textwrap import dedent import importlib # Local imports @@ -70,7 +69,6 @@ class PythonAssertionEvaluator(BaseEvaluator): success = False mark_fraction = 0.0 try: - tb = None _tests = compile(self.test_case, '', mode='exec') exec(_tests, self.exec_scope) except TimeoutException: @@ -89,5 +87,4 @@ class PythonAssertionEvaluator(BaseEvaluator): success = True err = None mark_fraction = 1.0 if self.partial_grading else 0.0 - del tb return success, err, mark_fraction diff --git a/yaksh/templates/exam.html b/yaksh/templates/exam.html index 1d7af9c..1cd3964 100644 --- a/yaksh/templates/exam.html +++ b/yaksh/templates/exam.html @@ -86,14 +86,14 @@
{% for error in error_message %}
-
Testcase No. {{ forloop.counter }}
+
Error No. {{ forloop.counter }}
{% if not error.type %} 
 {{error|safe}}
{% elif error.type == 'assertion' %} {% if error.test_case %} - We tried you code with the following test case:

+ We tried your code with the following test case:

{{error.test_case}}
{% endif %}

The following error took place:

-- cgit From 0bb6a4984d1e97c50a4e8da9394798c4c9a1e589 Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 10 Nov 2017 03:01:38 +0530 Subject: Pretty exceptions for hook code. --- yaksh/hook_evaluator.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index f5364d6..41ef6e4 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -2,13 +2,13 @@ import sys import traceback import os -import signal import psutil # Local imports from .file_utils import copy_files, delete_files from .base_evaluator import BaseEvaluator from .grader import TimeoutException +from .error_messages import prettify_exceptions class HookEvaluator(BaseEvaluator): @@ -60,19 +60,32 @@ class HookEvaluator(BaseEvaluator): success = False mark_fraction = 0.0 try: - tb = None _tests = compile(self.hook_code, '', mode='exec') hook_scope = {} exec(_tests, hook_scope) check = hook_scope["check_answer"] - success, err, mark_fraction = check(self.user_answer) + try: + success, err, mark_fraction = check(self.user_answer) + except Exception: + raise + except TimeoutException: processes = psutil.Process(os.getpid()).children(recursive=True) for process in processes: process.kill() raise except Exception: - msg = traceback.format_exc(limit=0) - err = "Error in Hook code: {0}".format(msg) - del tb + exc_type, exc_value, exc_tb = sys.exc_info() + tb_list = traceback.format_exception(exc_type, + exc_value, + exc_tb + ) + if len(tb_list) > 2: + del tb_list[1:3] + err = prettify_exceptions(exc_type.__name__, + str(exc_value), + "Error in Hook Code:\n" + + "".join(tb_list) + ) + return success, err, mark_fraction -- cgit From 4d5e801efff62ee63538e3c787ab74e6503e3d74 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 10 Nov 2017 16:50:18 +0530 Subject: Add textwrap for assertion error table --- yaksh/static/yaksh/css/exam.css | 2 +- yaksh/templates/exam.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yaksh/static/yaksh/css/exam.css b/yaksh/static/yaksh/css/exam.css index fff904e..ec48a14 100644 --- a/yaksh/static/yaksh/css/exam.css +++ b/yaksh/static/yaksh/css/exam.css @@ -2,6 +2,6 @@ table td, table th { border: black solid 1px !important; word-wrap: break-word !important; white-space: pre-wrap !important; } -output{ +#stdio, #assertion { table-layout: fixed } \ No newline at end of file diff --git a/yaksh/templates/exam.html b/yaksh/templates/exam.html index 1cd3964..a1f0df4 100644 --- a/yaksh/templates/exam.html +++ b/yaksh/templates/exam.html @@ -97,7 +97,7 @@ 
{{error.test_case}}
{% endif %}

The following error took place:

- +
@@ -123,7 +123,7 @@
Exception Name:
{% endif %} - +
-- cgit From 95f862caee8ca6077ee8f9a8fc88d9ca44db1cdf Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 10 Nov 2017 16:50:54 +0530 Subject: Remove traceback for recursion and/or runtime errors. --- yaksh/error_messages.py | 3 ++- yaksh/grader.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/yaksh/error_messages.py b/yaksh/error_messages.py index 77bb1c9..7ea8618 100644 --- a/yaksh/error_messages.py +++ b/yaksh/error_messages.py @@ -9,8 +9,9 @@ def prettify_exceptions(exception, message, traceback=None, testcase=None): "traceback": traceback, "message": message } - if traceback and traceback.count('\n') > 6: + if exception == 'RuntimeError' or exception == 'RecursionError': err["traceback"] = None + if exception == 'AssertionError': value = ("Expected answer from the" + " test case did not match the output") diff --git a/yaksh/grader.py b/yaksh/grader.py index 4b4c892..38cce8d 100644 --- a/yaksh/grader.py +++ b/yaksh/grader.py @@ -159,9 +159,6 @@ class Grader(object): self.timeout_msg ) ) - except OSError: - msg = traceback.format_exc(limit=0) - error.append("Error: {0}".format(msg)) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) -- cgit