diff options
Diffstat (limited to 'yaksh/evaluator_tests')
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 554 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_simple_question_types.py | 342 |
2 files changed, 581 insertions, 315 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 71d7732..1933d17 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import unittest import os -import sys import tempfile import shutil from textwrap import dedent @@ -26,13 +25,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): self.in_dir = tmp_in_dir_path self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', - 'weight': 0.0}, + 'weight': 0.0}, {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', - 'weight': 0.0}, + 'weight': 0.0}, {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', - 'weight': 0.0}, + 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -46,14 +45,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, } # When @@ -66,14 +63,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, } # When @@ -85,13 +80,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): given_test_case_list = [tc["test_case"] for tc in self.test_case_data] for error in result.get("error"): self.assertEqual(error['exception'], 'AssertionError') - self.assertEqual(error['message'], - "Expected answer from the test case did not match the output" - ) + self.assertEqual( + error['message'], + "Expected answer from the test case did not match the output" + ) error_testcase_list = [tc['test_case'] for tc in result.get('error')] self.assertEqual(error_testcase_list, given_test_case_list) - def test_partial_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" @@ -100,19 +95,17 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): 'weight': 1.0}, {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', - 'weight': 1.0}, + 'weight': 1.0}, {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': True, - 'language': 'python' - }, - 'test_case_data': test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -126,22 +119,22 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): given_test_case_list.remove('assert(add(1,2)==3)') for error in result.get("error"): self.assertEqual(error['exception'], 'AssertionError') - self.assertEqual(error['message'], - "Expected answer from the test case did not match the output" - ) + self.assertEqual( + error['message'], + "Expected answer from the test case did not match the output" + ) error_testcase_list = [tc['test_case'] for tc in result.get('error')] self.assertEqual(error_testcase_list, given_test_case_list) + def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, } # When @@ -168,14 +161,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "SyntaxError", "invalid syntax" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, } # When @@ -201,14 +192,12 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "IndentationError", "indented block" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, } # When @@ -220,9 +209,9 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: - self.assert_correct_output(msg, - result.get("error")[0]['traceback'] - ) + self.assert_correct_output( + msg, result.get("error")[0]['traceback'] + ) def test_name_error(self): # Given @@ -234,15 +223,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "defined" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, + } # When grader = Grader(self.in_dir) @@ -258,15 +245,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): return add(3, 3) """) recursion_error_msg = "maximum recursion depth exceeded" - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, + } # When grader = Grader(self.in_dir) @@ -289,15 +274,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "argument" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, + } # When grader = Grader(self.in_dir) @@ -323,25 +306,26 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "base" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, + } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error")[0]['traceback'] + errors = result.get("error") # Then self.assertFalse(result.get("success")) for msg in value_error_msg: - self.assert_correct_output(msg, err) + self.assert_correct_output(msg, errors[0]['traceback']) + for index, error in enumerate(errors): + self.assertEqual(error['test_case'], + self.test_case_data[index]['test_case']) def test_file_based_assert(self): # Given @@ -356,15 +340,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): return f.read()[0] """) - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data, + } # When grader = Grader(self.in_dir) @@ -390,25 +372,23 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): ] kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) - err = result.get("error")[0]['traceback'] + err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) for msg in syntax_error_msg: self.assert_correct_output(msg, err) - def test_multiple_testcase_error(self): """ Tests the user answer with an correct test case first and then with an incorrect test case """ @@ -418,7 +398,8 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "test_case": 'assert(palindrome("abba")==True)', "weight": 0.0}, {"test_case_type": "standardtestcase", - "test_case": 's="abbb"\nassert palindrome(S)==False', + "test_case": 's="abbb"\n' + 'assert palindrome(S)==False', "weight": 0.0} ] name_error_msg = ["Traceback", @@ -426,15 +407,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): "NameError", "name 'S' is not defined" ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } # When grader = Grader(self.in_dir) @@ -454,18 +433,15 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): return type(a) """) test_case_data = [{"test_case_type": "standardtestcase", - "test_case": 'assert(strchar("hello")==str)', - "weight": 0.0 - },] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, - } + "test_case": 'assert(strchar("hello")==str)', + "weight": 0.0}] + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) @@ -473,6 +449,31 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest): # Then self.assertTrue(result.get("success")) + def test_incorrect_answer_with_nose_assert(self): + user_answer = dedent("""\ + def add(a, b): + return a - b + """) + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert_equal(add(1, 2), 3)', + "weight": 0.0}] + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get("success")) + error = result.get("error")[0] + self.assertEqual(error['exception'], 'AssertionError') + self.assertEqual(error['message'], '-1 != 3') + class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): def setUp(self): @@ -501,13 +502,12 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): """ ) kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -534,13 +534,12 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): ) kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -551,11 +550,13 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): def test_correct_answer_string(self): # Given - self.test_case_data = [{"test_case_type": "stdiobasedtestcase", - "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), - "expected_output": "2", - "weight": 0.0 - }] + self.test_case_data = [{ + "test_case_type": "stdiobasedtestcase", + "expected_input": ("the quick brown fox jumps over " + "the lazy dog\nthe"), + "expected_output": "2", + "weight": 0.0 + }] user_answer = dedent(""" from six.moves import input a = str(input()) @@ -565,13 +566,12 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): ) kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -594,13 +594,12 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): """ ) kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -629,13 +628,12 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): """ ) kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -646,24 +644,24 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): def test_infinite_loop(self): # Given - self.test_case_data = [{"test_case_type": "stdiobasedtestcase", - "expected_input": "1\n2", - "expected_output": "3", - "weight": 0.0 - }] + self.test_case_data = [{ + "test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", + "expected_output": "3", + "weight": 0.0 + }] timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': self.test_case_data - } + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': self.test_case_data + } # When grader = Grader(self.in_dir) @@ -675,7 +673,6 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): ) self.assertFalse(result.get('success')) - def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ @@ -687,21 +684,44 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): "expected_output": "str", "weight": 0.0 }] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success")) + def test_get_error_lineno(self): + user_answer = dedent("""\ + print(1/0) + """) + test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "", + "expected_output": "1", + "weight": 0.0 + }] + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, + } + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + # Then + self.assertFalse(result.get("success")) + error = result.get("error")[0] + self.assertEqual(error['line_no'], 1) + self.assertEqual(error['exception'], "ZeroDivisionError") + class PythonHookEvaluationTestCases(EvaluatorBaseTest): @@ -733,19 +753,17 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) + ) test_case_data = [{"test_case_type": "hooktestcase", - "hook_code": hook_code,"weight": 1.0 - }] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': True, - 'language': 'python' - }, - 'test_case_data': test_case_data, + "hook_code": hook_code, "weight": 1.0 + }] + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -768,20 +786,18 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) + ) test_case_data = [{"test_case_type": "hooktestcase", - "hook_code": hook_code,"weight": 1.0 - }] - - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, + "hook_code": hook_code, "weight": 1.0 + }] + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -805,21 +821,19 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) + ) test_case_data = [{"test_case_type": "standardtestcase", "test_case": assert_test_case, 'weight': 1.0}, {"test_case_type": "hooktestcase", "hook_code": hook_code, 'weight': 1.0}, ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': True, - 'language': 'python' - }, - 'test_case_data': test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -842,7 +856,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 0.5 return success, err, mark_fraction """ - ) + ) hook_code_2 = dedent("""\ def check_answer(user_answer): success = False @@ -853,22 +867,19 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) - + ) test_case_data = [{"test_case_type": "hooktestcase", "hook_code": hook_code_1, 'weight': 1.0}, {"test_case_type": "hooktestcase", "hook_code": hook_code_2, 'weight': 1.0}, ] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': True, - 'language': 'python' - }, - 'test_case_data': test_case_data, + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -892,19 +903,18 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) + ) + test_case_data = [{"test_case_type": "hooktestcase", - "hook_code": hook_code,"weight": 1.0 - }] - - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, + "hook_code": hook_code, "weight": 1.0 + }] + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -931,19 +941,18 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ - ) + ) + test_case_data = [{"test_case_type": "hooktestcase", - "hook_code": hook_code,"weight": 1.0 - }] - kwargs = { - 'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'assign_files': [(self.tmp_file, False)], - 'partial_grading': False, - 'language': 'python' - }, - 'test_case_data': test_case_data, + "hook_code": hook_code, "weight": 1.0 + }] + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'assign_files': [(self.tmp_file, False)], + 'partial_grading': False, + 'language': 'python'}, + 'test_case_data': test_case_data, } # When @@ -953,5 +962,6 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): # Then self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_simple_question_types.py b/yaksh/evaluator_tests/test_simple_question_types.py index cbf2abd..dfb82a2 100644 --- a/yaksh/evaluator_tests/test_simple_question_types.py +++ b/yaksh/evaluator_tests/test_simple_question_types.py @@ -1,10 +1,11 @@ import unittest from datetime import datetime, timedelta from django.utils import timezone +from textwrap import dedent import pytz from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, IntegerTestCase, FloatTestCase,\ - StringTestCase, McqTestCase + StringTestCase, McqTestCase, ArrangeTestCase def setUpModule(): @@ -39,15 +40,17 @@ def setUpModule(): duration=30, active=True, attempts_allowed=1, time_between_attempts=0, pass_criteria=0, description='demo quiz 100', - instructions="Demo Instructions" + instructions="Demo Instructions", + creator=user ) question_paper = QuestionPaper.objects.create(quiz=quiz, total_marks=1.0) def tearDownModule(): - User.objects.get(username="demo_user_100").delete() - User.objects.get(username="demo_user_101").delete() + User.objects.filter(username__in=["demo_user_100", "demo_user_101"])\ + .delete() + class IntegerQuestionTestCases(unittest.TestCase): @classmethod @@ -116,11 +119,9 @@ class IntegerQuestionTestCases(unittest.TestCase): # Regrade # Given - self.answer.correct = True - self.answer.marks = 1 - - self.answer.answer = 200 - self.answer.save() + regrade_answer = Answer.objects.get(id=self.answer.id) + regrade_answer.answer = 200 + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -128,6 +129,7 @@ class IntegerQuestionTestCases(unittest.TestCase): # Then self.answer = self.answerpaper.answers.filter(question=self.question1 ).last() + self.assertEqual(self.answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(self.answer.marks, 0) self.assertFalse(self.answer.correct) @@ -153,11 +155,9 @@ class IntegerQuestionTestCases(unittest.TestCase): # Regrade # Given - self.answer.correct = True - self.answer.marks = 1 - - self.answer.answer = 25 - self.answer.save() + regrade_answer = Answer.objects.get(id=self.answer.id) + regrade_answer.answer = 25 + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -165,6 +165,7 @@ class IntegerQuestionTestCases(unittest.TestCase): # Then self.answer = self.answerpaper.answers.filter(question=self.question1 ).last() + self.assertEqual(self.answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(self.answer.marks, 1) self.assertTrue(self.answer.correct) @@ -250,11 +251,9 @@ class StringQuestionTestCases(unittest.TestCase): # Regrade # Given - answer.correct = True - answer.marks = 1 - - answer.answer = "hello, mars!" - answer.save() + regrade_answer = Answer.objects.get(id=answer.id) + regrade_answer.answer = "hello, mars!" + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -262,6 +261,7 @@ class StringQuestionTestCases(unittest.TestCase): # Then answer = self.answerpaper.answers.filter(question=self.question1)\ .last() + self.assertEqual(answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(answer.marks, 0) self.assertFalse(answer.correct) @@ -284,11 +284,9 @@ class StringQuestionTestCases(unittest.TestCase): # Regrade # Given - answer.correct = True - answer.marks = 1 - - answer.answer = "hello, earth!" - answer.save() + regrade_answer = Answer.objects.get(id=answer.id) + regrade_answer.answer = "hello, earth!" + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -296,6 +294,7 @@ class StringQuestionTestCases(unittest.TestCase): # Then answer = self.answerpaper.answers.filter(question=self.question1)\ .last() + self.assertEqual(answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(answer.marks, 1) self.assertTrue(answer.correct) @@ -317,11 +316,9 @@ class StringQuestionTestCases(unittest.TestCase): # Regrade # Given - answer.correct = True - answer.marks = 1 - - answer.answer = "hello, earth!" - answer.save() + regrade_answer = Answer.objects.get(id=answer.id) + regrade_answer.answer = "hello, earth!" + regrade_answer.save() # When details = self.answerpaper.regrade(self.question2.id) @@ -329,6 +326,7 @@ class StringQuestionTestCases(unittest.TestCase): # Then answer = self.answerpaper.answers.filter(question=self.question2)\ .last() + self.assertEqual(answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(answer.marks, 0) self.assertFalse(answer.correct) @@ -351,11 +349,9 @@ class StringQuestionTestCases(unittest.TestCase): # Regrade # Given - answer.correct = True - answer.marks = 1 - - answer.answer = "Hello, EARTH!" - answer.save() + regrade_answer = Answer.objects.get(id=answer.id) + regrade_answer.answer = "Hello, EARTH!" + regrade_answer.save() # When details = self.answerpaper.regrade(self.question2.id) @@ -363,6 +359,7 @@ class StringQuestionTestCases(unittest.TestCase): # Then answer = self.answerpaper.answers.filter(question=self.question2)\ .last() + self.assertEqual(answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(answer.marks, 1) self.assertTrue(answer.correct) @@ -432,13 +429,11 @@ class FloatQuestionTestCases(unittest.TestCase): # Then self.assertTrue(result['success']) - # Regrade + # Regrade with wrong answer # Given - self.answer.correct = True - self.answer.marks = 1 - - self.answer.answer = 0.0 - self.answer.save() + regrade_answer = Answer.objects.get(id=self.answer.id) + regrade_answer.answer = 0.0 + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -446,6 +441,7 @@ class FloatQuestionTestCases(unittest.TestCase): # Then self.answer = self.answerpaper.answers.filter(question=self.question1 ).last() + self.assertEqual(self.answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(self.answer.marks, 0) self.assertFalse(self.answer.correct) @@ -470,11 +466,270 @@ class FloatQuestionTestCases(unittest.TestCase): # Regrade # Given - self.answer.correct = True - self.answer.marks = 1 + regrade_answer = Answer.objects.get(id=self.answer.id) + regrade_answer.answer = 99.9 + regrade_answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertEqual(self.answer, regrade_answer) + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 1) + self.assertTrue(self.answer.correct) +class MCQQuestionTestCases(unittest.TestCase): + @classmethod + def setUpClass(self): + #Creating User + self.user = User.objects.get(username='demo_user_100') + self.user2 = User.objects.get(username='demo_user_101') + self.user_ip = '127.0.0.1' + + #Creating Course + self.course = Course.objects.get(name="Python Course 100") + # Creating Quiz + self.quiz = Quiz.objects.get(description="demo quiz 100") + # Creating Question paper + self.question_paper = QuestionPaper.objects.get(quiz=self.quiz) + self.question_paper.shuffle_testcases = True + self.question_paper.save() + #Creating Question + self.question1 = Question.objects.create(summary='mcq1', points=1, + type='code', user=self.user, + ) + self.question1.language = 'python' + self.question1.type = "mcq" + self.question1.test_case_type = 'Mcqtestcase' + self.question1.description = 'Which option is Correct?' + self.question1.save() + + # For questions + self.mcq_based_testcase_1 = McqTestCase(question=self.question1, + options="Correct", + correct=True, + type='mcqtestcase', + ) + self.mcq_based_testcase_1.save() + + self.mcq_based_testcase_2 = McqTestCase(question=self.question1, + options="Incorrect", + correct=False, + type='mcqtestcase', + ) + self.mcq_based_testcase_2.save() + + self.mcq_based_testcase_3 = McqTestCase(question=self.question1, + options="Incorrect", + correct=False, + type='mcqtestcase', + ) + self.mcq_based_testcase_3.save() + + self.mcq_based_testcase_4 = McqTestCase(question=self.question1, + options="Incorrect", + correct=False, + type='mcqtestcase', + ) + self.mcq_based_testcase_4.save() + + self.question_paper.fixed_questions.add(self.question1) + + self.answerpaper = self.question_paper.make_answerpaper( + user=self.user, ip=self.user_ip, + attempt_num=1, + course_id=self.course.id + ) + + # Answerpaper for user 2 + self.answerpaper2 = self.question_paper.make_answerpaper( + user=self.user2, ip=self.user_ip, + attempt_num=1, + course_id=self.course.id + ) + + @classmethod + def tearDownClass(self): + self.question1.delete() + self.answerpaper.delete() + self.answerpaper2.delete() - self.answer.answer = 99.9 + def test_shuffle_test_cases(self): + # Given + # When + + user_testcase = self.question1.get_ordered_test_cases( + self.answerpaper + ) + order1 = [tc.id for tc in user_testcase] + user2_testcase = self.question1.get_ordered_test_cases( + self.answerpaper2 + ) + order2 = [tc.id for tc in user2_testcase] + self.question_paper.shuffle_testcases = False + self.question_paper.save() + answerpaper3 = self.question_paper.make_answerpaper( + user=self.user2, ip=self.user_ip, + attempt_num=self.answerpaper.attempt_number+1, + course_id=self.course.id + ) + not_ordered_testcase = self.question1.get_ordered_test_cases( + answerpaper3 ) + get_test_cases = self.question1.get_test_cases() + # Then + self.assertNotEqual(order1, order2) + self.assertEqual(get_test_cases, not_ordered_testcase) + + +class ArrangeQuestionTestCases(unittest.TestCase): + @classmethod + def setUpClass(self): + # Creating Course + self.course = Course.objects.get(name="Python Course 100") + # Creating Quiz + self.quiz = Quiz.objects.get(description="demo quiz 100") + # Creating Question paper + self.question_paper = QuestionPaper.objects.get(quiz=self.quiz, + total_marks=1.0) + + #Creating User + self.user = User.objects.get(username='demo_user_100') + #Creating Question + self.question1 = Question.objects.create(summary='arrange1', + points=1.0, + user=self.user + ) + self.question1.language = 'python' + self.question1.type = "arrange" + self.question1.description = "Arrange alphabets in ascending order" + self.question1.test_case_type = 'arrangetestcase' + self.question1.save() + + #Creating answerpaper + + self.answerpaper = AnswerPaper.objects.create(user=self.user, + user_ip='101.0.0.1', + start_time=timezone.now(), + question_paper=self.question_paper, + end_time=timezone.now() + +timedelta(minutes=5), + attempt_number=1, + course=self.course + ) + self.answerpaper.questions.add(self.question1) + self.answerpaper.save() + # For question + self.arrange_testcase_1 = ArrangeTestCase(question=self.question1, + options="A", + type = 'arrangetestcase', + ) + self.arrange_testcase_1.save() + self.testcase_1_id = self.arrange_testcase_1.id + self.arrange_testcase_2 = ArrangeTestCase(question=self.question1, + options="B", + type = 'arrangetestcase', + ) + self.arrange_testcase_2.save() + self.testcase_2_id = self.arrange_testcase_2.id + self.arrange_testcase_3 = ArrangeTestCase(question=self.question1, + options="C", + type = 'arrangetestcase', + ) + self.arrange_testcase_3.save() + self.testcase_3_id = self.arrange_testcase_3.id + @classmethod + def tearDownClass(self): + self.question1.delete() + self.answerpaper.delete() + + def test_validate_regrade_arrange_correct_answer(self): + # Given + arrange_answer = [self.testcase_1_id, + self.testcase_2_id, + self.testcase_3_id, + ] + self.answer = Answer(question=self.question1, + answer=arrange_answer, + ) + self.answer.save() + self.answerpaper.answers.add(self.answer) + + # When + json_data = None + result = self.answerpaper.validate_answer(arrange_answer, + self.question1, + json_data, + ) + # Then + self.assertTrue(result['success']) + + # Regrade with wrong answer + # Given + regrade_answer = Answer.objects.get(id=self.answer.id) + + # Try regrade with wrong data structure + # When + regrade_answer.answer = 1 + regrade_answer.save() + details = self.answerpaper.regrade(self.question1.id) + err_msg = dedent("""\ + User: {0}; Quiz: {1}; Question: {2}. + {3} answer not a list.""".format( + self.user.username, + self.quiz.description, + self.question1.summary, + self.question1.type + ) ) + self.assertFalse(details[0]) + self.assertEqual(details[1], err_msg) + + + # Try regrade with incorrect answer + # When + regrade_answer.answer = [self.testcase_1_id, + self.testcase_3_id, + self.testcase_2_id, + ] + regrade_answer.save() + # Then + details = self.answerpaper.regrade(self.question1.id) + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertEqual(self.answer, regrade_answer) + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 0) + self.assertFalse(self.answer.correct) + + def test_validate_regrade_arrange_incorrect_answer(self): + # Given + arrange_answer = [self.testcase_1_id, + self.testcase_3_id, + self.testcase_2_id, + ] + self.answer = Answer(question=self.question1, + answer=arrange_answer, + ) self.answer.save() + self.answerpaper.answers.add(self.answer) + + # When + json_data = None + result = self.answerpaper.validate_answer(arrange_answer, + self.question1, json_data + ) + + # Then + self.assertFalse(result['success']) + # Regrade with wrong answer + # Given + regrade_answer = Answer.objects.get(id=self.answer.id) + regrade_answer.answer = [self.testcase_1_id, + self.testcase_2_id, + self.testcase_3_id, + ] + regrade_answer.save() # When details = self.answerpaper.regrade(self.question1.id) @@ -482,6 +737,7 @@ class FloatQuestionTestCases(unittest.TestCase): # Then self.answer = self.answerpaper.answers.filter(question=self.question1 ).last() + self.assertEqual(self.answer, regrade_answer) self.assertTrue(details[0]) self.assertEqual(self.answer.marks, 1) self.assertTrue(self.answer.correct) |