diff options
| author | Prabhu Ramachandran | 2016-11-17 12:54:56 +0530 |
|---|---|---|
| committer | GitHub | 2016-11-17 12:54:56 +0530 |
| commit | aa6ed71496c4a36faed9b42104c8426345bdc2e3 (patch) | |
| tree | e5ff410809c60e735cc1a64ce667ba1d4caa6cf5 /yaksh | |
| parent | 7ae8584a4f4d095e005d6239102c0f26611ac006 (diff) | |
| parent | c4a39baa980578b4e35452d7e1978b1bf0b38e13 (diff) | |
| download | online_test-aa6ed71496c4a36faed9b42104c8426345bdc2e3.tar.gz online_test-aa6ed71496c4a36faed9b42104c8426345bdc2e3.tar.bz2 online_test-aa6ed71496c4a36faed9b42104c8426345bdc2e3.zip | |
Merge pull request #152 from ankitjavalkar/partial
Partial Grading
Diffstat (limited to 'yaksh')
24 files changed, 533 insertions, 253 deletions
diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index e4b961c..b5974d2 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -17,6 +17,7 @@ class BashCodeEvaluator(CodeEvaluator): # Private Protocol ########## def setup(self): super(BashCodeEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('submit.sh') self._set_file_as_executable(self.submit_code_path) @@ -27,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, test_case): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -39,41 +40,44 @@ class BashCodeEvaluator(CodeEvaluator): Returns -------- + success - Boolean, indicating if code was executed successfully, correctly + weight - Float, indicating total weight of all successful test cases + error - String, error message if success is false - returns (True, "Correct answer") : If the student script passes all + returns (True, "Correct answer", 1.0) : If the student script passes all test cases/have same output, when compared to the instructor script - returns (False, error_msg): If the student script fails a single + returns (False, error_msg, 0.0): If the student script fails a single test/have dissimilar output, when compared to the instructor script. - Returns (False, error_msg): If mandatory arguments are not files or if + Returns (False, error_msg, 0.0): If mandatory arguments are not files or if the required permissions are not given to the file(s). - """ ref_code_path = test_case + success = False + test_case_weight = 0.0 + get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() get_test_case_path = get_test_case_path.strip() clean_ref_code_path, clean_test_case_path = \ self._set_test_code_file_path(get_ref_path, get_test_case_path) - self.files = [] if file_paths: self.files = copy_files(file_paths) if not isfile(clean_ref_code_path): msg = "No file at %s or Incorrect path" % clean_ref_code_path - return False, msg + return False, msg, 0.0 if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path - return False, msg + return False, msg, 0.0 if not os.access(clean_ref_code_path, os.X_OK): msg = "Script %s is not executable" % clean_ref_code_path - return False, msg + return False, msg, 0.0 if not os.access(self.submit_code_path, os.X_OK): msg = "Script %s is not executable" % self.submit_code_path - return False, msg + return False, msg, 0.0 - success = False user_answer = user_answer.replace("\r", "") self.write_to_submit_code_file(self.submit_code_path, user_answer) @@ -91,19 +95,20 @@ class BashCodeEvaluator(CodeEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - return True, "Correct answer" + test_case_weight = float(weight) if partial_grading else 0.0 + return True, "Correct answer", test_case_weight else: err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr ) - return False, err + return False, err, 0.0 else: if not isfile(clean_test_case_path): msg = "No test case at %s" % clean_test_case_path - return False, msg + return False, msg, 0.0 if not os.access(clean_ref_code_path, os.R_OK): msg = "Test script %s, not readable" % clean_test_case_path - return False, msg + return False, msg, 0.0 # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. @@ -133,10 +138,11 @@ class BashCodeEvaluator(CodeEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - return True, "Correct answer" + test_case_weight = float(weight) if partial_grading else 0.0 + return True, "Correct answer", test_case_weight else: err = ("Error:expected" " {0}, got {1}").format(inst_stdout+inst_stderr, stdnt_stdout+stdnt_stderr ) - return False, err + return False, err, 0.0 diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index a7ea1a4..1dd9fd5 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -14,6 +14,7 @@ class BashStdioEvaluator(StdIOEvaluator): def setup(self): super(BashStdioEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('Test.sh') def teardown(self): @@ -22,8 +23,7 @@ class BashStdioEvaluator(StdIOEvaluator): delete_files(self.files) super(BashStdioEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, expected_input, expected_output): - self.files = [] + def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): if file_paths: self.files = copy_files(file_paths) if not isfile(self.submit_code_path): @@ -33,8 +33,11 @@ class BashStdioEvaluator(StdIOEvaluator): user_answer = user_answer.replace("\r", "") self.write_to_submit_code_file(self.submit_code_path, user_answer) - def check_code(self, user_answer, file_paths, expected_input, expected_output): + def check_code(self, user_answer, file_paths, partial_grading, + expected_input, expected_output, weight): success = False + test_case_weight = 0.0 + expected_input = str(expected_input).replace('\r', '') proc = subprocess.Popen("bash ./Test.sh", shell=True, @@ -46,4 +49,5 @@ class BashStdioEvaluator(StdIOEvaluator): expected_input, expected_output ) - return success, err + test_case_weight = float(weight) if partial_grading and success else 0.0 + return success, err, test_case_weight diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 79f616d..afe18c3 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -82,14 +82,14 @@ class CodeEvaluator(object): Returns ------- - A tuple: (success, error message). + A tuple: (success, error message, weight). """ self.setup() - success, err = self.safe_evaluate(**kwargs) + success, error, weight = self.safe_evaluate(**kwargs) self.teardown() - result = {'success': success, 'error': err} + result = {'success': success, 'error': error, 'weight': weight} return result # Private Protocol ########## @@ -99,7 +99,7 @@ class CodeEvaluator(object): os.makedirs(self.in_dir) self._change_dir(self.in_dir) - def safe_evaluate(self, user_answer, test_case_data, file_paths=None): + def safe_evaluate(self, user_answer, partial_grading, test_case_data, file_paths=None): """ Handles code evaluation along with compilation, signal handling and Exception handling @@ -108,32 +108,44 @@ class CodeEvaluator(object): # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False + test_case_success_status = [False] * len(test_case_data) + error = "" + weight = 0.0 # Do whatever testing needed. try: - for test_case in test_case_data: - success = False + for idx, test_case in enumerate(test_case_data): + test_case_success = False self.compile_code(user_answer, file_paths, **test_case) - success, err = self.check_code(user_answer, file_paths, **test_case) - if not success: - break + test_case_success, err, test_case_weight = self.check_code(user_answer, + file_paths, + partial_grading, + **test_case + ) + if test_case_success: + weight += test_case_weight + + error += err + "\n" + test_case_success_status[idx] = test_case_success + + success = all(test_case_success_status) except TimeoutException: - err = self.timeout_msg + error = self.timeout_msg except OSError: msg = traceback.format_exc(limit=0) - err = "Error: {0}".format(msg) + error = "Error: {0}".format(msg) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) if len(tb_list) > 2: del tb_list[1:3] - err = "Error: {0}".format("".join(tb_list)) + error = "Error: {0}".format("".join(tb_list)) finally: # Set back any original signal handler. set_original_signal_handler(prev_handler) - return success, err + return success, error, weight def teardown(self): # Cancel the signal diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 5380dea..716a522 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -16,6 +16,7 @@ class CppCodeEvaluator(CodeEvaluator): """Tests the C code obtained from Code Server""" def setup(self): super(CppCodeEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('submit.c') self.compiled_user_answer = None self.compiled_test_code = None @@ -49,8 +50,7 @@ class CppCodeEvaluator(CodeEvaluator): ref_output_path) return compile_command, compile_main - def compile_code(self, user_answer, file_paths, test_case): - self.files = [] + def compile_code(self, user_answer, file_paths, test_case, weight): if self.compiled_user_answer and self.compiled_test_code: return None else: @@ -89,7 +89,7 @@ class CppCodeEvaluator(CodeEvaluator): return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, test_case): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -109,6 +109,8 @@ class CppCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ success = False + test_case_weight = 0.0 + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -127,6 +129,7 @@ class CppCodeEvaluator(CodeEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" + test_case_weight = float(weight) if partial_grading else 0.0 else: err = "{0} \n {1}".format(stdout, stderr) else: @@ -152,4 +155,4 @@ class CppCodeEvaluator(CodeEvaluator): except: err = "{0} \n {1}".format(err, stdnt_stderr) - return success, err + return success, err, test_case_weight diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 9d2b969..00fad92 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -14,6 +14,7 @@ class CppStdioEvaluator(StdIOEvaluator): def setup(self): super(CppStdioEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('main.c') def teardown(self): @@ -34,9 +35,7 @@ class CppStdioEvaluator(StdIOEvaluator): ref_output_path) return compile_command, compile_main - def compile_code(self, user_answer, file_paths, expected_input, expected_output): - - self.files = [] + def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): if file_paths: self.files = copy_files(file_paths) if not isfile(self.submit_code_path): @@ -62,8 +61,11 @@ class CppStdioEvaluator(StdIOEvaluator): ) return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, expected_input, expected_output): + def check_code(self, user_answer, file_paths, partial_grading, + expected_input, expected_output, weight): success = False + test_case_weight = 0.0 + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) if stdnt_stderr == '': @@ -104,4 +106,5 @@ class CppStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - return success, err + test_case_weight = float(weight) if partial_grading and success else 0.0 + return success, err, test_case_weight diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 66ade19..99e5122 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -13,12 +13,12 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) - tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ - {"test_case": "bash_files/sample.sh,bash_files/sample.args"} + {"test_case": "bash_files/sample.sh,bash_files/sample.args", + "weight": 0.0 + } ] - tmp_in_dir_path = tempfile.mkdtemp() - self.in_dir = tmp_in_dir_path + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) @@ -33,19 +33,21 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") def test_error(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -57,7 +59,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -68,20 +71,24 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ - {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"} + {"test_case": "bash_files/sample1.sh,bash_files/sample1.args", + "weight": 0.0 + } ] user_answer = ("#!/bin/bash\ncat $1") get_class = BashCodeEvaluator() kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + self.assertEqual(result.get("error"), "Correct answer\n") class BashStdioEvaluationTestCases(unittest.TestCase): def setUp(self): + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) @@ -93,13 +100,17 @@ class BashStdioEvaluationTestCases(unittest.TestCase): echo -n `expr $A + $B` """ ) - test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'weight': 0.0 + }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): @@ -112,14 +123,16 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', - 'expected_input': '1,2,3\n4,5,6\n7,8,9' + 'expected_input': '1,2,3\n4,5,6\n7,8,9', + 'weight': 0.0 }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): @@ -129,11 +142,15 @@ class BashStdioEvaluationTestCases(unittest.TestCase): echo -n `expr $A - $B` """ ) - test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'weight': 0.0 + }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -146,14 +163,16 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '10', - 'expected_input': '' + 'expected_input': '', + 'weight': 0.0 }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) if __name__ == '__main__': diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index c990436..d5193d3 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -14,7 +14,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() - self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] + self.test_case_data = [{"test_case": "c_cpp_files/main.cpp", + "weight": 0.0 + }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" @@ -29,17 +31,19 @@ class CAssertionEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a-b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -52,7 +56,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -63,7 +68,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -73,7 +79,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] - self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}] + self.test_case_data = [{"test_case": "c_cpp_files/file_data.c", + "weight": 0.0 + }] user_answer = dedent(""" #include<stdio.h> char ans() @@ -88,18 +96,21 @@ class CAssertionEvaluationTestCases(unittest.TestCase): """) get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': False, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") class CppStdioEvaluationTestCases(unittest.TestCase): - def setUp(self): - self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] - self.in_dir = os.getcwd() + self.test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'weight': 0.0 + }] + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) @@ -114,15 +125,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'weight': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -134,15 +148,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'abc', - 'expected_input': 'abc'}] + 'expected_input': 'abc', + 'weight': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -152,10 +169,11 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): @@ -167,8 +185,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -184,8 +203,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -199,15 +219,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'weight': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -216,10 +239,11 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_correct_answer(self): @@ -233,15 +257,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'weight': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -254,15 +281,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_string_input(self): self.test_case_data = [{'expected_output': 'abc', - 'expected_input': 'abc'}] + 'expected_input': 'abc', + 'weight': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -273,10 +303,11 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_incorrect_answer(self): @@ -289,8 +320,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -307,8 +339,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -323,15 +356,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_cpp_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'weight': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -341,10 +377,11 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) if __name__ == '__main__': diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index e375bdb..f7ecd97 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -16,7 +16,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ - {"test_case": "java_files/main_square.java"} + {"test_case": "java_files/main_square.java", + "weight": 0.0 + } ] self.in_dir = tmp_in_dir_path evaluator.SERVER_TIMEOUT = 9 @@ -32,18 +34,20 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -57,7 +61,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_error(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -68,7 +73,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -79,7 +85,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [ - {"test_case": "java_files/read_file.java"} + {"test_case": "java_files/read_file.java", + "weight": 0.0 + } ] user_answer = dedent(""" import java.io.BufferedReader; @@ -101,12 +109,13 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): """) get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': True, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + self.assertEqual(result.get("error"), "Correct answer\n") class JavaStdioEvaluationTestCases(unittest.TestCase): @@ -116,7 +125,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path self.test_case_data = [{'expected_output': '11', - 'expected_input': '5\n6'}] + 'expected_input': '5\n6', + 'weight': 0.0 + }] evaluator.SERVER_TIMEOUT = 4 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -139,16 +150,19 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'weight': 0.0 + }] user_answer = dedent(""" import java.util.Scanner; class Test @@ -161,10 +175,11 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): @@ -180,8 +195,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -197,8 +213,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -214,15 +231,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'weight': 0.0 + }] user_answer = dedent(""" class Test {public static void main(String[] args){ @@ -232,15 +252,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'HelloWorld', - 'expected_input': 'Hello\nWorld'}] + 'expected_input': 'Hello\nWorld', + 'weight': 0.0 + }] user_answer = dedent(""" import java.util.Scanner; class Test @@ -252,16 +275,19 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_file_based_stdout(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [{'expected_output': '2', - 'expected_input': ''}] + 'expected_input': '', + 'weight': 0.0 + }] user_answer = dedent(""" import java.io.BufferedReader; import java.io.FileReader; @@ -282,12 +308,13 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): """) get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': True, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + self.assertEqual(result.get("error"), "Correct answer\n") if __name__ == '__main__': diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 45cc40d..9796fa2 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -17,9 +17,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, + self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, + {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, + {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -35,7 +35,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): user_answer = "def add(a,b):\n\treturn a + b" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -44,14 +45,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertTrue(result.get('success')) - self.assertEqual(result.get('error'), "Correct answer") + self.assertIn("Correct answer", result.get('error')) def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -60,16 +62,50 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), - "AssertionError in: assert(add(1,2)==3)" - ) + self.assertIn('AssertionError in: assert(add(1,2)==3)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) + + def test_partial_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" + test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, + {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, + {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} + ] + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data, + 'file_paths': self.file_paths, + 'partial_grading': True + } + + # When + evaluator = PythonAssertionEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assertEqual(result.get('weight'), 2.0) + self.assertIn('AssertionError in: assert(add(-1,2)==1)', + result.get('error') + ) + self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', + result.get('error') + ) def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -96,7 +132,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -125,7 +162,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False + } # When @@ -150,7 +189,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -176,7 +216,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -202,7 +243,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -231,7 +273,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -247,7 +290,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')"}] + self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): @@ -256,7 +299,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -264,7 +308,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result = evaluator.evaluate(**kwargs) # Then - self.assertEqual(result.get('error'), "Correct answer") + self.assertIn("Correct answer", result.get('error')) self.assertTrue(result.get('success')) def test_single_testcase_error(self): @@ -272,7 +316,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'} + test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + "weight": 0.0 + } ] syntax_error_msg = ["Traceback", "call", @@ -284,7 +330,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -304,8 +351,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'}, - {"test_case": 's="abbb"\nassert palindrome(S)==False'} + test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + "weight": 0.0 + }, + {"test_case": 's="abbb"\nassert palindrome(S)==False', + "weight": 0.0 + } ] name_error_msg = ["Traceback", "call", @@ -317,7 +368,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -341,7 +393,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_integer(self): # Given self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" + "expected_output": "3", + "weight": 0.0 }] user_answer = dedent(""" a = int(input()) @@ -350,7 +403,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -364,7 +418,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", - "expected_output": "[1, 2, 3, 5, 6, 7]" + "expected_output": "[1, 2, 3, 5, 6, 7]", + "weight": 0.0 }] user_answer = dedent(""" from six.moves import input @@ -376,7 +431,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -390,7 +446,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), - "expected_output": "2" + "expected_output": "2", + "weight": 0.0 }] user_answer = dedent(""" from six.moves import input @@ -400,7 +457,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -414,7 +472,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_incorrect_answer_integer(self): # Given self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" + "expected_output": "3", + "weight": 0.0 }] user_answer = dedent(""" a = int(input()) @@ -424,6 +483,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): ) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -436,7 +496,10 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given - self.test_case_data = [{"expected_input": "", "expected_output": "2"}] + self.test_case_data = [{"expected_input": "", + "expected_output": "2", + "weight": 0.0 + }] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" @@ -447,7 +510,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): ) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -455,20 +519,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): result = evaluator.evaluate(**kwargs) # Then - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_infinite_loop(self): # Given test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" - }] + "expected_output": "3", + "weight": 0.0 + }] timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data + 'test_case_data': test_case_data, + 'partial_grading': False } # When diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index b366480..c30f652 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -11,7 +11,9 @@ from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): tmp_in_dir_path = tempfile.mkdtemp() - self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] + self.test_case_data = [{"test_case": "scilab_files/test_add.sce", + "weight": 0.0 + }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" @@ -25,19 +27,21 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_error(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\ndis(\tendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -50,7 +54,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a-b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -64,7 +69,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index 1ce1c0e..d87e6e3 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -16,6 +16,7 @@ class JavaCodeEvaluator(CodeEvaluator): """Tests the Java code obtained from Code Server""" def setup(self): super(JavaCodeEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('Test.java') self.compiled_user_answer = None self.compiled_test_code = None @@ -46,8 +47,7 @@ class JavaCodeEvaluator(CodeEvaluator): output_path = "{0}{1}.class".format(directory, file_name) return output_path - def compile_code(self, user_answer, file_paths, test_case): - self.files = [] + def compile_code(self, user_answer, file_paths, test_case, weight): if self.compiled_user_answer and self.compiled_test_code: return None else: @@ -96,7 +96,7 @@ class JavaCodeEvaluator(CodeEvaluator): return self.compiled_user_answer, self.compiled_test_code - def check_code(self, user_answer, file_paths, test_case): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -117,6 +117,8 @@ class JavaCodeEvaluator(CodeEvaluator): """ success = False + test_case_weight = 0.0 + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -134,6 +136,7 @@ class JavaCodeEvaluator(CodeEvaluator): proc, stdout, stderr = ret if proc.returncode == 0: success, err = True, "Correct answer" + test_case_weight = float(weight) if partial_grading else 0.0 else: err = stdout + "\n" + stderr else: @@ -158,4 +161,4 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - return success, err + return success, err, test_case_weight diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index bc9cf80..88d4c88 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -14,6 +14,7 @@ class JavaStdioEvaluator(StdIOEvaluator): def setup(self): super(JavaStdioEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('Test.java') def teardown(self): @@ -30,8 +31,7 @@ class JavaStdioEvaluator(StdIOEvaluator): compile_command = 'javac {0}'.format(self.submit_code_path) return compile_command - def compile_code(self, user_answer, file_paths, expected_input, expected_output): - self.files = [] + def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path return False, msg @@ -50,8 +50,10 @@ class JavaStdioEvaluator(StdIOEvaluator): ) return self.compiled_user_answer - def check_code(self, user_answer, file_paths, expected_input, expected_output): + def check_code(self, user_answer, file_paths, partial_grading, + expected_input, expected_output, weight): success = False + test_case_weight = 0.0 proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) if stdnt_stderr == '' or "error" not in stdnt_stderr: @@ -77,4 +79,5 @@ class JavaStdioEvaluator(StdIOEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - return success, err + test_case_weight = float(weight) if partial_grading and success else 0.0 + return success, err, test_case_weight diff --git a/yaksh/models.py b/yaksh/models.py index 7f9eead..8907df0 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -245,6 +245,9 @@ class Question(models.Model): # user for particular question user = models.ForeignKey(User, related_name="user") + # Does this question allow partial grading + partial_grading = models.BooleanField(default=False) + def consolidate_answer_data(self, user_answer): question_data = {} test_case_data = [] @@ -257,6 +260,7 @@ class Question(models.Model): question_data['test_case_data'] = test_case_data question_data['user_answer'] = user_answer + question_data['partial_grading'] = self.partial_grading files = FileUpload.objects.filter(question=self) if files: question_data['file_paths'] = [(file.file.path, file.extract) @@ -321,6 +325,13 @@ class Question(models.Model): return test_case + def get_maximum_test_case_weight(self, **kwargs): + max_weight = 0.0 + for test_case in self.get_test_cases(): + max_weight += test_case.weight + + return max_weight + def _add_and_get_files(self, zip_file): files = FileUpload.objects.filter(question=self) files_list = [] @@ -937,11 +948,13 @@ class AnswerPaper(models.Model): def _update_marks_obtained(self): """Updates the total marks earned by student for this paper.""" - marks = sum([x.marks for x in self.answers.filter(marks__gt=0.0)]) - if not marks: - self.marks_obtained = 0 - else: - self.marks_obtained = marks + marks = 0 + for question in self.questions.all(): + marks_list = [a.marks for a in self.answers.filter(question=question)] + max_marks = max(marks_list) if marks_list else 0.0 + marks += max_marks + self.marks_obtained = marks + def _update_percent(self): """Updates the percent gained by the student for this paper.""" @@ -1023,7 +1036,7 @@ class AnswerPaper(models.Model): For code questions success is True only if the answer is correct. """ - result = {'success': True, 'error': 'Incorrect answer'} + result = {'success': True, 'error': 'Incorrect answer', 'weight': 0.0} correct = False if user_answer is not None: if question.type == 'mcq': @@ -1071,11 +1084,18 @@ class AnswerPaper(models.Model): json_data = question.consolidate_answer_data(answer) \ if question.type == 'code' else None correct, result = self.validate_answer(answer, question, json_data) - user_answer.marks = question.points if correct else 0.0 user_answer.correct = correct user_answer.error = result.get('error') + if correct: + user_answer.marks = (question.points * result['weight'] / + question.get_maximum_test_case_weight()) \ + if question.partial_grading and question.type == 'code' else question.points + else: + user_answer.marks = (question.points * result['weight'] / + question.get_maximum_test_case_weight()) \ + if question.partial_grading and question.type == 'code' else 0 user_answer.save() - self.update_marks('complete') + self.update_marks('completed') return True, msg def __str__(self): @@ -1098,9 +1118,11 @@ class TestCase(models.Model): class StandardTestCase(TestCase): test_case = models.TextField(blank=True) + weight = models.FloatField(default=0.0) def get_field_value(self): - return {"test_case": self.test_case} + return {"test_case": self.test_case, + "weight": self.weight} def __str__(self): return u'Question: {0} | Test Case: {1}'.format(self.question, @@ -1111,10 +1133,12 @@ class StandardTestCase(TestCase): class StdioBasedTestCase(TestCase): expected_input = models.TextField(blank=True) expected_output = models.TextField() + weight = models.IntegerField(default=0.0) def get_field_value(self): return {"expected_output": self.expected_output, - "expected_input": self.expected_input} + "expected_input": self.expected_input, + "weight": self.weight} def __str__(self): return u'Question: {0} | Exp. Output: {1} | Exp. Input: {2}'.format(self.question, diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index dd1c041..275244a 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -17,6 +17,7 @@ class PythonAssertionEvaluator(CodeEvaluator): def setup(self): super(PythonAssertionEvaluator, self).setup() self.exec_scope = None + self.files = [] def teardown(self): # Delete the created file. @@ -24,8 +25,7 @@ class PythonAssertionEvaluator(CodeEvaluator): delete_files(self.files) super(PythonAssertionEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, test_case): - self.files = [] + def compile_code(self, user_answer, file_paths, test_case, weight): if file_paths: self.files = copy_files(file_paths) if self.exec_scope: @@ -36,8 +36,29 @@ class PythonAssertionEvaluator(CodeEvaluator): exec(submitted, self.exec_scope) return self.exec_scope - def check_code(self, user_answer, file_paths, test_case): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): + """ Function validates user answer by running an assertion based test case + against it + + Returns + -------- + Returns a tuple (success, error, test_case_weight) + + success - Boolean, indicating if code was executed successfully, correctly + weight - Float, indicating total weight of all successful test cases + error - String, error message if success is false + + returns (True, "Correct answer", 1.0) : If the student script passes all + test cases/have same output, when compared to the instructor script + + returns (False, error_msg, 0.0): If the student script fails a single + test/have dissimilar output, when compared to the instructor script. + + Returns (False, error_msg, 0.0): If mandatory arguments are not files or if + the required permissions are not given to the file(s). + """ success = False + test_case_weight = 0.0 try: tb = None _tests = compile(test_case, '<string>', mode='exec') @@ -47,11 +68,13 @@ class PythonAssertionEvaluator(CodeEvaluator): info = traceback.extract_tb(tb) fname, lineno, func, text = info[-1] text = str(test_case).splitlines()[lineno-1] - err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + err = ("-----\nExpected Test Case:\n{0}\n" + "Error - {1} {2} in: {3}\n-----").format(test_case, type.__name__, str(value), text) except Exception: raise # Exception will be caught in CodeEvaluator. else: success = True - err = 'Correct answer' + err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(test_case) + test_case_weight = float(weight) if partial_grading else 0.0 del tb - return success, err + return success, err, test_case_weight diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index cbbbfd6..1506685 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -31,6 +31,10 @@ def redirect_stdout(): class PythonStdioEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" + def setup(self): + super(PythonStdioEvaluator, self).setup() + self.files = [] + def teardown(self): # Delete the created file. if self.files: @@ -38,8 +42,7 @@ class PythonStdioEvaluator(CodeEvaluator): super(PythonStdioEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, expected_input, expected_output): - self.files = [] + def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight): if file_paths: self.files = copy_files(file_paths) submitted = compile(user_answer, '<string>', mode='exec') @@ -54,13 +57,16 @@ class PythonStdioEvaluator(CodeEvaluator): self.output_value = output_buffer.getvalue().rstrip("\n") return self.output_value - def check_code(self, user_answer, file_paths, expected_input, expected_output): + def check_code(self, user_answer, file_paths, partial_grading, expected_input, + expected_output, weight): success = False + test_case_weight = 0.0 tb = None if self.output_value == expected_output: success = True err = "Correct answer" + test_case_weight = weight else: success = False err = dedent(""" @@ -68,10 +74,10 @@ class PythonStdioEvaluator(CodeEvaluator): Given input - {0} Expected output - {1} Your output - {2} - """ - .format(expected_input, - expected_output, self.output_value - ) - ) + """.format(expected_input, + expected_output, + self.output_value + ) + ) del tb - return success, err + return success, err, test_case_weight diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 915491c..3c2d44c 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -16,6 +16,7 @@ class ScilabCodeEvaluator(CodeEvaluator): """Tests the Scilab code obtained from Code Server""" def setup(self): super(ScilabCodeEvaluator, self).setup() + self.files = [] self.submit_code_path = \ self.create_submit_code_file('function.sci') @@ -26,8 +27,7 @@ class ScilabCodeEvaluator(CodeEvaluator): delete_files(self.files) super(ScilabCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, test_case): - self.files = [] + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): if file_paths: self.files = copy_files(file_paths) ref_code_path = test_case @@ -37,6 +37,8 @@ class ScilabCodeEvaluator(CodeEvaluator): self._remove_scilab_exit(user_answer.lstrip()) success = False + test_case_weight = 0.0 + self.write_to_submit_code_file(self.submit_code_path, user_answer) # Throw message if there are commmands that terminates scilab add_err = "" @@ -63,11 +65,12 @@ class ScilabCodeEvaluator(CodeEvaluator): stdout = self._strip_output(stdout) if proc.returncode == 5: success, err = True, "Correct answer" + test_case_weight = float(weight) if partial_grading else 0.0 else: err = add_err + stdout else: err = add_err + stderr - return success, err + return success, err, test_case_weight def _remove_scilab_exit(self, string): """ diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index c0d53f8..9822333 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -24,6 +24,7 @@ <tr><td>Description: <td>{{ form.description}} {{form.description.errors}} <tr><td>Tags: <td>{{ form.tags }} <tr><td>Snippet: <td>{{ form.snippet }} + <tr><td>Partial Grading: <td>{{ form.partial_grading }} <tr><td> Test Case Type: <td> {{ form.test_case_type }}{{ form.test_case_type.errors }} <tr><td> File: <td> {{ upload_form.file_field }}{{ upload_form.file_field.errors }} {% if uploaded_files %}<br><b>Uploaded files:</b><br>Check the box to delete or extract files<br> diff --git a/yaksh/templates/yaksh/grade_user.html b/yaksh/templates/yaksh/grade_user.html index ced3ca2..38f31ca 100644 --- a/yaksh/templates/yaksh/grade_user.html +++ b/yaksh/templates/yaksh/grade_user.html @@ -157,7 +157,7 @@ Status : <b style="color: green;"> Passed </b><br/> <h5>Student answer: </h5> {% for answer in answers %} {% if not answer.skipped %} - {% if "Correct answer" in answer.error %} + {% if answer.correct %} <div class="panel panel-success"> {% else %} <div class="panel panel-danger"> diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html index a8de448..18b6388 100644 --- a/yaksh/templates/yaksh/question.html +++ b/yaksh/templates/yaksh/question.html @@ -163,9 +163,11 @@ function call_skip(url) <div class="panel-footer"> {% if question.type == "code" %} {% if error_message %} - <p> Output Message</p> + <p><h4> Output Message </h4></p> <div class="alert alert-danger" role="alert"> + <textarea class="output" id="output" rows=10 cols=155> {{ error_message }} + </textarea> </div> {% endif %} {% endif %} diff --git a/yaksh/templates/yaksh/user_data.html b/yaksh/templates/yaksh/user_data.html index 0a7e4aa..378e7fd 100644 --- a/yaksh/templates/yaksh/user_data.html +++ b/yaksh/templates/yaksh/user_data.html @@ -94,7 +94,7 @@ User IP address: {{ paper.user_ip }} <h5>Student answer: </h5> {% for answer in answers %} {% if not answer.skipped %} - {% if "Correct answer" in answer.error %} + {% if answer.correct %} <div class="panel panel-success"> {% else %} <div class="panel panel-danger"> diff --git a/yaksh/test_models.py b/yaksh/test_models.py index 019a339..e7f3016 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -141,7 +141,9 @@ class QuestionTestCases(unittest.TestCase): ) self.upload_test_case.save() self.user_answer = "demo_answer" - self.test_case_upload_data = [{"test_case": "assert fact(3)==6"}] + self.test_case_upload_data = [{"test_case": "assert fact(3)==6", + "weight": 0.0 + }] questions_data = [{"snippet": "def fact()", "active": True, "points": 1.0, "description": "factorial of a no", @@ -877,7 +879,9 @@ class TestCaseTestCases(unittest.TestCase): self.stdout_based_testcase.save() answer_data = {"user_answer": "demo_answer", "test_case_data": [ - {"test_case": "assert myfunc(12, 13) == 15"} + {"test_case": "assert myfunc(12, 13) == 15", + "weight": 0.0 + } ] } self.answer_data_json = json.dumps(answer_data) diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 8835110..7efd20b 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -35,10 +35,14 @@ class TestCodeServer(unittest.TestCase): def setUp(self): self.code_server = CodeServerProxy() - def test_inifinite_loop(self): + def test_infinite_loop(self): # Given testdata = {'user_answer': 'while True: pass', - 'test_case_data': [{'test_case':'assert 1==2'}]} + 'partial_grading': False, + 'test_case_data': [{'test_case':'assert 1==2', + 'weight': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -53,7 +57,11 @@ class TestCodeServer(unittest.TestCase): def test_correct_answer(self): # Given testdata = {'user_answer': 'def f(): return 1', - 'test_case_data': [{'test_case':'assert f() == 1'}]} + 'partial_grading': False, + 'test_case_data': [{'test_case':'assert f() == 1', + 'weight': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -63,12 +71,16 @@ class TestCodeServer(unittest.TestCase): # Then data = json.loads(result) self.assertTrue(data['success']) - self.assertEqual(data['error'], 'Correct answer') + self.assertIn('Correct answer', data['error']) def test_wrong_answer(self): # Given testdata = {'user_answer': 'def f(): return 1', - 'test_case_data': [{'test_case':'assert f() == 2'}]} + 'partial_grading': False, + 'test_case_data': [{'test_case':'assert f() == 2', + 'weight': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -87,7 +99,11 @@ class TestCodeServer(unittest.TestCase): def run_code(): """Run an infinite loop.""" testdata = {'user_answer': 'while True: pass', - 'test_case_data': [{'test_case':'assert 1==2'}]} + 'partial_grading': False, + 'test_case_data': [{'test_case':'assert 1==2', + 'weight': 0.0 + } + ]} result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) diff --git a/yaksh/views.py b/yaksh/views.py index 1afcef7..c3d743b 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -517,11 +517,16 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): if question.type == 'code' else None correct, result = paper.validate_answer(user_answer, question, json_data) if correct: + new_answer.marks = (question.points * result['weight'] / + question.get_maximum_test_case_weight()) \ + if question.partial_grading and question.type == 'code' else question.points new_answer.correct = correct - new_answer.marks = question.points new_answer.error = result.get('error') else: new_answer.error = result.get('error') + new_answer.marks = (question.points * result['weight'] / + question.get_maximum_test_case_weight()) \ + if question.partial_grading and question.type == 'code' else 0 new_answer.save() paper.update_marks('inprogress') paper.set_end_time(timezone.now()) diff --git a/yaksh/xmlrpc_clients.py b/yaksh/xmlrpc_clients.py index 4da70dd..bb8260d 100644 --- a/yaksh/xmlrpc_clients.py +++ b/yaksh/xmlrpc_clients.py @@ -55,14 +55,21 @@ class CodeServerProxy(object): Returns ------- - A json string of a dict: {success: success, err: error message}. + A json string of a dict containing: + {"success": success, "weight": weight, "error": error message} + + success - Boolean, indicating if code was executed successfully, correctly + weight - Float, indicating total weight of all successful test cases + error - String, error message if success is false """ try: server = self._get_server() result = server.check_code(language, test_case_type, json_data, user_dir) except ConnectionError: - result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'}) + result = json.dumps({'success': False, + 'weight': 0.0, + 'error': 'Unable to connect to any code servers!'}) return result def _get_server(self): |