From 0bfa58d8705fa08b45a208a4cec98dd267799f8a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 3 Nov 2016 11:47:50 +0530 Subject: Add partial grading to multiple evaluators --- yaksh/bash_code_evaluator.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index e4b961c..7dc0f0f 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -17,6 +17,7 @@ class BashCodeEvaluator(CodeEvaluator): # Private Protocol ########## def setup(self): super(BashCodeEvaluator, self).setup() + self.files = [] self.submit_code_path = self.create_submit_code_file('submit.sh') self._set_file_as_executable(self.submit_code_path) @@ -27,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, test_case): + def check_code(self, file_paths, partial_grading, test_case, marks): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -57,7 +58,6 @@ class BashCodeEvaluator(CodeEvaluator): clean_ref_code_path, clean_test_case_path = \ self._set_test_code_file_path(get_ref_path, get_test_case_path) - self.files = [] if file_paths: self.files = copy_files(file_paths) if not isfile(clean_ref_code_path): @@ -74,6 +74,8 @@ class BashCodeEvaluator(CodeEvaluator): return False, msg success = False + test_case_marks = 0.0 + user_answer = user_answer.replace("\r", "") self.write_to_submit_code_file(self.submit_code_path, user_answer) @@ -91,19 +93,20 @@ class BashCodeEvaluator(CodeEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - return True, "Correct answer" + test_case_marks = float(marks) if partial_grading else 0.0 + return True, "Correct answer", test_case_marks else: err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr ) - return False, err + return False, err, test_case_marks else: if not isfile(clean_test_case_path): msg = "No test case at %s" % clean_test_case_path - return False, msg + return False, msg, test_case_marks if not os.access(clean_ref_code_path, os.R_OK): msg = "Test script %s, not readable" % clean_test_case_path - return False, msg + return False, msg, test_case_marks # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. @@ -133,10 +136,11 @@ class BashCodeEvaluator(CodeEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - return True, "Correct answer" + test_case_marks = float(marks) if partial_grading else 0.0 + return True, "Correct answer", test_case_marks else: err = ("Error:expected" " {0}, got {1}").format(inst_stdout+inst_stderr, stdnt_stdout+stdnt_stderr ) - return False, err + return False, err, test_case_marks -- cgit From 576c92b3b7d8989bf6bb99316d096f48f5b4cc24 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 4 Nov 2016 18:07:35 +0530 Subject: Fix test cases for partial grading feature --- yaksh/bash_code_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 7dc0f0f..4ca5445 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -28,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, file_paths, partial_grading, test_case, marks): + def check_code(self, user_answer, file_paths, partial_grading, test_case, marks): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to -- cgit From a09df642d4f3623ee517aaed2eac1372ebacc0e0 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 10 Nov 2016 12:36:13 +0530 Subject: Add point based weightage for partial grading instead of percentage based partial grading --- yaksh/bash_code_evaluator.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 4ca5445..978586f 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -28,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, partial_grading, test_case, marks): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weightage): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -74,7 +74,7 @@ class BashCodeEvaluator(CodeEvaluator): return False, msg success = False - test_case_marks = 0.0 + test_case_weightage = 0.0 user_answer = user_answer.replace("\r", "") self.write_to_submit_code_file(self.submit_code_path, user_answer) @@ -93,20 +93,20 @@ class BashCodeEvaluator(CodeEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - test_case_marks = float(marks) if partial_grading else 0.0 - return True, "Correct answer", test_case_marks + test_case_weightage = float(weightage) if partial_grading else 0.0 + return True, "Correct answer", test_case_weightage else: err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr ) - return False, err, test_case_marks + return False, err, test_case_weightage else: if not isfile(clean_test_case_path): msg = "No test case at %s" % clean_test_case_path - return False, msg, test_case_marks + return False, msg, test_case_weightage if not os.access(clean_ref_code_path, os.R_OK): msg = "Test script %s, not readable" % clean_test_case_path - return False, msg, test_case_marks + return False, msg, test_case_weightage # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. @@ -136,11 +136,11 @@ class BashCodeEvaluator(CodeEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - test_case_marks = float(marks) if partial_grading else 0.0 - return True, "Correct answer", test_case_marks + test_case_weightage = float(weightage) if partial_grading else 0.0 + return True, "Correct answer", test_case_weightage else: err = ("Error:expected" " {0}, got {1}").format(inst_stdout+inst_stderr, stdnt_stdout+stdnt_stderr ) - return False, err, test_case_marks + return False, err, test_case_weightage -- cgit From 232c4480d47b75fcdb523e5ebb601959c56e40dc Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 16 Nov 2016 10:58:25 +0530 Subject: Fix docstrings, Fix return values of bash evaluator --- yaksh/bash_code_evaluator.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 978586f..dd4445c 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -49,9 +49,11 @@ class BashCodeEvaluator(CodeEvaluator): Returns (False, error_msg): If mandatory arguments are not files or if the required permissions are not given to the file(s). - """ ref_code_path = test_case + success = False + test_case_weightage = 0.0 + get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() get_test_case_path = get_test_case_path.strip() @@ -62,19 +64,16 @@ class BashCodeEvaluator(CodeEvaluator): self.files = copy_files(file_paths) if not isfile(clean_ref_code_path): msg = "No file at %s or Incorrect path" % clean_ref_code_path - return False, msg + return False, msg, 0.0 if not isfile(self.submit_code_path): msg = "No file at %s or Incorrect path" % self.submit_code_path - return False, msg + return False, msg, 0.0 if not os.access(clean_ref_code_path, os.X_OK): msg = "Script %s is not executable" % clean_ref_code_path - return False, msg + return False, msg, 0.0 if not os.access(self.submit_code_path, os.X_OK): msg = "Script %s is not executable" % self.submit_code_path - return False, msg - - success = False - test_case_weightage = 0.0 + return False, msg, 0.0 user_answer = user_answer.replace("\r", "") self.write_to_submit_code_file(self.submit_code_path, user_answer) @@ -99,14 +98,14 @@ class BashCodeEvaluator(CodeEvaluator): err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr ) - return False, err, test_case_weightage + return False, err, 0.0 else: if not isfile(clean_test_case_path): msg = "No test case at %s" % clean_test_case_path - return False, msg, test_case_weightage + return False, msg, 0.0 if not os.access(clean_ref_code_path, os.R_OK): msg = "Test script %s, not readable" % clean_test_case_path - return False, msg, test_case_weightage + return False, msg, 0.0 # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. @@ -143,4 +142,4 @@ class BashCodeEvaluator(CodeEvaluator): " {0}, got {1}").format(inst_stdout+inst_stderr, stdnt_stdout+stdnt_stderr ) - return False, err, test_case_weightage + return False, err, 0.0 -- cgit From b32d7e91fe608c4cbd09b50520d4a3cab75d2e53 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 16 Nov 2016 11:25:30 +0530 Subject: Change test_case weightage field name to weight --- yaksh/bash_code_evaluator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index dd4445c..7575725 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -28,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, user_answer, file_paths, partial_grading, test_case, weightage): + def check_code(self, user_answer, file_paths, partial_grading, test_case, weight): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -52,7 +52,7 @@ class BashCodeEvaluator(CodeEvaluator): """ ref_code_path = test_case success = False - test_case_weightage = 0.0 + test_case_weight = 0.0 get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() @@ -92,8 +92,8 @@ class BashCodeEvaluator(CodeEvaluator): ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: - test_case_weightage = float(weightage) if partial_grading else 0.0 - return True, "Correct answer", test_case_weightage + test_case_weight = float(weight) if partial_grading else 0.0 + return True, "Correct answer", test_case_weight else: err = "Error: expected %s, got %s" % (inst_stderr, stdnt_stderr @@ -135,8 +135,8 @@ class BashCodeEvaluator(CodeEvaluator): proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): - test_case_weightage = float(weightage) if partial_grading else 0.0 - return True, "Correct answer", test_case_weightage + test_case_weight = float(weight) if partial_grading else 0.0 + return True, "Correct answer", test_case_weight else: err = ("Error:expected" " {0}, got {1}").format(inst_stdout+inst_stderr, -- cgit From 31a15a666e69d96b5062596c79641645a64fcdb2 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 16 Nov 2016 11:47:26 +0530 Subject: Modify docstrings of evaluators --- yaksh/bash_code_evaluator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'yaksh/bash_code_evaluator.py') diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 7575725..b5974d2 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -40,14 +40,17 @@ class BashCodeEvaluator(CodeEvaluator): Returns -------- + success - Boolean, indicating if code was executed successfully, correctly + weight - Float, indicating total weight of all successful test cases + error - String, error message if success is false - returns (True, "Correct answer") : If the student script passes all + returns (True, "Correct answer", 1.0) : If the student script passes all test cases/have same output, when compared to the instructor script - returns (False, error_msg): If the student script fails a single + returns (False, error_msg, 0.0): If the student script fails a single test/have dissimilar output, when compared to the instructor script. - Returns (False, error_msg): If mandatory arguments are not files or if + Returns (False, error_msg, 0.0): If mandatory arguments are not files or if the required permissions are not given to the file(s). """ ref_code_path = test_case -- cgit