diff options
Diffstat (limited to 'yaksh')
| -rw-r--r-- | yaksh/code_evaluator.py | 28 | ||||
| -rw-r--r-- | yaksh/cpp_code_evaluator.py | 290 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 69 |
3 files changed, 234 insertions, 153 deletions
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 535daa3..b777b67 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -3,7 +3,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer import pwd import os import stat -from os.path import isdir, dirname, abspath, join, isfile +from os.path import isdir, dirname, abspath, join, isfile, exists import signal from multiprocessing import Process, Queue import subprocess @@ -144,15 +144,31 @@ class CodeEvaluator(object): def check_code(self): raise NotImplementedError("check_code method not implemented") + # def create_submit_code_file(self, file_name): + # """ Write the code (`answer`) to a file and set the file path""" + # submit_f = open(file_name, 'w') + # submit_f.write(self.user_answer.lstrip()) + # submit_f.close() + # submit_path = abspath(submit_f.name) + + # return submit_path + def create_submit_code_file(self, file_name): - """ Write the code (`answer`) to a file and set the file path""" - submit_f = open(file_name, 'w') - submit_f.write(self.user_answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) + """ Set the file path for code (`answer`)""" + submit_path = abspath(file_name) + if not exists(submit_path): + submit_f = open(submit_path, 'w') + submit_f.close() return submit_path + + def write_to_submit_code_file(self, file_path, user_answer): + """ Write the code (`answer`) to a file""" + submit_f = open(file_path, 'w') + submit_f.write(user_answer.lstrip()) + submit_f.close() + def _set_file_as_executable(self, fname): os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 129a1c8..6d18982 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -12,113 +12,211 @@ from code_evaluator import CodeEvaluator class CppCodeEvaluator(CodeEvaluator): """Tests the C code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): - super(CppCodeEvaluator, self).__init__(test_case_data, test, language, - user_answer, ref_code_path, - in_dir) - self.test_case_args = self.setup() - def setup(self): super(CppCodeEvaluator, self).setup() - - get_ref_path = self.ref_code_path - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) - self.submit_path = self.create_submit_code_file('submit.c') - - # Set file paths - c_user_output_path = os.getcwd() + '/output' - c_ref_output_path = os.getcwd() + '/executable' - - # Set command variables - compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, - c_user_output_path) - compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, - c_user_output_path, - c_ref_output_path) - run_command_args = [c_ref_output_path] - remove_user_output = c_user_output_path - remove_ref_output = c_ref_output_path - - return (ref_path, self.submit_path, compile_command, compile_main, - run_command_args, remove_user_output, remove_ref_output) + self.submit_code_path = self.create_submit_code_file('submit.c') def teardown(self): - # Delete the created file. super(CppCodeEvaluator, self).teardown() - os.remove(self.submit_path) - - def check_code(self, ref_code_path, submit_code_path, compile_command, - compile_main, run_command_args, remove_user_output, - remove_ref_output): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - if not isfile(ref_code_path): - return False, "No file at %s or Incorrect path" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s or Incorrect path' % submit_code_path - - success = False - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret - stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - ret = self._compile_command(compile_main) - proc, main_err = ret - main_err = self._remove_null_substitute_char(main_err) - - if main_err == '': - ret = self._run_command(run_command_args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" + # Delete the created file. + os.remove(self.submit_code_path) + + def set_file_paths(self): + user_output_path = os.getcwd() + '/output' + ref_output_path = os.getcwd() + '/executable' + + return user_output_path, ref_output_path + + def get_commands(self, clean_ref_code_path, user_output_path, + ref_output_path): + compile_command = 'g++ {0} -c -o {1}'.format(self.submit_code_path, + user_output_path) + compile_main = 'g++ {0} {1} -o {2}'.format(clean_ref_code_path, + user_output_path, + ref_output_path) + return compile_command, compile_main + + def check_code(self, user_answer, test_case_data): + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + + """ + ref_code_path = test_case_data[0] + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, 'No file at %s or Incorrect path' % self.submit_code_path + + success = False + self.write_to_submit_code_file(self.submit_code_path, user_answer) + user_output_path, ref_output_path = self.set_file_paths() + compile_command, compile_main = self.get_commands(clean_ref_code_path, user_output_path, ref_output_path) + ret = self._compile_command(compile_command) + proc, stdnt_stderr = ret + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + ret = self._compile_command(compile_main) + proc, main_err = ret + main_err = self._remove_null_substitute_char(main_err) + + if main_err == '': + ret = self._run_command([ref_output_path], stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" + else: + err = stdout + "\n" + stderr + os.remove(ref_output_path) else: - err = stdout + "\n" + stderr - os.remove(remove_ref_output) + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(user_output_path) else: - err = "Error:" + err = "Compilation Error:" try: - error_lines = main_err.splitlines() + error_lines = stdnt_stderr.splitlines() for e in error_lines: if ':' in e: err = err + "\n" + e.split(":", 1)[1] else: err = err + "\n" + e except: - err = err + "\n" + main_err - os.remove(remove_user_output) - else: - err = "Compilation Error:" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + stdnt_stderr - - return success, err + err = err + "\n" + stdnt_stderr + + return success, err + + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + # super(CppCodeEvaluator, self).__init__(test_case_data, test, language, + # user_answer, ref_code_path, + # in_dir) + # self.test_case_args = self.setup() + + # def setup(self): + # super(CppCodeEvaluator, self).setup() + + # get_ref_path = self.ref_code_path + # ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + # self.submit_path = self.create_submit_code_file('submit.c') + + # # Set file paths #@@@ in different func get_output_file_paths + # c_user_output_path = os.getcwd() + '/output' + # c_ref_output_path = os.getcwd() + '/executable' + + # # Set command variables #@@@ This section in different func get_commands + # compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, + # c_user_output_path) + # compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, + # c_user_output_path, + # c_ref_output_path) + # run_command_args = [c_ref_output_path] + # remove_user_output = c_user_output_path #@@@ not required + # remove_ref_output = c_ref_output_path #@@@ not required + + # return (ref_path, self.submit_path, compile_command, compile_main, + # run_command_args, remove_user_output, remove_ref_output) + + # def check_code(self, ref_code_path, submit_code_path, compile_command, + # compile_main, run_command_args, remove_user_output, + # remove_ref_output): + # """ Function validates student code using instructor code as + # reference.The first argument ref_code_path, is the path to + # instructor code, it is assumed to have executable permission. + # The second argument submit_code_path, is the path to the student + # code, it is assumed to have executable permission. + + # Returns + # -------- + + # returns (True, "Correct answer") : If the student function returns + # expected output when called by reference code. + + # returns (False, error_msg): If the student function fails to return + # expected output when called by reference code. + + # Returns (False, error_msg): If mandatory arguments are not files or + # if the required permissions are not given to the file(s). + + # """ + # if not isfile(ref_code_path): + # return False, "No file at %s or Incorrect path" % ref_code_path + # if not isfile(submit_code_path): + # return False, 'No file at %s or Incorrect path' % submit_code_path + + # success = False + # ret = self._compile_command(compile_command) + # proc, stdnt_stderr = ret + # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # # Only if compilation is successful, the program is executed + # # And tested with testcases + # if stdnt_stderr == '': + # ret = self._compile_command(compile_main) + # proc, main_err = ret + # main_err = self._remove_null_substitute_char(main_err) + + # if main_err == '': + # ret = self._run_command(run_command_args, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdout, stderr = ret + # if proc.returncode == 0: + # success, err = True, "Correct answer" + # else: + # err = stdout + "\n" + stderr + # os.remove(remove_ref_output) + # else: + # err = "Error:" + # try: + # error_lines = main_err.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + main_err + # os.remove(remove_user_output) + # else: + # err = "Compilation Error:" + # try: + # error_lines = stdnt_stderr.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + stdnt_stderr + + # return success, err diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d01cc9c..c5f8775 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -5,71 +5,38 @@ from yaksh.settings import SERVER_TIMEOUT class CEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "C" - self.ref_code_path = "c_cpp_files/main.cpp" + self.test_case_data = ["c_cpp_files/main.cpp"] self.in_dir = "/tmp" - self.test_case_data = [] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEqual(result.get('error'), "Correct answer") def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertFalse(result.get("success")) - self.assertTrue("Compilation Error" in result.get("error")) - - def test_infinite_loop(self): - user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), self.timeout_msg) - - -############################################################################### -class CppEvaluationTestCases(unittest.TestCase): - def setUp(self): - self.language = "CPP" - self.ref_code_path = "c_cpp_files/main.cpp" - self.in_dir = "/tmp" - self.test_case_data = [] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None - - def test_correct_answer(self): - user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") - - def test_compilation_error(self): - user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) |