diff options
Diffstat (limited to 'testapp')
| -rwxr-xr-x | testapp/exam/c_cpp_files/main.cpp (renamed from testapp/c_cpp_files/main.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main2.c (renamed from testapp/c_cpp_files/main2.c) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_array_check.cpp (renamed from testapp/c_cpp_files/main_array_check.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_array_check_all.cpp (renamed from testapp/c_cpp_files/main_array_check_all.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_array_sum.cpp (renamed from testapp/c_cpp_files/main_array_sum.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_blackJack.cpp (renamed from testapp/c_cpp_files/main_blackJack.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_check_digit.cpp (renamed from testapp/c_cpp_files/main_check_digit.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_count667.cpp (renamed from testapp/c_cpp_files/main_count667.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_count7.cpp (renamed from testapp/c_cpp_files/main_count7.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_fact.cpp (renamed from testapp/c_cpp_files/main_fact.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_greatest.cpp (renamed from testapp/c_cpp_files/main_greatest.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_hello_name.c (renamed from testapp/c_cpp_files/main_hello_name.c) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_lessThan9.cpp (renamed from testapp/c_cpp_files/main_lessThan9.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_mean.cpp (renamed from testapp/c_cpp_files/main_mean.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_palindrome.cpp (renamed from testapp/c_cpp_files/main_palindrome.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_roundTo10.cpp (renamed from testapp/c_cpp_files/main_roundTo10.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_specialSum.cpp (renamed from testapp/c_cpp_files/main_specialSum.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/c_cpp_files/main_within.cpp (renamed from testapp/c_cpp_files/main_within.cpp) | 0 | ||||
| -rwxr-xr-x | testapp/exam/code_server.py | 198 | ||||
| -rw-r--r-- | testapp/exam/evaluate_bash.py | 17 | ||||
| -rw-r--r-- | testapp/exam/evaluate_c.py | 33 | ||||
| -rw-r--r-- | testapp/exam/evaluate_cpp.py | 16 | ||||
| -rw-r--r-- | testapp/exam/evaluate_java.py | 10 | ||||
| -rw-r--r-- | testapp/exam/evaluate_python.py | 5 | ||||
| -rw-r--r-- | testapp/exam/evaluate_scilab.py | 13 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_array_sum.java (renamed from testapp/java_files/main_array_sum.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_fact.java (renamed from testapp/java_files/main_fact.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_great.java (renamed from testapp/java_files/main_great.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_hello_name.java (renamed from testapp/java_files/main_hello_name.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_lastDigit.java (renamed from testapp/java_files/main_lastDigit.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_moreThan30.java (renamed from testapp/java_files/main_moreThan30.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_palindrome.java (renamed from testapp/java_files/main_palindrome.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/java_files/main_square.java (renamed from testapp/java_files/main_square.java) | 0 | ||||
| -rw-r--r-- | testapp/exam/models.py | 4 | ||||
| -rw-r--r-- | testapp/exam/scilab_files/test_add.sce (renamed from testapp/scilab_files/test_add.sce) | 0 | ||||
| -rw-r--r-- | testapp/exam/test_code.py | 182 | ||||
| -rw-r--r-- | testapp/exam/views.py | 2 | ||||
| -rw-r--r-- | testapp/exam/xmlrpc_clients.py | 4 | ||||
| -rw-r--r-- | testapp/test_server.py | 19 |
39 files changed, 259 insertions, 244 deletions
diff --git a/testapp/c_cpp_files/main.cpp b/testapp/exam/c_cpp_files/main.cpp index ebe1f08..ebe1f08 100755 --- a/testapp/c_cpp_files/main.cpp +++ b/testapp/exam/c_cpp_files/main.cpp diff --git a/testapp/c_cpp_files/main2.c b/testapp/exam/c_cpp_files/main2.c index ccd1768..ccd1768 100755 --- a/testapp/c_cpp_files/main2.c +++ b/testapp/exam/c_cpp_files/main2.c diff --git a/testapp/c_cpp_files/main_array_check.cpp b/testapp/exam/c_cpp_files/main_array_check.cpp index ea34fdd..ea34fdd 100755 --- a/testapp/c_cpp_files/main_array_check.cpp +++ b/testapp/exam/c_cpp_files/main_array_check.cpp diff --git a/testapp/c_cpp_files/main_array_check_all.cpp b/testapp/exam/c_cpp_files/main_array_check_all.cpp index 140578e..140578e 100755 --- a/testapp/c_cpp_files/main_array_check_all.cpp +++ b/testapp/exam/c_cpp_files/main_array_check_all.cpp diff --git a/testapp/c_cpp_files/main_array_sum.cpp b/testapp/exam/c_cpp_files/main_array_sum.cpp index 55b2ebf..55b2ebf 100755 --- a/testapp/c_cpp_files/main_array_sum.cpp +++ b/testapp/exam/c_cpp_files/main_array_sum.cpp diff --git a/testapp/c_cpp_files/main_blackJack.cpp b/testapp/exam/c_cpp_files/main_blackJack.cpp index cc54e78..cc54e78 100755 --- a/testapp/c_cpp_files/main_blackJack.cpp +++ b/testapp/exam/c_cpp_files/main_blackJack.cpp diff --git a/testapp/c_cpp_files/main_check_digit.cpp b/testapp/exam/c_cpp_files/main_check_digit.cpp index d3bf3d6..d3bf3d6 100755 --- a/testapp/c_cpp_files/main_check_digit.cpp +++ b/testapp/exam/c_cpp_files/main_check_digit.cpp diff --git a/testapp/c_cpp_files/main_count667.cpp b/testapp/exam/c_cpp_files/main_count667.cpp index f146e8c..f146e8c 100755 --- a/testapp/c_cpp_files/main_count667.cpp +++ b/testapp/exam/c_cpp_files/main_count667.cpp diff --git a/testapp/c_cpp_files/main_count7.cpp b/testapp/exam/c_cpp_files/main_count7.cpp index 982e930..982e930 100755 --- a/testapp/c_cpp_files/main_count7.cpp +++ b/testapp/exam/c_cpp_files/main_count7.cpp diff --git a/testapp/c_cpp_files/main_fact.cpp b/testapp/exam/c_cpp_files/main_fact.cpp index a4ff230..a4ff230 100755 --- a/testapp/c_cpp_files/main_fact.cpp +++ b/testapp/exam/c_cpp_files/main_fact.cpp diff --git a/testapp/c_cpp_files/main_greatest.cpp b/testapp/exam/c_cpp_files/main_greatest.cpp index 6d0a7c2..6d0a7c2 100755 --- a/testapp/c_cpp_files/main_greatest.cpp +++ b/testapp/exam/c_cpp_files/main_greatest.cpp diff --git a/testapp/c_cpp_files/main_hello_name.c b/testapp/exam/c_cpp_files/main_hello_name.c index 71b83a2..71b83a2 100755 --- a/testapp/c_cpp_files/main_hello_name.c +++ b/testapp/exam/c_cpp_files/main_hello_name.c diff --git a/testapp/c_cpp_files/main_lessThan9.cpp b/testapp/exam/c_cpp_files/main_lessThan9.cpp index 722b4bb..722b4bb 100755 --- a/testapp/c_cpp_files/main_lessThan9.cpp +++ b/testapp/exam/c_cpp_files/main_lessThan9.cpp diff --git a/testapp/c_cpp_files/main_mean.cpp b/testapp/exam/c_cpp_files/main_mean.cpp index 21a4b1a..21a4b1a 100755 --- a/testapp/c_cpp_files/main_mean.cpp +++ b/testapp/exam/c_cpp_files/main_mean.cpp diff --git a/testapp/c_cpp_files/main_palindrome.cpp b/testapp/exam/c_cpp_files/main_palindrome.cpp index 0e66928..0e66928 100755 --- a/testapp/c_cpp_files/main_palindrome.cpp +++ b/testapp/exam/c_cpp_files/main_palindrome.cpp diff --git a/testapp/c_cpp_files/main_roundTo10.cpp b/testapp/exam/c_cpp_files/main_roundTo10.cpp index 12c961d..12c961d 100755 --- a/testapp/c_cpp_files/main_roundTo10.cpp +++ b/testapp/exam/c_cpp_files/main_roundTo10.cpp diff --git a/testapp/c_cpp_files/main_specialSum.cpp b/testapp/exam/c_cpp_files/main_specialSum.cpp index d614536..d614536 100755 --- a/testapp/c_cpp_files/main_specialSum.cpp +++ b/testapp/exam/c_cpp_files/main_specialSum.cpp diff --git a/testapp/c_cpp_files/main_within.cpp b/testapp/exam/c_cpp_files/main_within.cpp index 50f9ad0..50f9ad0 100755 --- a/testapp/c_cpp_files/main_within.cpp +++ b/testapp/exam/c_cpp_files/main_within.cpp diff --git a/testapp/exam/code_server.py b/testapp/exam/code_server.py index f5c5698..db30798 100755 --- a/testapp/exam/code_server.py +++ b/testapp/exam/code_server.py @@ -19,7 +19,6 @@ settings.py:SERVER_POOL_PORT. This port exposes a `get_server_port` function that returns an available server. """ import sys -import traceback from SimpleXMLRPCServer import SimpleXMLRPCServer import pwd import os @@ -34,13 +33,15 @@ import importlib # Local imports. from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT from registry import registry - +from evaluate_python import EvaluatePython +from evaluate_c import EvaluateC +from evaluate_cpp import EvaluateCpp +from evaluate_java import EvaluateJava +from evaluate_scilab import EvaluateScilab +from evaluate_bash import EvaluateBash MY_DIR = abspath(dirname(__file__)) -registry.register('python', ) -registry.register('py', MyTestCode) - def run_as_nobody(): """Runs the current process as nobody.""" # Set the effective uid and to that of nobody. @@ -48,166 +49,6 @@ def run_as_nobody(): os.setegid(nobody.pw_gid) os.seteuid(nobody.pw_uid) - -# Raised when the code times-out. -# c.f. http://pguides.net/python/timeout-a-function -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - """A handler for the ALARM signal.""" - raise TimeoutException('Code took too long to run.') - -def create_signal_handler(): - """Add a new signal handler for the execution of this code.""" - prev_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - return prev_handler - -def set_original_signal_handler(old_handler=None): - """Set back any original signal handler.""" - if old_handler is not None: - signal.signal(signal.SIGALRM, old_handler) - return - else: - raise Exception("Signal Handler: object cannot be NoneType") - -def delete_signal_handler(): - signal.alarm(0) - return - - - -############################################################################### -# `TestCode` class. -############################################################################### -class TestCode(object): - """Tests the code obtained from Code Server""" - def __init__(self, test_parameter, language, user_answer, ref_code_path=None, in_dir=None): - msg = 'Code took more than %s seconds to run. You probably '\ - 'have an infinite loop in your code.' % SERVER_TIMEOUT - self.timeout_msg = msg - self.test_parameter = test_parameter - self.language = language.lower() - self.user_answer = user_answer - self.ref_code_path = ref_code_path - self.in_dir = in_dir - - def run_code(self): - """Tests given code (`answer`) with the test cases based on - given arguments. - - The ref_code_path is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - """ - self._change_dir(self.in_dir) - - # Add a new signal handler for the execution of this code. - prev_handler = create_signal_handler() - success = False - - # Do whatever testing needed. - try: - success, err = self.evaluate_code() - - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - set_original_signal_handler(prev_handler) - - # Cancel the signal - delete_signal_handler() - - result = {'success': success, 'error': err} - return result - - def evaluate_code(self): - raise NotImplementedError("evaluate_code method not implemented") - - def _create_submit_code_file(self, file_name): - """ Write the code (`answer`) to a file and set the file path""" - # File name/extension depending on the question language - submit_f = open(file_name, 'w') - submit_f.write(self.user_answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - if sfile_elf.language == "bash": - self._set_file_as_executable(submit_path) - - return submit_path - - def _set_file_as_executable(self, fname): - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) - - def _set_test_code_file_path(self, ref_path=None, test_case_path=None): - # ref_path, test_case_path = self.ref_code_path.split(',') - - if ref_path and not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - if test_case_path and not test_case_path.startswith('/'): - test_case_path = join(MY_DIR, test_case_path) - - return ref_path, test_case_path - - def _run_command(self, cmd_args, *args, **kw): - """Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stdout and stderr. - """ - try: - proc = subprocess.Popen(cmd_args, *args, **kw) - stdout, stderr = proc.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc.kill() - # Re-raise exception. - raise - return proc, stdout, stderr - - def _compile_command(self, cmd, *args, **kw): - """Compiles C/C++/java code and returns errors if any. - Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stderr. - """ - try: - proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc_compile.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc_compile.kill() - # Re-raise exception. - raise - return proc_compile, err - - def _change_dir(self, in_dir): - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - ############################################################################### # `CodeServer` class. ############################################################################### @@ -219,34 +60,17 @@ class CodeServer(object): self.port = port self.queue = queue - def check_code(self, info_parameter, in_dir=None): - """Calls the TestCode Class to test the current code""" - info_parameter = json.loads(info_parameter) - test_parameter = info_parameter.get("test_parameter") - language = info_parameter.get("language") - user_answer = info_parameter.get("user_answer") - ref_code_path = info_parameter.get("ref_code_path") - - eval_module_name = "evaluate_{0}".format(language.lower()) - eval_class_name = "Evaluate{0}".format(language.capitalize()) - - get_class = self._sub_class_factory(eval_module_name, eval_class_name) + def check_code(self, info_parameter, language, in_dir=None): + """Calls the TestCode SUb Class based on language to test the current code""" + evaluate_code_class = registry.get_class(language) + evaluate_code_instance = evaluate_code_class.from_json(info_parameter, language, in_dir) + result = evaluate_code_instance.run_code() - test_code_class = get_class(test_parameter, language, user_answer, ref_code_path, in_dir) - result = test_code_class.run_code() # Put us back into the server pool queue since we are free now. self.queue.put(self.port) return json.dumps(result) - def _sub_class_factory(self, module_name, class_name): - # load the module, will raise ImportError if module cannot be loaded - get_module = importlib.import_module(module_name) - # get the class, will raise AttributeError if class cannot be found - get_class = getattr(get_module, class_name) - - return get_class - def run(self): """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("localhost", self.port)) diff --git a/testapp/exam/evaluate_bash.py b/testapp/exam/evaluate_bash.py index 57c89ae..fd769cd 100644 --- a/testapp/exam/evaluate_bash.py +++ b/testapp/exam/evaluate_bash.py @@ -7,16 +7,17 @@ import subprocess import importlib # local imports -from code_server import TestCode +from test_code import TestCode from registry import registry class EvaluateBash(TestCode): """Tests the Bash code obtained from Code Server""" def evaluate_code(self): - submit_path = self._create_submit_code_file('submit.sh') + fpath = self.create_submit_code_file('submit.sh') + submit_path = self.set_file_as_executable(fpath) get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path, get_test_case_path) + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path, get_test_case_path) success = False success, err = self.check_bash_script(ref_path, submit_path, @@ -61,11 +62,11 @@ class EvaluateBash(TestCode): return False, "Script %s is not executable" % submit_path if test_case_path is None or "": - ret = self._run_command(ref_path, stdin=None, + ret = self.run_command(ref_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, inst_stdout, inst_stderr = ret - ret = self._run_command(submit_path, stdin=None, + ret = self.run_command(submit_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, stdnt_stdout, stdnt_stderr = ret @@ -92,12 +93,12 @@ class EvaluateBash(TestCode): loop_count += 1 if valid_answer: args = [ref_path] + [x for x in test_case.split()] - ret = self._run_command(args, stdin=None, + ret = self.run_command(args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, inst_stdout, inst_stderr = ret args = [submit_path]+[x for x in test_case.split()] - ret = self._run_command(args, stdin=None, + ret = self.run_command(args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, stdnt_stdout, stdnt_stderr = ret @@ -109,4 +110,4 @@ class EvaluateBash(TestCode): stdnt_stdout+stdnt_stderr) return False, err -registry.register('bash', evaluate_bash, EvaluateBash)
\ No newline at end of file +registry.register('bash', EvaluateBash)
\ No newline at end of file diff --git a/testapp/exam/evaluate_c.py b/testapp/exam/evaluate_c.py index bbe0e87..0700daa 100644 --- a/testapp/exam/evaluate_c.py +++ b/testapp/exam/evaluate_c.py @@ -7,29 +7,29 @@ import subprocess import importlib # local imports -from code_server import TestCode +from test_code import TestCode from registry import registry class EvaluateC(TestCode): """Tests the C code obtained from Code Server""" def evaluate_code(self): - submit_path = self._create_submit_code_file('submit.c') + submit_path = self.create_submit_code_file('submit.c') get_ref_path = self.ref_code_path - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) success = False # Set file paths - c_user_output_path = os.getcwd() + '/output', - c_ref_output_path = os.getcwd() + '/executable', + c_user_output_path = os.getcwd() + '/output' + c_ref_output_path = os.getcwd() + '/executable' # Set command variables - compile_command = 'g++ {0} -c -o {1}'.format(submit_path, - c_user_output_path), + compile_command = 'g++ {0} -c -o {1}'.format(submit_path, + c_user_output_path) compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, c_user_output_path, - c_ref_output_path), - run_command_args = c_ref_output_path + c_ref_output_path) + run_command_args = [c_ref_output_path] remove_user_output = c_user_output_path remove_ref_output = c_ref_output_path @@ -72,21 +72,21 @@ class EvaluateC(TestCode): success = False # output_path = os.getcwd() + '/output' - ret = self._compile_command(compile_command) + ret = self.compile_command(compile_command) proc, stdnt_stderr = ret # if self.language == "java": - stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + stdnt_stderr = self.remove_null_substitute_char(stdnt_stderr) # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - ret = self._compile_command(compile_main) + ret = self.compile_command(compile_main) proc, main_err = ret # if self.language == "java": - # main_err = self._remove_null_substitute_char(main_err) + main_err = self.remove_null_substitute_char(main_err) if main_err == '': - ret = self._run_command(run_command_args, stdin=None, + ret = self.run_command(run_command_args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, stdout, stderr = ret @@ -121,7 +121,8 @@ class EvaluateC(TestCode): return success, err - def _remove_null_substitute_char(self, string): + + def remove_null_substitute_char(self, string): """Returns a string without any null and substitute characters""" stripped = "" for c in string: @@ -129,4 +130,4 @@ class EvaluateC(TestCode): stripped = stripped + c return ''.join(stripped) -registry.register('c', evaluate_c, EvaluateC)
\ No newline at end of file +registry.register('c', EvaluateC)
\ No newline at end of file diff --git a/testapp/exam/evaluate_cpp.py b/testapp/exam/evaluate_cpp.py index 87bd7a3..cffe744 100644 --- a/testapp/exam/evaluate_cpp.py +++ b/testapp/exam/evaluate_cpp.py @@ -8,7 +8,7 @@ import importlib # local imports from evaluate_c import EvaluateC -from code_server import TestCode +from test_code import TestCode from registry import registry @@ -16,21 +16,21 @@ from registry import registry class EvaluateCpp(EvaluateC, TestCode): """Tests the C code obtained from Code Server""" def evaluate_code(self): - submit_path = self._create_submit_code_file('submitstd.cpp') + submit_path = self.create_submit_code_file('submitstd.cpp') get_ref_path = self.ref_code_path - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) success = False # Set file paths - c_user_output_path = os.getcwd() + '/output', - c_ref_output_path = os.getcwd() + '/executable', + c_user_output_path = os.getcwd() + '/output' + c_ref_output_path = os.getcwd() + '/executable' # Set command variables compile_command = 'g++ {0} -c -o {1}'.format(submit_path, - c_user_output_path), + c_user_output_path) compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, c_user_output_path, - c_ref_output_path), + c_ref_output_path) run_command_args = c_ref_output_path remove_user_output = c_user_output_path remove_ref_output = c_ref_output_path @@ -44,4 +44,4 @@ class EvaluateCpp(EvaluateC, TestCode): return success, err -registry.register('cpp', evaluate_cpp, EvaluateCpp)
\ No newline at end of file +registry.register('cpp', EvaluateCpp)
\ No newline at end of file diff --git a/testapp/exam/evaluate_java.py b/testapp/exam/evaluate_java.py index f908102..d3d4e2a 100644 --- a/testapp/exam/evaluate_java.py +++ b/testapp/exam/evaluate_java.py @@ -8,7 +8,7 @@ import importlib # local imports from evaluate_c import EvaluateC -from code_server import TestCode +from test_code import TestCode from registry import registry @@ -16,14 +16,14 @@ from registry import registry class EvaluateJava(EvaluateC, TestCode): """Tests the C code obtained from Code Server""" def evaluate_code(self): - submit_path = self._create_submit_code_file('Test.java') + submit_path = self.create_submit_code_file('Test.java') get_ref_path = self.ref_code_path - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) success = False # Set file paths java_student_directory = os.getcwd() + '/' - java_ref_file_name = (ref_code_path.split('/')[-1]).split('.')[0], + java_ref_file_name = (ref_code_path.split('/')[-1]).split('.')[0] # Set command variables compile_command = 'javac {0}'.format(submit_code_path), @@ -46,4 +46,4 @@ class EvaluateJava(EvaluateC, TestCode): return success, err -registry.register('java', evaluate_java, EvaluateJava)
\ No newline at end of file +registry.register('java', EvaluateJava)
\ No newline at end of file diff --git a/testapp/exam/evaluate_python.py b/testapp/exam/evaluate_python.py index 18fde43..3af82b6 100644 --- a/testapp/exam/evaluate_python.py +++ b/testapp/exam/evaluate_python.py @@ -6,7 +6,7 @@ from os.path import join import importlib # local imports -from code_server import TestCode +from test_code import TestCode from registry import registry class EvaluatePython(TestCode): @@ -36,6 +36,7 @@ class EvaluatePython(TestCode): del tb return success, err + # Private Protocol def _create_test_case(self): """ Create assert based test cases in python @@ -52,4 +53,4 @@ class EvaluatePython(TestCode): test_code += tcode + "\n" return test_code -registry.register('python', evaluate_python, EvaluatePython)
\ No newline at end of file +registry.register('python', EvaluatePython)
\ No newline at end of file diff --git a/testapp/exam/evaluate_scilab.py b/testapp/exam/evaluate_scilab.py index 1874cf6..f4253ff 100644 --- a/testapp/exam/evaluate_scilab.py +++ b/testapp/exam/evaluate_scilab.py @@ -7,21 +7,21 @@ import re import importlib # local imports -from code_server import TestCode -from registry import registryz +from test_code import TestCode +from registry import registry class EvaluateScilab(TestCode): """Tests the Scilab code obtained from Code Server""" # def evaluate_scilab_code(self): def evaluate_code(self): - submit_path = self._create_submit_code_file('function.sci') - ref_path, test_case_path = self._set_test_code_file_path() + submit_path = self.create_submit_code_file('function.sci') + ref_path, test_case_path = self.set_test_code_file_path() success = False cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) cmd += ' | timeout 8 scilab-cli -nb' - ret = self._run_command(cmd, + ret = self.run_command(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -44,6 +44,7 @@ class EvaluateScilab(TestCode): return success, err + # Private Protocol def _remove_scilab_exit(self, string): """ Removes exit, quit and abort from the scilab code @@ -79,4 +80,4 @@ class EvaluateScilab(TestCode): strip_out = strip_out+"\n"+l.strip() return strip_out -registry.register('scilab', evaluate_scilab, EvaluateScilab)
\ No newline at end of file +registry.register('scilab', EvaluateScilab)
\ No newline at end of file diff --git a/testapp/java_files/main_array_sum.java b/testapp/exam/java_files/main_array_sum.java index 5eae299..5eae299 100644 --- a/testapp/java_files/main_array_sum.java +++ b/testapp/exam/java_files/main_array_sum.java diff --git a/testapp/java_files/main_fact.java b/testapp/exam/java_files/main_fact.java index 325dab6..325dab6 100644 --- a/testapp/java_files/main_fact.java +++ b/testapp/exam/java_files/main_fact.java diff --git a/testapp/java_files/main_great.java b/testapp/exam/java_files/main_great.java index 4bfcb1f..4bfcb1f 100644 --- a/testapp/java_files/main_great.java +++ b/testapp/exam/java_files/main_great.java diff --git a/testapp/java_files/main_hello_name.java b/testapp/exam/java_files/main_hello_name.java index 84bb282..84bb282 100644 --- a/testapp/java_files/main_hello_name.java +++ b/testapp/exam/java_files/main_hello_name.java diff --git a/testapp/java_files/main_lastDigit.java b/testapp/exam/java_files/main_lastDigit.java index 05439e2..05439e2 100644 --- a/testapp/java_files/main_lastDigit.java +++ b/testapp/exam/java_files/main_lastDigit.java diff --git a/testapp/java_files/main_moreThan30.java b/testapp/exam/java_files/main_moreThan30.java index 7da31cb..7da31cb 100644 --- a/testapp/java_files/main_moreThan30.java +++ b/testapp/exam/java_files/main_moreThan30.java diff --git a/testapp/java_files/main_palindrome.java b/testapp/exam/java_files/main_palindrome.java index c0745f9..c0745f9 100644 --- a/testapp/java_files/main_palindrome.java +++ b/testapp/exam/java_files/main_palindrome.java diff --git a/testapp/java_files/main_square.java b/testapp/exam/java_files/main_square.java index 5cb8c35..5cb8c35 100644 --- a/testapp/java_files/main_square.java +++ b/testapp/exam/java_files/main_square.java diff --git a/testapp/exam/models.py b/testapp/exam/models.py index 874c81c..d0c9cc2 100644 --- a/testapp/exam/models.py +++ b/testapp/exam/models.py @@ -87,7 +87,7 @@ class Question(models.Model): # Tags for the Question. tags = TaggableManager() - def consolidate_answer_data(self, test_cases, user_answer): #test + def consolidate_answer_data(self, test_cases, user_answer): test_case_parameter = [] info_parameter = {} @@ -113,7 +113,7 @@ class Question(models.Model): parameter_dict['pos_args'] = pos_args_list test_case_parameter.append(parameter_dict) - info_parameter['language'] = self.language + # info_parameter['language'] = self.language info_parameter['id'] = self.id info_parameter['user_answer'] = user_answer info_parameter['test_parameter'] = test_case_parameter diff --git a/testapp/scilab_files/test_add.sce b/testapp/exam/scilab_files/test_add.sce index a317cdb..a317cdb 100644 --- a/testapp/scilab_files/test_add.sce +++ b/testapp/exam/scilab_files/test_add.sce diff --git a/testapp/exam/test_code.py b/testapp/exam/test_code.py new file mode 100644 index 0000000..8930f55 --- /dev/null +++ b/testapp/exam/test_code.py @@ -0,0 +1,182 @@ +import sys +from SimpleXMLRPCServer import SimpleXMLRPCServer +import pwd +import os +import stat +from os.path import isdir, dirname, abspath, join, isfile +import signal +from multiprocessing import Process, Queue +import subprocess +import re +import json +import importlib +# Local imports. +from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT + + +MY_DIR = abspath(dirname(__file__)) + +# Raised when the code times-out. +# c.f. http://pguides.net/python/timeout-a-function +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + """A handler for the ALARM signal.""" + raise TimeoutException('Code took too long to run.') + +def create_signal_handler(): + """Add a new signal handler for the execution of this code.""" + prev_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + return prev_handler + +def set_original_signal_handler(old_handler=None): + """Set back any original signal handler.""" + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + return + else: + raise Exception("Signal Handler: object cannot be NoneType") + +def delete_signal_handler(): + signal.alarm(0) + return + + + +############################################################################### +# `TestCode` class. +############################################################################### +class TestCode(object): + """Tests the code obtained from Code Server""" + def __init__(self, test_parameter, language, user_answer, ref_code_path=None, in_dir=None): + msg = 'Code took more than %s seconds to run. You probably '\ + 'have an infinite loop in your code.' % SERVER_TIMEOUT + self.timeout_msg = msg + self.test_parameter = test_parameter + self.language = language.lower() + self.user_answer = user_answer + self.ref_code_path = ref_code_path + self.in_dir = in_dir + + @classmethod + def from_json(cls, blob, language, in_dir): + info_parameter = json.loads(blob) + test_parameter = info_parameter.get("test_parameter") + user_answer = info_parameter.get("user_answer") + ref_code_path = info_parameter.get("ref_code_path") + + instance = cls(test_parameter, language, user_answer, ref_code_path, in_dir) + return instance + + def run_code(self): + """Tests given code (`answer`) with the test cases based on + given arguments. + + The ref_code_path is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + If the optional `in_dir` keyword argument is supplied it changes the + directory to that directory (it does not change it back to the original + when done). + + Returns + ------- + + A tuple: (success, error message). + """ + self._change_dir(self.in_dir) + + # Add a new signal handler for the execution of this code. + prev_handler = create_signal_handler() + success = False + + # Do whatever testing needed. + try: + success, err = self.evaluate_code() + + except TimeoutException: + err = self.timeout_msg + except: + type, value = sys.exc_info()[:2] + err = "Error: {0}".format(repr(value)) + finally: + # Set back any original signal handler. + set_original_signal_handler(prev_handler) + + # Cancel the signal + delete_signal_handler() + + result = {'success': success, 'error': err} + return result + + def evaluate_code(self): + raise NotImplementedError("evaluate_code method not implemented") + + def create_submit_code_file(self, file_name): + """ Write the code (`answer`) to a file and set the file path""" + # File name/extension depending on the question language + submit_f = open(file_name, 'w') + submit_f.write(self.user_answer.lstrip()) + submit_f.close() + submit_path = abspath(submit_f.name) + + return submit_path + + def set_file_as_executable(self, fname): + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + + def set_test_code_file_path(self, ref_path=None, test_case_path=None): + if ref_path and not ref_path.startswith('/'): + ref_path = join(MY_DIR, ref_path) + + if test_case_path and not test_case_path.startswith('/'): + test_case_path = join(MY_DIR, test_case_path) + + return ref_path, test_case_path + + def run_command(self, cmd_args, *args, **kw): + """Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stdout and stderr. + """ + try: + proc = subprocess.Popen(cmd_args, *args, **kw) + stdout, stderr = proc.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc.kill() + # Re-raise exception. + raise + return proc, stdout, stderr + + def compile_command(self, cmd, *args, **kw): + """Compiles C/C++/java code and returns errors if any. + Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stderr. + """ + try: + proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc_compile.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc_compile.kill() + # Re-raise exception. + raise + return proc_compile, err + + def _change_dir(self, in_dir): + if in_dir is not None and isdir(in_dir): + os.chdir(in_dir)
\ No newline at end of file diff --git a/testapp/exam/views.py b/testapp/exam/views.py index 840f30a..508b623 100644 --- a/testapp/exam/views.py +++ b/testapp/exam/views.py @@ -1025,7 +1025,7 @@ def validate_answer(user, user_answer, question, info_parameter=None): message = 'Correct answer' elif question.type == 'code': user_dir = get_user_dir(user) - json_result = code_server.run_code(info_parameter, user_dir) + json_result = code_server.run_code(info_parameter, question.language, user_dir) result = json.loads(json_result) if result.get('success'): correct = True diff --git a/testapp/exam/xmlrpc_clients.py b/testapp/exam/xmlrpc_clients.py index 33fbc57..5d95cae 100644 --- a/testapp/exam/xmlrpc_clients.py +++ b/testapp/exam/xmlrpc_clients.py @@ -22,7 +22,7 @@ class CodeServerProxy(object): pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT) self.pool_server = ServerProxy(pool_url) - def run_code(self, info_parameter, user_dir): + def run_code(self, info_parameter, language, user_dir): """Tests given code (`answer`) with the `test_code` supplied. If the optional `in_dir` keyword argument is supplied it changes the directory to that directory (it does not change it back to the original when @@ -50,7 +50,7 @@ class CodeServerProxy(object): try: server = self._get_server() - result = server.check_code(info_parameter, user_dir) + result = server.check_code(info_parameter, language, user_dir) except ConnectionError: result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'}) return result diff --git a/testapp/test_server.py b/testapp/test_server.py index b837365..c35c411 100644 --- a/testapp/test_server.py +++ b/testapp/test_server.py @@ -19,7 +19,7 @@ class TestPythonEvaluation(unittest.TestCase): self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): - user_answer = "def add(a, b):\n\treturn a - b""" + user_answer = "def add(a, b):\n\treturn a - b" test_parameter = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', @@ -48,7 +48,7 @@ class TestPythonEvaluation(unittest.TestCase): class TestCEvaluation(unittest.TestCase): def setUp(self): self.language = "C" - self.ref_code_path = os.getcwd() + "/c_cpp_files/main.cpp" + self.ref_code_path = "c_cpp_files/main.cpp" self.in_dir = "/tmp" self.test_parameter = [] @@ -56,21 +56,23 @@ class TestCEvaluation(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = evaluate_c.EvaluateC(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() + self.assertTrue(result.get("success")) - self.assertEqual(result.get("error")) + self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a+b}" get_class = evaluate_c.EvaluateC(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() + self.assertFalse(result.get("success")) - self.assertTrue("compilation error" in result.get("error").lower()) + self.assertTrue("Compilation Error" in result.get("error")) ############################################################################### class TestCPPEvaluation(unittest.TestCase): def setUp(self): self.language = "CPP" - self.ref_code_path = os.getcwd() + "/c_cpp_files/main.cpp" + self.ref_code_path = "c_cpp_files/main.cpp" self.in_dir = "/tmp" self.test_parameter = [] @@ -78,15 +80,18 @@ class TestCPPEvaluation(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = evaluate_cpp.EvaluateCpp(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() + self.assertTrue(result.get("success")) - self.assertEqual(result.get("error")) + self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a+b}" get_class = evaluate_cpp.EvaluateCpp(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() + error_msg = "" + self.assertFalse(result.get("success")) - self.assertTrue("compilation error" in result.get("error").lower()) + self.assertTrue("Compilation Error" in result.get("error")) if __name__ == '__main__': unittest.main()
\ No newline at end of file |