From 8664a766406d6acf0d6a1688948153c407ea27f2 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 24 Apr 2015 14:25:26 +0530 Subject: Code Review: Code refactoring - Rename files - Create function for @classmethod call - Fix current, add new testcases - Fix views to fetch solution/ref_code_path fields in question post save - Fix errors --- testapp/docs/sample.args | 2 - testapp/docs/sample.sh | 2 - testapp/docs/sample_questions.py | 84 ---------------- testapp/docs/sample_questions.xml | 43 -------- testapp/exam/bash_files/sample.args | 2 + testapp/exam/bash_files/sample.sh | 2 + testapp/exam/code_server.py | 34 ++++--- testapp/exam/evaluate_bash.py | 113 --------------------- testapp/exam/evaluate_bash_code.py | 119 ++++++++++++++++++++++ testapp/exam/evaluate_c.py | 133 ------------------------- testapp/exam/evaluate_c_code.py | 136 +++++++++++++++++++++++++ testapp/exam/evaluate_code.py | 186 ++++++++++++++++++++++++++++++++++ testapp/exam/evaluate_cpp.py | 47 --------- testapp/exam/evaluate_cpp_code.py | 48 +++++++++ testapp/exam/evaluate_java.py | 49 --------- testapp/exam/evaluate_java_code.py | 49 +++++++++ testapp/exam/evaluate_python.py | 56 ----------- testapp/exam/evaluate_python_code.py | 58 +++++++++++ testapp/exam/evaluate_scilab.py | 83 ---------------- testapp/exam/evaluate_scilab_code.py | 86 ++++++++++++++++ testapp/exam/language_registry.py | 16 +++ testapp/exam/models.py | 30 +++--- testapp/exam/registry.py | 14 --- testapp/exam/test_code.py | 182 ---------------------------------- testapp/exam/views.py | 19 ++-- testapp/exam/xmlrpc_clients.py | 6 +- testapp/test_server.py | 187 +++++++++++++++++++++++++++++++---- 27 files changed, 923 insertions(+), 863 deletions(-) delete mode 100644 testapp/docs/sample.args delete mode 100755 testapp/docs/sample.sh delete mode 100644 testapp/docs/sample_questions.py delete mode 100644 testapp/docs/sample_questions.xml create mode 100644 testapp/exam/bash_files/sample.args create mode 100755 testapp/exam/bash_files/sample.sh delete mode 100644 testapp/exam/evaluate_bash.py create mode 100644 testapp/exam/evaluate_bash_code.py delete mode 100644 testapp/exam/evaluate_c.py create mode 100644 testapp/exam/evaluate_c_code.py create mode 100644 testapp/exam/evaluate_code.py delete mode 100644 testapp/exam/evaluate_cpp.py create mode 100644 testapp/exam/evaluate_cpp_code.py delete mode 100644 testapp/exam/evaluate_java.py create mode 100644 testapp/exam/evaluate_java_code.py delete mode 100644 testapp/exam/evaluate_python.py create mode 100644 testapp/exam/evaluate_python_code.py delete mode 100644 testapp/exam/evaluate_scilab.py create mode 100644 testapp/exam/evaluate_scilab_code.py create mode 100644 testapp/exam/language_registry.py delete mode 100644 testapp/exam/registry.py delete mode 100644 testapp/exam/test_code.py diff --git a/testapp/docs/sample.args b/testapp/docs/sample.args deleted file mode 100644 index 4d9f00d..0000000 --- a/testapp/docs/sample.args +++ /dev/null @@ -1,2 +0,0 @@ -1 2 -2 1 diff --git a/testapp/docs/sample.sh b/testapp/docs/sample.sh deleted file mode 100755 index e935cb3..0000000 --- a/testapp/docs/sample.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 )) diff --git a/testapp/docs/sample_questions.py b/testapp/docs/sample_questions.py deleted file mode 100644 index 60f32cb..0000000 --- a/testapp/docs/sample_questions.py +++ /dev/null @@ -1,84 +0,0 @@ -from datetime import date - -questions = [ -[Question( - summary='Factorial', - points=2, - language='python', - type='code', - description=''' -Write a function called fact which takes a single integer argument -(say n) and returns the factorial of the number. -For example:
-fact(3) -> 6 -''', - test=''' -assert fact(0) == 1 -assert fact(5) == 120 -''', - snippet="def fact(num):" - ), -#Add tags here as a list of string. -['Python','function','factorial'], -], - -[Question( - summary='Simple function', - points=1, - language='python', - type='code', - description='''Create a simple function called sqr which takes a single -argument and returns the square of the argument. For example:
-sqr(3) -> 9.''', - test=''' -import math -assert sqr(3) == 9 -assert abs(sqr(math.sqrt(2)) - 2.0) < 1e-14 - ''', - snippet="def sqr(num):" - ), -#Add tags here as a list of string. -['Python','function'], -], - -[Question( - summary='Bash addition', - points=2, - language='bash', - type='code', - description='''Write a shell script which takes two arguments on the - command line and prints the sum of the two on the output.''', - test='''\ -docs/sample.sh -docs/sample.args -''', - snippet="#!/bin/bash" - ), -#Add tags here as a list of string. -[''], -], - -[Question( - summary='Size of integer in Python', - points=0.5, - language='python', - type='mcq', - description='''What is the largest integer value that can be represented -in Python?''', - options='''No Limit -2**32 -2**32 - 1 -None of the above -''', - test = "No Limit" - ), -#Add tags here as a list of string. -['mcq'], -], - -] #list of questions ends here - -quiz = Quiz(start_date=date.today(), - duration=10, - description='Basic Python Quiz 1' - ) diff --git a/testapp/docs/sample_questions.xml b/testapp/docs/sample_questions.xml deleted file mode 100644 index 53c76f8..0000000 --- a/testapp/docs/sample_questions.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - -Factorial - - -Write a function called "fact" which takes a single integer argument (say "n") -and returns the factorial of the number. -For example fact(3) -> 6 - -2 -python - -assert fact(0) == 1 -assert fact(5) == 120 - - - - - - - -Simple function - - -Create a simple function called "sqr" which takes a single argument and -returns the square of the argument -For example sqr(3) -> 9. - -1 -python - -import math -assert sqr(3) == 9 -assert abs(sqr(math.sqrt(2)) - 2.0) < 1e-14 - - - - - - - diff --git a/testapp/exam/bash_files/sample.args b/testapp/exam/bash_files/sample.args new file mode 100644 index 0000000..4d9f00d --- /dev/null +++ b/testapp/exam/bash_files/sample.args @@ -0,0 +1,2 @@ +1 2 +2 1 diff --git a/testapp/exam/bash_files/sample.sh b/testapp/exam/bash_files/sample.sh new file mode 100755 index 0000000..e935cb3 --- /dev/null +++ b/testapp/exam/bash_files/sample.sh @@ -0,0 +1,2 @@ +#!/bin/bash +[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 )) diff --git a/testapp/exam/code_server.py b/testapp/exam/code_server.py index db30798..111562a 100755 --- a/testapp/exam/code_server.py +++ b/testapp/exam/code_server.py @@ -32,16 +32,17 @@ import json import importlib # Local imports. from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT -from registry import registry -from evaluate_python import EvaluatePython -from evaluate_c import EvaluateC -from evaluate_cpp import EvaluateCpp -from evaluate_java import EvaluateJava -from evaluate_scilab import EvaluateScilab -from evaluate_bash import EvaluateBash +from language_registry import registry +from evaluate_python_code import EvaluatePythonCode +from evaluate_c_code import EvaluateCCode +from evaluate_cpp_code import EvaluateCppCode +from evaluate_java_code import EvaluateJavaCode +from evaluate_scilab_code import EvaluateScilabCode +from evaluate_bash_code import EvaluateBashCode MY_DIR = abspath(dirname(__file__)) +## Private Protocol ########## def run_as_nobody(): """Runs the current process as nobody.""" # Set the effective uid and to that of nobody. @@ -60,10 +61,11 @@ class CodeServer(object): self.port = port self.queue = queue - def check_code(self, info_parameter, language, in_dir=None): - """Calls the TestCode SUb Class based on language to test the current code""" - evaluate_code_class = registry.get_class(language) - evaluate_code_instance = evaluate_code_class.from_json(info_parameter, language, in_dir) + ## Public Protocol ########## + def check_code(self, language, json_data, in_dir=None): + """Calls relevant EvaluateCode class based on language to check the answer code""" + evaluate_code_instance = self.create_class_instance(language, json_data, in_dir) + result = evaluate_code_instance.run_code() # Put us back into the server pool queue since we are free now. @@ -71,6 +73,14 @@ class CodeServer(object): return json.dumps(result) + ## Public Protocol ########## + def create_class_instance(self, language, json_data, in_dir): + """Create instance of relevant EvaluateCode class based on language""" + cls = registry.get_class(language) + instance = cls.from_json(language, json_data, in_dir) + return instance + + ## Public Protocol ########## def run(self): """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("localhost", self.port)) @@ -110,6 +120,8 @@ class ServerPool(object): p.start() self.servers = servers + ## Public Protocol ########## + def get_server_port(self): """Get available server port from ones in the pool. This will block till it gets an available server. diff --git a/testapp/exam/evaluate_bash.py b/testapp/exam/evaluate_bash.py deleted file mode 100644 index fd769cd..0000000 --- a/testapp/exam/evaluate_bash.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -import traceback -import pwd -import os -from os.path import join, isfile -import subprocess -import importlib - -# local imports -from test_code import TestCode -from registry import registry - - -class EvaluateBash(TestCode): - """Tests the Bash code obtained from Code Server""" - def evaluate_code(self): - fpath = self.create_submit_code_file('submit.sh') - submit_path = self.set_file_as_executable(fpath) - get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') - ref_path, test_case_path = self.set_test_code_file_path(get_ref_path, get_test_case_path) - success = False - - success, err = self.check_bash_script(ref_path, submit_path, - test_case_path) - - # Delete the created file. - os.remove(submit_path) - - return success, err - - def check_bash_script(self, ref_path, submit_path, - test_case_path=None): - """ Function validates student script using instructor script as - reference. Test cases can optionally be provided. The first argument - ref_path, is the path to instructor script, it is assumed to - have executable permission. The second argument submit_path, is - the path to the student script, it is assumed to have executable - permission. The Third optional argument is the path to test the - scripts. Each line in this file is a test case and each test case is - passed to the script as standard arguments. - - Returns - -------- - - returns (True, "Correct answer") : If the student script passes all - test cases/have same output, when compared to the instructor script - - returns (False, error_msg): If the student script fails a single - test/have dissimilar output, when compared to the instructor script. - - Returns (False, error_msg): If mandatory arguments are not files or if - the required permissions are not given to the file(s). - - """ - if not isfile(ref_path): - return False, "No file at %s or Incorrect path" % ref_path - if not isfile(submit_path): - return False, "No file at %s or Incorrect path" % submit_path - if not os.access(ref_path, os.X_OK): - return False, "Script %s is not executable" % ref_path - if not os.access(submit_path, os.X_OK): - return False, "Script %s is not executable" % submit_path - - if test_case_path is None or "": - ret = self.run_command(ref_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, inst_stdout, inst_stderr = ret - ret = self.run_command(submit_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdnt_stdout, stdnt_stderr = ret - if inst_stdout == stdnt_stdout: - return True, "Correct answer" - else: - err = "Error: expected %s, got %s" % (inst_stderr, - stdnt_stderr) - return False, err - else: - if not isfile(test_case_path): - return False, "No test case at %s" % test_case_path - if not os.access(ref_path, os.R_OK): - return False, "Test script %s, not readable" % test_case_path - valid_answer = True # We initially make it one, so that we can - # stop once a test case fails - loop_count = 0 # Loop count has to be greater than or - # equal to one. - # Useful for caching things like empty - # test files,etc. - test_cases = open(test_case_path).readlines() - num_lines = len(test_cases) - for test_case in test_cases: - loop_count += 1 - if valid_answer: - args = [ref_path] + [x for x in test_case.split()] - ret = self.run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, inst_stdout, inst_stderr = ret - args = [submit_path]+[x for x in test_case.split()] - ret = self.run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdnt_stdout, stdnt_stderr = ret - valid_answer = inst_stdout == stdnt_stdout - if valid_answer and (num_lines == loop_count): - return True, "Correct answer" - else: - err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, - stdnt_stdout+stdnt_stderr) - return False, err - -registry.register('bash', EvaluateBash) \ No newline at end of file diff --git a/testapp/exam/evaluate_bash_code.py b/testapp/exam/evaluate_bash_code.py new file mode 100644 index 0000000..49f20fa --- /dev/null +++ b/testapp/exam/evaluate_bash_code.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluateBashCode(EvaluateCode): + """Tests the Bash code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + submit_path = self.create_submit_code_file('submit.sh') + self.set_file_as_executable(submit_path) + get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') + get_ref_path = get_ref_path.strip() + get_test_case_path = get_test_case_path.strip() + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path, get_test_case_path) + + success, err = self._check_bash_script(ref_path, submit_path, + test_case_path) + + # Delete the created file. + os.remove(submit_path) + + return success, err + + ## Private Protocol ########## + def _check_bash_script(self, ref_path, submit_path, + test_case_path=None): + """ Function validates student script using instructor script as + reference. Test cases can optionally be provided. The first argument + ref_path, is the path to instructor script, it is assumed to + have executable permission. The second argument submit_path, is + the path to the student script, it is assumed to have executable + permission. The Third optional argument is the path to test the + scripts. Each line in this file is a test case and each test case is + passed to the script as standard arguments. + + Returns + -------- + + returns (True, "Correct answer") : If the student script passes all + test cases/have same output, when compared to the instructor script + + returns (False, error_msg): If the student script fails a single + test/have dissimilar output, when compared to the instructor script. + + Returns (False, error_msg): If mandatory arguments are not files or if + the required permissions are not given to the file(s). + + """ + if not isfile(ref_path): + return False, "No file at %s or Incorrect path" % ref_path + if not isfile(submit_path): + return False, "No file at %s or Incorrect path" % submit_path + if not os.access(ref_path, os.X_OK): + return False, "Script %s is not executable" % ref_path + if not os.access(submit_path, os.X_OK): + return False, "Script %s is not executable" % submit_path + + success = False + + if test_case_path is None or "": + ret = self.run_command(ref_path, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, inst_stdout, inst_stderr = ret + ret = self.run_command(submit_path, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdnt_stdout, stdnt_stderr = ret + if inst_stdout == stdnt_stdout: + return True, "Correct answer" + else: + err = "Error: expected %s, got %s" % (inst_stderr, + stdnt_stderr) + return False, err + else: + if not isfile(test_case_path): + return False, "No test case at %s" % test_case_path + if not os.access(ref_path, os.R_OK): + return False, "Test script %s, not readable" % test_case_path + valid_answer = True # We initially make it one, so that we can + # stop once a test case fails + loop_count = 0 # Loop count has to be greater than or + # equal to one. + # Useful for caching things like empty + # test files,etc. + test_cases = open(test_case_path).readlines() + num_lines = len(test_cases) + for test_case in test_cases: + loop_count += 1 + if valid_answer: + args = [ref_path] + [x for x in test_case.split()] + ret = self.run_command(args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, inst_stdout, inst_stderr = ret + args = [submit_path]+[x for x in test_case.split()] + ret = self.run_command(args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdnt_stdout, stdnt_stderr = ret + valid_answer = inst_stdout == stdnt_stdout + if valid_answer and (num_lines == loop_count): + return True, "Correct answer" + else: + err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, + stdnt_stdout+stdnt_stderr) + return False, err + + +registry.register('bash', EvaluateBashCode) \ No newline at end of file diff --git a/testapp/exam/evaluate_c.py b/testapp/exam/evaluate_c.py deleted file mode 100644 index 0700daa..0000000 --- a/testapp/exam/evaluate_c.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -import traceback -import pwd -import os -from os.path import join, isfile -import subprocess -import importlib - -# local imports -from test_code import TestCode -from registry import registry - - -class EvaluateC(TestCode): - """Tests the C code obtained from Code Server""" - def evaluate_code(self): - submit_path = self.create_submit_code_file('submit.c') - get_ref_path = self.ref_code_path - ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) - success = False - - # Set file paths - c_user_output_path = os.getcwd() + '/output' - c_ref_output_path = os.getcwd() + '/executable' - - # Set command variables - compile_command = 'g++ {0} -c -o {1}'.format(submit_path, - c_user_output_path) - compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, - c_user_output_path, - c_ref_output_path) - run_command_args = [c_ref_output_path] - remove_user_output = c_user_output_path - remove_ref_output = c_ref_output_path - - success, err = self.check_code(ref_path, submit_path, compile_command, - compile_main, run_command_args, - remove_user_output, remove_ref_output) - - # Delete the created file. - os.remove(submit_path) - - return success, err - - def check_code(self, ref_code_path, submit_code_path, compile_command, - compile_main, run_command_args, remove_user_output, - remove_ref_output): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - - if not isfile(ref_code_path): - return False, "No file at %s or Incorrect path" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s or Incorrect path' % submit_code_path - - success = False - # output_path = os.getcwd() + '/output' - ret = self.compile_command(compile_command) - proc, stdnt_stderr = ret - # if self.language == "java": - stdnt_stderr = self.remove_null_substitute_char(stdnt_stderr) - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - ret = self.compile_command(compile_main) - proc, main_err = ret - # if self.language == "java": - main_err = self.remove_null_substitute_char(main_err) - - if main_err == '': - ret = self.run_command(run_command_args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" - else: - err = stdout + "\n" + stderr - os.remove(remove_ref_output) - else: - err = "Error:" - try: - error_lines = main_err.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + main_err - os.remove(remove_user_output) - else: - err = "Compilation Error:" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + stdnt_stderr - - return success, err - - - def remove_null_substitute_char(self, string): - """Returns a string without any null and substitute characters""" - stripped = "" - for c in string: - if ord(c) is not 26 and ord(c) is not 0: - stripped = stripped + c - return ''.join(stripped) - -registry.register('c', EvaluateC) \ No newline at end of file diff --git a/testapp/exam/evaluate_c_code.py b/testapp/exam/evaluate_c_code.py new file mode 100644 index 0000000..d52beae --- /dev/null +++ b/testapp/exam/evaluate_c_code.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluateCCode(EvaluateCode): + """Tests the C code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + submit_path = self.create_submit_code_file('submit.c') + get_ref_path = self.ref_code_path + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) + success = False + + # Set file paths + c_user_output_path = os.getcwd() + '/output' + c_ref_output_path = os.getcwd() + '/executable' + + # Set command variables + compile_command = 'g++ {0} -c -o {1}'.format(submit_path, + c_user_output_path) + compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, + c_user_output_path, + c_ref_output_path) + run_command_args = [c_ref_output_path] + remove_user_output = c_user_output_path + remove_ref_output = c_ref_output_path + + success, err = self.check_code(ref_path, submit_path, compile_command, + compile_main, run_command_args, + remove_user_output, remove_ref_output) + + # Delete the created file. + os.remove(submit_path) + + return success, err + + ## Public Protocol ########## + def check_code(self, ref_code_path, submit_code_path, compile_command, + compile_main, run_command_args, remove_user_output, + remove_ref_output): + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + + """ + if not isfile(ref_code_path): + return False, "No file at %s or Incorrect path" % ref_code_path + if not isfile(submit_code_path): + return False, 'No file at %s or Incorrect path' % submit_code_path + + success = False + # output_path = os.getcwd() + '/output' + ret = self.compile_command(compile_command) + proc, stdnt_stderr = ret + # if self.language == "java": + stdnt_stderr = self.remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + ret = self.compile_command(compile_main) + proc, main_err = ret + # if self.language == "java": + main_err = self.remove_null_substitute_char(main_err) + + if main_err == '': + ret = self.run_command(run_command_args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" + else: + err = stdout + "\n" + stderr + os.remove(remove_ref_output) + else: + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(remove_user_output) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err + + + ## Public Protocol ########## + def remove_null_substitute_char(self, string): + """Returns a string without any null and substitute characters""" + stripped = "" + for c in string: + if ord(c) is not 26 and ord(c) is not 0: + stripped = stripped + c + return ''.join(stripped) + + +registry.register('c', EvaluateCCode) \ No newline at end of file diff --git a/testapp/exam/evaluate_code.py b/testapp/exam/evaluate_code.py new file mode 100644 index 0000000..161c1a2 --- /dev/null +++ b/testapp/exam/evaluate_code.py @@ -0,0 +1,186 @@ +import sys +from SimpleXMLRPCServer import SimpleXMLRPCServer +import pwd +import os +import stat +from os.path import isdir, dirname, abspath, join, isfile +import signal +from multiprocessing import Process, Queue +import subprocess +import re +import json +import importlib +# Local imports. +from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT + + +MY_DIR = abspath(dirname(__file__)) + +# Raised when the code times-out. +# c.f. http://pguides.net/python/timeout-a-function +class TimeoutException(Exception): + pass + +## Private Protocol ########## + +def timeout_handler(signum, frame): + """A handler for the ALARM signal.""" + raise TimeoutException('Code took too long to run.') + +def create_signal_handler(): + """Add a new signal handler for the execution of this code.""" + prev_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + return prev_handler + +def set_original_signal_handler(old_handler=None): + """Set back any original signal handler.""" + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + return + else: + raise Exception("Signal Handler: object cannot be NoneType") + +def delete_signal_handler(): + signal.alarm(0) + return + + + +############################################################################### +# `TestCode` class. +############################################################################### +class EvaluateCode(object): + """Tests the code obtained from Code Server""" + def __init__(self, test_case_data, language, user_answer, ref_code_path=None, in_dir=None): + msg = 'Code took more than %s seconds to run. You probably '\ + 'have an infinite loop in your code.' % SERVER_TIMEOUT + self.timeout_msg = msg + self.test_case_data = test_case_data + self.language = language.lower() + self.user_answer = user_answer + self.ref_code_path = ref_code_path + self.in_dir = in_dir + + ## Public Protocol ########## + + @classmethod + def from_json(cls, language, json_data, in_dir): + json_data = json.loads(json_data) + test_case_data = json_data.get("test_case_data") + user_answer = json_data.get("user_answer") + ref_code_path = json_data.get("ref_code_path") + + instance = cls(Test_case_data, language, user_answer, ref_code_path, in_dir) + return instance + + def run_code(self): + """Tests given code (`answer`) with the test cases based on + given arguments. + + The ref_code_path is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + If the optional `in_dir` keyword argument is supplied it changes the + directory to that directory (it does not change it back to the original + when done). + + Returns + ------- + + A tuple: (success, error message). + """ + self._change_dir(self.in_dir) + + # Add a new signal handler for the execution of this code. + prev_handler = create_signal_handler() + success = False + + # Do whatever testing needed. + try: + success, err = self.evaluate_code() + + except TimeoutException: + err = self.timeout_msg + except: + type, value = sys.exc_info()[:2] + err = "Error: {0}".format(repr(value)) + finally: + # Set back any original signal handler. + set_original_signal_handler(prev_handler) + + # Cancel the signal + delete_signal_handler() + + result = {'success': success, 'error': err} + return result + + def evaluate_code(self): + raise NotImplementedError("evaluate_code method not implemented") + + def create_submit_code_file(self, file_name): + """ Write the code (`answer`) to a file and set the file path""" + submit_f = open(file_name, 'w') + submit_f.write(self.user_answer.lstrip()) + submit_f.close() + submit_path = abspath(submit_f.name) + + return submit_path + + def set_file_as_executable(self, fname): + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + + def set_test_code_file_path(self, ref_path=None, test_case_path=None): + if ref_path and not ref_path.startswith('/'): + ref_path = join(MY_DIR, ref_path) + + if test_case_path and not test_case_path.startswith('/'): + test_case_path = join(MY_DIR, test_case_path) + + return ref_path, test_case_path + + def run_command(self, cmd_args, *args, **kw): + """Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stdout and stderr. + """ + try: + proc = subprocess.Popen(cmd_args, *args, **kw) + stdout, stderr = proc.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc.kill() + # Re-raise exception. + raise + return proc, stdout, stderr + + def compile_command(self, cmd, *args, **kw): + """Compiles C/C++/java code and returns errors if any. + Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stderr. + """ + try: + proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc_compile.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc_compile.kill() + # Re-raise exception. + raise + return proc_compile, err + + ## Private Protocol ########## + + def _change_dir(self, in_dir): + if in_dir is not None and isdir(in_dir): + os.chdir(in_dir) \ No newline at end of file diff --git a/testapp/exam/evaluate_cpp.py b/testapp/exam/evaluate_cpp.py deleted file mode 100644 index cffe744..0000000 --- a/testapp/exam/evaluate_cpp.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -import traceback -import pwd -import os -from os.path import join, isfile -import subprocess -import importlib - -# local imports -from evaluate_c import EvaluateC -from test_code import TestCode -from registry import registry - - - -class EvaluateCpp(EvaluateC, TestCode): - """Tests the C code obtained from Code Server""" - def evaluate_code(self): - submit_path = self.create_submit_code_file('submitstd.cpp') - get_ref_path = self.ref_code_path - ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) - success = False - - # Set file paths - c_user_output_path = os.getcwd() + '/output' - c_ref_output_path = os.getcwd() + '/executable' - - # Set command variables - compile_command = 'g++ {0} -c -o {1}'.format(submit_path, - c_user_output_path) - compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, - c_user_output_path, - c_ref_output_path) - run_command_args = c_ref_output_path - remove_user_output = c_user_output_path - remove_ref_output = c_ref_output_path - - success, err = self.check_code(ref_path, submit_path, compile_command, - compile_main, run_command_args, - remove_user_output, remove_ref_output) - - # Delete the created file. - os.remove(submit_path) - - return success, err - -registry.register('cpp', EvaluateCpp) \ No newline at end of file diff --git a/testapp/exam/evaluate_cpp_code.py b/testapp/exam/evaluate_cpp_code.py new file mode 100644 index 0000000..ed744d1 --- /dev/null +++ b/testapp/exam/evaluate_cpp_code.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from evaluate_c_code import EvaluateCCode +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluateCppCode(EvaluateCCode, EvaluateCode): + """Tests the C code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + submit_path = self.create_submit_code_file('submitstd.cpp') + get_ref_path = self.ref_code_path + ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) + success = False + + # Set file paths + c_user_output_path = os.getcwd() + '/output' + c_ref_output_path = os.getcwd() + '/executable' + + # Set command variables + compile_command = 'g++ {0} -c -o {1}'.format(submit_path, + c_user_output_path) + compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, + c_user_output_path, + c_ref_output_path) + run_command_args = c_ref_output_path + remove_user_output = c_user_output_path + remove_ref_output = c_ref_output_path + + success, err = self.check_code(ref_path, submit_path, compile_command, + compile_main, run_command_args, + remove_user_output, remove_ref_output) + + # Delete the created file. + os.remove(submit_path) + + return success, err + + +registry.register('cpp', EvaluateCppCode) \ No newline at end of file diff --git a/testapp/exam/evaluate_java.py b/testapp/exam/evaluate_java.py deleted file mode 100644 index d3d4e2a..0000000 --- a/testapp/exam/evaluate_java.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import traceback -import pwd -import os -from os.path import join, isfile -import subprocess -import importlib - -# local imports -from evaluate_c import EvaluateC -from test_code import TestCode -from registry import registry - - - -class EvaluateJava(EvaluateC, TestCode): - """Tests the C code obtained from Code Server""" - def evaluate_code(self): - submit_path = self.create_submit_code_file('Test.java') - get_ref_path = self.ref_code_path - ref_path, test_case_path = self.set_test_code_file_path(get_ref_path) - success = False - - # Set file paths - java_student_directory = os.getcwd() + '/' - java_ref_file_name = (ref_code_path.split('/')[-1]).split('.')[0] - - # Set command variables - compile_command = 'javac {0}'.format(submit_code_path), - compile_main = 'javac {0} -classpath {1} -d {2}'.format(ref_code_path, - java_student_directory, - java_student_directory) - run_command_args = "java -cp {0} {1}".format(java_student_directory, - java_ref_file_name) - remove_user_output = "{0}{1}.class".format(java_student_directory, - 'Test') - remove_ref_output = "{0}{1}.class".format(java_student_directory, - java_ref_file_name) - - success, err = self.check_code(ref_path, submit_path, compile_command, - compile_main, run_command_args, - remove_user_output, remove_ref_output) - - # Delete the created file. - os.remove(submit_path) - - return success, err - -registry.register('java', EvaluateJava) \ No newline at end of file diff --git a/testapp/exam/evaluate_java_code.py b/testapp/exam/evaluate_java_code.py new file mode 100644 index 0000000..9ddbbed --- /dev/null +++ b/testapp/exam/evaluate_java_code.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from evaluate_c_code import EvaluateCCode +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluateJavaCode(EvaluateCCode, EvaluateCode): + """Tests the C code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + submit_path = self.create_submit_code_file('Test.java') + ref_path, test_case_path = self.set_test_code_file_path(self.ref_code_path) + success = False + + # Set file paths + java_student_directory = os.getcwd() + '/' + java_ref_file_name = (ref_path.split('/')[-1]) #.split('.')[0] + + # Set command variables + compile_command = 'javac {0}'.format(submit_path), + compile_main = 'javac {0} -classpath {1} -d {2}'.format(ref_path, + java_student_directory, + java_student_directory) + run_command_args = "java -cp {0} {1}".format(java_student_directory, + java_ref_file_name) + remove_user_output = "{0}{1}.class".format(java_student_directory, + 'Test') + remove_ref_output = "{0}{1}.class".format(java_student_directory, + java_ref_file_name) + + success, err = self.check_code(ref_path, submit_path, compile_command, + compile_main, run_command_args, + remove_user_output, remove_ref_output) + + # Delete the created file. + os.remove(submit_path) + + return success, err + + +registry.register('java', EvaluateJavaCode) \ No newline at end of file diff --git a/testapp/exam/evaluate_python.py b/testapp/exam/evaluate_python.py deleted file mode 100644 index 3af82b6..0000000 --- a/testapp/exam/evaluate_python.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -import sys -import traceback -import os -from os.path import join -import importlib - -# local imports -from test_code import TestCode -from registry import registry - -class EvaluatePython(TestCode): - """Tests the Python code obtained from Code Server""" - # def evaluate_python_code(self): - def evaluate_code(self): - success = False - - try: - tb = None - test_code = self._create_test_case() - submitted = compile(self.user_answer, '', mode='exec') - g = {} - exec submitted in g - _tests = compile(test_code, '', mode='exec') - exec _tests in g - except AssertionError: - type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(test_code).splitlines()[lineno-1] - err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - else: - success = True - err = 'Correct answer' - - del tb - return success, err - - # Private Protocol - def _create_test_case(self): - """ - Create assert based test cases in python - """ - test_code = "" - for test_case in self.test_parameter: - pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) if test_case.get('pos_args') \ - else "" - kw_args = ", ".join(str(k+"="+a) for k, a in test_case.get('kw_args').iteritems()) \ - if test_case.get('kw_args') else "" - args = pos_args + ", " + kw_args if pos_args and kw_args else pos_args or kw_args - tcode = "assert {0}({1}) == {2}" \ - .format(test_case.get('func_name'), args, test_case.get('expected_answer')) - test_code += tcode + "\n" - return test_code - -registry.register('python', EvaluatePython) \ No newline at end of file diff --git a/testapp/exam/evaluate_python_code.py b/testapp/exam/evaluate_python_code.py new file mode 100644 index 0000000..2682897 --- /dev/null +++ b/testapp/exam/evaluate_python_code.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib + +# local imports +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluatePythonCode(EvaluateCode): + """Tests the Python code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + success = False + + try: + tb = None + test_code = self._create_test_case() + submitted = compile(self.user_answer, '', mode='exec') + g = {} + exec submitted in g + _tests = compile(test_code, '', mode='exec') + exec _tests in g + except AssertionError: + type, value, tb = sys.exc_info() + info = traceback.extract_tb(tb) + fname, lineno, func, text = info[-1] + text = str(test_code).splitlines()[lineno-1] + err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + else: + success = True + err = 'Correct answer' + + del tb + return success, err + + ## Private Protocol ########## + def _create_test_case(self): + """ + Create assert based test cases in python + """ + test_code = "" + for test_case in self.test_case_data: + pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) if test_case.get('pos_args') \ + else "" + kw_args = ", ".join(str(k+"="+a) for k, a in test_case.get('kw_args').iteritems()) \ + if test_case.get('kw_args') else "" + args = pos_args + ", " + kw_args if pos_args and kw_args else pos_args or kw_args + tcode = "assert {0}({1}) == {2}" \ + .format(test_case.get('func_name'), args, test_case.get('expected_answer')) + test_code += tcode + "\n" + return test_code + + +registry.register('python', EvaluatePythonCode) \ No newline at end of file diff --git a/testapp/exam/evaluate_scilab.py b/testapp/exam/evaluate_scilab.py deleted file mode 100644 index f4253ff..0000000 --- a/testapp/exam/evaluate_scilab.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -import traceback -import os -from os.path import join, isfile -import subprocess -import re -import importlib - -# local imports -from test_code import TestCode -from registry import registry - - -class EvaluateScilab(TestCode): - """Tests the Scilab code obtained from Code Server""" - # def evaluate_scilab_code(self): - def evaluate_code(self): - submit_path = self.create_submit_code_file('function.sci') - ref_path, test_case_path = self.set_test_code_file_path() - success = False - - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) - cmd += ' | timeout 8 scilab-cli -nb' - ret = self.run_command(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - - # Get only the error. - stderr = self._get_error(stdout) - if stderr is None: - # Clean output - stdout = self._strip_output(stdout) - if proc.returncode == 5: - success, err = True, "Correct answer" - else: - err = add_err + stdout - else: - err = add_err + stderr - - # Delete the created file. - os.remove(submit_path) - - return success, err - - # Private Protocol - def _remove_scilab_exit(self, string): - """ - Removes exit, quit and abort from the scilab code - """ - new_string = "" - i=0 - for line in string.splitlines(): - new_line = re.sub(r"exit.*$","",line) - new_line = re.sub(r"quit.*$","",new_line) - new_line = re.sub(r"abort.*$","",new_line) - if line != new_line: - i=i+1 - new_string = new_string +'\n'+ new_line - return new_string, i - - def _get_error(self, string): - """ - Fetches only the error from the string. - Returns None if no error. - """ - obj = re.search("!.+\n.+",string); - if obj: - return obj.group() - return None - - def _strip_output(self, out): - """ - Cleans whitespace from the output - """ - strip_out = "Message" - for l in out.split('\n'): - if l.strip(): - strip_out = strip_out+"\n"+l.strip() - return strip_out - -registry.register('scilab', EvaluateScilab) \ No newline at end of file diff --git a/testapp/exam/evaluate_scilab_code.py b/testapp/exam/evaluate_scilab_code.py new file mode 100644 index 0000000..cc46605 --- /dev/null +++ b/testapp/exam/evaluate_scilab_code.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +import traceback +import os +from os.path import join, isfile +import subprocess +import re +import importlib + +# local imports +from evaluate_code import EvaluateCode +from language_registry import registry + + +class EvaluateScilabCode(EvaluateCode): + """Tests the Scilab code obtained from Code Server""" + ## Public Protocol ########## + def evaluate_code(self): + submit_path = self.create_submit_code_file('function.sci') + ref_path, test_case_path = self.set_test_code_file_path() + success = False + + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) + cmd += ' | timeout 8 scilab-cli -nb' + ret = self.run_command(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + + # Get only the error. + stderr = self._get_error(stdout) + if stderr is None: + # Clean output + stdout = self._strip_output(stdout) + if proc.returncode == 5: + success, err = True, "Correct answer" + else: + err = add_err + stdout + else: + err = add_err + stderr + + # Delete the created file. + os.remove(submit_path) + + return success, err + + ## Private Protocol ########## + def _remove_scilab_exit(self, string): + """ + Removes exit, quit and abort from the scilab code + """ + new_string = "" + i=0 + for line in string.splitlines(): + new_line = re.sub(r"exit.*$","",line) + new_line = re.sub(r"quit.*$","",new_line) + new_line = re.sub(r"abort.*$","",new_line) + if line != new_line: + i=i+1 + new_string = new_string +'\n'+ new_line + return new_string, i + + ## Private Protocol ########## + def _get_error(self, string): + """ + Fetches only the error from the string. + Returns None if no error. + """ + obj = re.search("!.+\n.+",string); + if obj: + return obj.group() + return None + + ## Private Protocol ########## + def _strip_output(self, out): + """ + Cleans whitespace from the output + """ + strip_out = "Message" + for l in out.split('\n'): + if l.strip(): + strip_out = strip_out+"\n"+l.strip() + return strip_out + + +registry.register('scilab', EvaluateScilabCode) \ No newline at end of file diff --git a/testapp/exam/language_registry.py b/testapp/exam/language_registry.py new file mode 100644 index 0000000..207cd21 --- /dev/null +++ b/testapp/exam/language_registry.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +class LanguageRegistry(object): + def __init__(self): + self._registry = {} + + ## Public Protocol ########## + def get_class(self, language): + return self._registry[language] + + ## Public Protocol ########## + def register(self, language, cls): + self._registry[language] = cls + + +registry = LanguageRegistry() \ No newline at end of file diff --git a/testapp/exam/models.py b/testapp/exam/models.py index d0c9cc2..51e773a 100644 --- a/testapp/exam/models.py +++ b/testapp/exam/models.py @@ -88,17 +88,17 @@ class Question(models.Model): tags = TaggableManager() def consolidate_answer_data(self, test_cases, user_answer): - test_case_parameter = [] - info_parameter = {} + test_case_data_dict = [] + question_info_dict = {} for test_case in test_cases: kw_args_dict = {} pos_args_list = [] - parameter_dict = {} - parameter_dict['test_id'] = test_case.id - parameter_dict['func_name'] = test_case.func_name - parameter_dict['expected_answer'] = test_case.expected_answer + test_case_data = {} + test_case_data['test_id'] = test_case.id + test_case_data['func_name'] = test_case.func_name + test_case_data['expected_answer'] = test_case.expected_answer if test_case.kw_args: for args in test_case.kw_args.split(","): @@ -109,17 +109,17 @@ class Question(models.Model): for args in test_case.pos_args.split(","): pos_args_list.append(args.strip()) - parameter_dict['kw_args'] = kw_args_dict - parameter_dict['pos_args'] = pos_args_list - test_case_parameter.append(parameter_dict) + test_case_data['kw_args'] = kw_args_dict + test_case_data['pos_args'] = pos_args_list + test_case_data_dict.append(test_case_data) - # info_parameter['language'] = self.language - info_parameter['id'] = self.id - info_parameter['user_answer'] = user_answer - info_parameter['test_parameter'] = test_case_parameter - info_parameter['ref_code_path'] = self.ref_code_path + # question_info_dict['language'] = self.language + question_info_dict['id'] = self.id + question_info_dict['user_answer'] = user_answer + question_info_dict['test_parameter'] = test_case_data_dict + question_info_dict['ref_code_path'] = self.ref_code_path - return json.dumps(info_parameter) + return json.dumps(question_info_dict) def __unicode__(self): return self.summary diff --git a/testapp/exam/registry.py b/testapp/exam/registry.py deleted file mode 100644 index ef995ac..0000000 --- a/testapp/exam/registry.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -class Registry(object): - def __init__(self): - self._registry = {} - - def get_class(self, language): - return self._registry[language] - - def register(self, language, cls): - self._registry[language] = cls - - -registry = Registry() \ No newline at end of file diff --git a/testapp/exam/test_code.py b/testapp/exam/test_code.py deleted file mode 100644 index 8930f55..0000000 --- a/testapp/exam/test_code.py +++ /dev/null @@ -1,182 +0,0 @@ -import sys -from SimpleXMLRPCServer import SimpleXMLRPCServer -import pwd -import os -import stat -from os.path import isdir, dirname, abspath, join, isfile -import signal -from multiprocessing import Process, Queue -import subprocess -import re -import json -import importlib -# Local imports. -from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT - - -MY_DIR = abspath(dirname(__file__)) - -# Raised when the code times-out. -# c.f. http://pguides.net/python/timeout-a-function -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - """A handler for the ALARM signal.""" - raise TimeoutException('Code took too long to run.') - -def create_signal_handler(): - """Add a new signal handler for the execution of this code.""" - prev_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - return prev_handler - -def set_original_signal_handler(old_handler=None): - """Set back any original signal handler.""" - if old_handler is not None: - signal.signal(signal.SIGALRM, old_handler) - return - else: - raise Exception("Signal Handler: object cannot be NoneType") - -def delete_signal_handler(): - signal.alarm(0) - return - - - -############################################################################### -# `TestCode` class. -############################################################################### -class TestCode(object): - """Tests the code obtained from Code Server""" - def __init__(self, test_parameter, language, user_answer, ref_code_path=None, in_dir=None): - msg = 'Code took more than %s seconds to run. You probably '\ - 'have an infinite loop in your code.' % SERVER_TIMEOUT - self.timeout_msg = msg - self.test_parameter = test_parameter - self.language = language.lower() - self.user_answer = user_answer - self.ref_code_path = ref_code_path - self.in_dir = in_dir - - @classmethod - def from_json(cls, blob, language, in_dir): - info_parameter = json.loads(blob) - test_parameter = info_parameter.get("test_parameter") - user_answer = info_parameter.get("user_answer") - ref_code_path = info_parameter.get("ref_code_path") - - instance = cls(test_parameter, language, user_answer, ref_code_path, in_dir) - return instance - - def run_code(self): - """Tests given code (`answer`) with the test cases based on - given arguments. - - The ref_code_path is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - """ - self._change_dir(self.in_dir) - - # Add a new signal handler for the execution of this code. - prev_handler = create_signal_handler() - success = False - - # Do whatever testing needed. - try: - success, err = self.evaluate_code() - - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - set_original_signal_handler(prev_handler) - - # Cancel the signal - delete_signal_handler() - - result = {'success': success, 'error': err} - return result - - def evaluate_code(self): - raise NotImplementedError("evaluate_code method not implemented") - - def create_submit_code_file(self, file_name): - """ Write the code (`answer`) to a file and set the file path""" - # File name/extension depending on the question language - submit_f = open(file_name, 'w') - submit_f.write(self.user_answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - - return submit_path - - def set_file_as_executable(self, fname): - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) - - def set_test_code_file_path(self, ref_path=None, test_case_path=None): - if ref_path and not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - - if test_case_path and not test_case_path.startswith('/'): - test_case_path = join(MY_DIR, test_case_path) - - return ref_path, test_case_path - - def run_command(self, cmd_args, *args, **kw): - """Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stdout and stderr. - """ - try: - proc = subprocess.Popen(cmd_args, *args, **kw) - stdout, stderr = proc.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc.kill() - # Re-raise exception. - raise - return proc, stdout, stderr - - def compile_command(self, cmd, *args, **kw): - """Compiles C/C++/java code and returns errors if any. - Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stderr. - """ - try: - proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc_compile.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc_compile.kill() - # Re-raise exception. - raise - return proc_compile, err - - def _change_dir(self, in_dir): - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) \ No newline at end of file diff --git a/testapp/exam/views.py b/testapp/exam/views.py index 508b623..2542a28 100644 --- a/testapp/exam/views.py +++ b/testapp/exam/views.py @@ -286,7 +286,6 @@ def edit_question(request): user = request.user if not user.is_authenticated() or not is_moderator(user): raise Http404('You are not allowed to view this page!') - question_list = request.POST.getlist('questions') summary = request.POST.getlist('summary') description = request.POST.getlist('description') @@ -311,6 +310,8 @@ def edit_question(request): question.active = active[j] question.language = language[j] question.snippet = snippet[j] + question.ref_code_path = ref_code_path[j] + question.solution = solution[j] question.type = type[j] question.save() return my_redirect("/exam/manage/questions") @@ -374,6 +375,8 @@ def add_question(request, question_id=None): d.active = form['active'].data d.language = form['language'].data d.snippet = form['snippet'].data + d.ref_code_path = form['ref_code_path'].data + d.solution = form['solution'].data d.save() question = Question.objects.get(id=question_id) for tag in question.tags.all(): @@ -428,6 +431,8 @@ def add_question(request, question_id=None): form.initial['active'] = d.active form.initial['language'] = d.language form.initial['snippet'] = d.snippet + form.initial['ref_code_path'] = d.ref_code_path + form.initial['solution'] = d.solution form_tags = d.tags.all() form_tags_split = form_tags.values('name') initial_tags = "" @@ -953,9 +958,9 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): # questions, we obtain the results via XML-RPC with the code executed # safely in a separate process (the code_server.py) running as nobody. if not question.type == 'upload': - info_parameter = question.consolidate_answer_data(test, user_answer) \ - if question.type == 'code' else None - correct, result = validate_answer(user, user_answer, question, info_parameter) + json_data = question.consolidate_answer_data(test, user_answer) \ + if question.type == 'code' else None + correct, result = validate_answer(user, user_answer, question, json_data) if correct: new_answer.correct = correct new_answer.marks = question.points @@ -1000,7 +1005,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): questionpaper_id, success_msg) -def validate_answer(user, user_answer, question, info_parameter=None): +def validate_answer(user, user_answer, question, json_data=None): """ Checks whether the answer submitted by the user is right or wrong. If right then returns correct = True, success and @@ -1025,7 +1030,7 @@ def validate_answer(user, user_answer, question, info_parameter=None): message = 'Correct answer' elif question.type == 'code': user_dir = get_user_dir(user) - json_result = code_server.run_code(info_parameter, question.language, user_dir) + json_result = code_server.run_code(question.language, json_data, user_dir) result = json.loads(json_result) if result.get('success'): correct = True @@ -1247,6 +1252,8 @@ def show_all_questions(request): form.initial['active'] = d.active form.initial['language'] = d.language form.initial['snippet'] = d.snippet + form.initial['ref_code_path'] = d.ref_code_path + form.initial['solution'] = d.solution form_tags = d.tags.all() form_tags_split = form_tags.values('name') initial_tags = "" diff --git a/testapp/exam/xmlrpc_clients.py b/testapp/exam/xmlrpc_clients.py index 5d95cae..8f5642e 100644 --- a/testapp/exam/xmlrpc_clients.py +++ b/testapp/exam/xmlrpc_clients.py @@ -22,7 +22,7 @@ class CodeServerProxy(object): pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT) self.pool_server = ServerProxy(pool_url) - def run_code(self, info_parameter, language, user_dir): + def run_code(self, language, json_data, user_dir): """Tests given code (`answer`) with the `test_code` supplied. If the optional `in_dir` keyword argument is supplied it changes the directory to that directory (it does not change it back to the original when @@ -31,7 +31,7 @@ class CodeServerProxy(object): Parameters ---------- - info_parameter contains; + json_data contains; user_answer : str The user's answer for the question. test_code : str @@ -50,7 +50,7 @@ class CodeServerProxy(object): try: server = self._get_server() - result = server.check_code(info_parameter, language, user_dir) + result = server.check_code(language, json_data, user_dir) except ConnectionError: result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'}) return result diff --git a/testapp/test_server.py b/testapp/test_server.py index c35c411..39995e1 100644 --- a/testapp/test_server.py +++ b/testapp/test_server.py @@ -1,97 +1,244 @@ import unittest import os -from exam import evaluate_c, evaluate_cpp, evaluate_bash, evaluate_python +from exam import evaluate_c_code, evaluate_cpp_code, evaluate_java_code, evaluate_python_code, evaluate_scilab_code, evaluate_bash_code +from exam.language_registry import registry +from exam.settings import SERVER_TIMEOUT -class TestPythonEvaluation(unittest.TestCase): +class RegistryTestCase(unittest.TestCase): + def setUp(self): + self.registry_object = registry + + def test_set_register(self): + self.registry_object.register("demo_language", "demo_object") + self.assertEquals(self.registry_object._registry["demo_language"], "demo_object") + + def test_get_class(self): + self.test_set_register() + cls = self.registry_object.get_class("demo_language") + self.assertEquals(cls, "demo_object") + + +############################################################################### +class PythonEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "Python" - self.test_parameter = [{"func_name": "add", + self.test_case_data = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + def test_correct_answer(self): user_answer = "def add(a, b):\n\treturn a + b""" - get_class = evaluate_python.EvaluatePython(self.test_parameter, self.language, user_answer, ref_code_path=None, in_dir=None) + get_class = evaluate_python_code.EvaluatePythonCode(self.test_case_data, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.run_code() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") def test_incorrect_answer(self): user_answer = "def add(a, b):\n\treturn a - b" - test_parameter = [{"func_name": "add", + test_case_data = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] - get_class = evaluate_python.EvaluatePython(self.test_parameter, self.language, user_answer, ref_code_path=None, in_dir=None) + get_class = evaluate_python_code.EvaluatePythonCode(self.test_case_data, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.run_code() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass""" - test_parameter = [{"func_name": "add", + test_case_data = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] - get_class = evaluate_python.EvaluatePython(self.test_parameter, self.language, user_answer, ref_code_path=None, in_dir=None) + get_class = evaluate_python_code.EvaluatePythonCode(self.test_case_data, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.run_code() self.assertFalse(result.get("success")) - self.assertTrue("Code took more than" in result.get("error")) + self.assertEquals(result.get("error"), self.timeout_msg) + ############################################################################### -class TestCEvaluation(unittest.TestCase): +class CEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "C" self.ref_code_path = "c_cpp_files/main.cpp" self.in_dir = "/tmp" - self.test_parameter = [] + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = evaluate_c.EvaluateC(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) + get_class = evaluate_c_code.EvaluateCCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") - def test_incorrect_answer(self): + def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = evaluate_c.EvaluateC(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) + get_class = evaluate_c_code.EvaluateCCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) + def test_infinite_loop(self): + user_answer = "int add(int a, int b)\n{while(1>0){}}" + get_class = evaluate_c_code.EvaluateCCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + ############################################################################### -class TestCPPEvaluation(unittest.TestCase): +class CppEvaluationTestCases(unittest.TestCase): def setUp(self): self.language = "CPP" self.ref_code_path = "c_cpp_files/main.cpp" self.in_dir = "/tmp" - self.test_parameter = [] + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = evaluate_cpp.EvaluateCpp(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) + get_class = evaluate_cpp_code.EvaluateCppCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") - def test_incorrect_answer(self): + def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = evaluate_cpp.EvaluateCpp(self.test_parameter, self.language, user_answer, self.ref_code_path, self.in_dir) + get_class = evaluate_cpp_code.EvaluateCppCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) result = get_class.run_code() - error_msg = "" self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) + def test_infinite_loop(self): + user_answer = "int add(int a, int b)\n{while(1>0){}}" + get_class = evaluate_cpp_code.EvaluateCppCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + +############################################################################### +class BashEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "bash" + self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + get_class = evaluate_bash_code.EvaluateBashCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" + get_class = evaluate_bash_code.EvaluateBashCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + get_class = evaluate_bash_code.EvaluateBashCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + +############################################################################### +class JavaEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "java" + self.ref_code_path = "java_files/main_square.java" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" + get_class = evaluate_java_code.EvaluateJavaCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" + get_class = evaluate_java_code.EvaluateJavaCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" + get_class = evaluate_java_code.EvaluateJavaCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + +############################################################################### +class ScilabEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "scilab" + self.ref_code_path = "scilab_files/test_add.sce" + self.in_dir = "/tmp" + self.test_case_data = [] + + def test_correct_answer(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" + get_class = evaluate_scilab_code.EvaluateScilabCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_correct_answer_2(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction" + get_class = evaluate_scilab_code.EvaluateScilabCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\t c=a+b;\ndis(\tendfunction" + get_class = evaluate_java_code.EvaluateJavaCode(self.test_case_data, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.run_code() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + + if __name__ == '__main__': unittest.main() \ No newline at end of file -- cgit