From 2c8176e4a37ca3a3ecc900bc011da894a90ce801 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Mon, 3 Mar 2014 17:54:18 +0530 Subject: Autochecks scilab function. A method is created that checks the scilab function submitted by the student. The method removes the terminating commands in scilab. If errors are present then it returns error, else provides input to the function and checks for the expected output. If function returns correct output for all the inputs then the function is graded correct, else throws the error message. --- testapp/code_server.py | 128 +++++++++++++++++++++++++++++++++++++- testapp/exam/forms.py | 1 + testapp/exam/models.py | 1 + testapp/exam/xmlrpc_clients.py | 1 + testapp/scilab_files/test_add.sce | 29 +++++++++ 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 testapp/scilab_files/test_add.sce (limited to 'testapp') diff --git a/testapp/code_server.py b/testapp/code_server.py index 8b3f8f1..c0b92b1 100755 --- a/testapp/code_server.py +++ b/testapp/code_server.py @@ -28,7 +28,7 @@ from os.path import isdir, dirname, abspath, join, isfile import signal from multiprocessing import Process, Queue import subprocess - +import re # Local imports. from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT @@ -659,6 +659,132 @@ class CodeServer(object): if ord(c) is not 26 and ord(c) is not 0: stripped = stripped + c return ''.join(stripped) + + def run_scilab_code(self, answer, test_code, in_dir=None): + """Tests given Scilab function (`answer`) with the `test_code` + supplied. If the optional `in_dir` keyword argument is supplied + it changes the directory to that directory (it does not change + it back to the original when done). This function also timesout + when the function takes more than SERVER_TIMEOUT seconds to run + to prevent runaway code. + + The testcode is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + Returns + ------- + + A tuple: (success, error message). + + """ + if in_dir is not None and isdir(in_dir): + os.chdir(in_dir) + + # Removes all the commands that terminates scilab + answer,i = self._remove_scilab_exit(answer.lstrip()) + + # Throw message if there are commmands that terminates scilab + add_err="" + if i > 0: + add_err = "Please do not use exit, quit and abort commands in your\ + code.\n Otherwise your code will not be evaluated\ + correctly.\n" + + # The file extension should be .sci + submit_f = open('function.sci','w') + submit_f.write(answer) + submit_f.close() + submit_path = abspath(submit_f.name) + + ref_path = test_code.strip() + if not ref_path.startswith('/'): + ref_path = join(MY_DIR, ref_path) + + # Add a new signal handler for the execution of this code. + old_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + + # Do whatever testing needed. + success = False + try: + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) + cmd += ' | scilab-cli -nb' + ret = self._run_command(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + + # Get only the error. + stderr = self._get_error(stdout) + if stderr is None: + # Clean output + stdout = self._strip_output(stdout) + if proc.returncode == 5: + success, err = True, "Correct answer" + else: + err = add_err + stdout + else: + err = add_err + stderr + except TimeoutException: + err = self.timeout_msg + except: + type, value = sys.exc_info()[:2] + err = "Error: {0}".format(repr(value)) + finally: + # Set back any original signal handler. + signal.signal(signal.SIGALRM, old_handler) + + # Delete the created file. + os.remove(submit_path) + + # Cancel the signal if any, see signal.alarm documentation. + signal.alarm(0) + + # Put us back into the server pool queue since we are free now. + self.queue.put(self.port) + + return success, err + + def _remove_scilab_exit(self, string): + """ + Removes exit, quit and abort from the scilab code + """ + new_string = "" + i=0 + for line in string.splitlines(): + new_line = re.sub(r"exit.*$","",line) + new_line = re.sub(r"quit.*$","",new_line) + new_line = re.sub(r"abort.*$","",new_line) + if line != new_line: + i=i+1 + new_string = new_string +'\n'+ new_line + return new_string, i + + def _get_error(self, string): + """ + Fetches only the error from the string. + Returns None if no error. + """ + obj = re.search("!.+\n.+",string); + if obj: + return obj.group() + return None + + def _strip_output(self, out): + """ + Cleans whitespace from the output + """ + strip_out = "Message" + for l in out.split('\n'): + if l.strip(): + strip_out = strip_out+"\n"+l.strip() + return strip_out def run(self): """Run XMLRPC server, serving our methods. diff --git a/testapp/exam/forms.py b/testapp/exam/forms.py index 917bea7..26d9a2d 100644 --- a/testapp/exam/forms.py +++ b/testapp/exam/forms.py @@ -19,6 +19,7 @@ QUESTION_TYPE_CHOICES = ( ("C", "C Language"), ("C++", "C++ Language"), ("java", "Java Language"), + ("scilab", "Scilab"), ) UNAME_CHARS = letters + "._" + digits diff --git a/testapp/exam/models.py b/testapp/exam/models.py index babde0f..8718968 100644 --- a/testapp/exam/models.py +++ b/testapp/exam/models.py @@ -21,6 +21,7 @@ QUESTION_TYPE_CHOICES = ( ("C", "C Language"), ("C++", "C++ Language"), ("java", "Java Language"), + ("scilab", "Scilab"), ) ################################################################################ diff --git a/testapp/exam/xmlrpc_clients.py b/testapp/exam/xmlrpc_clients.py index cc21e62..14ebf27 100644 --- a/testapp/exam/xmlrpc_clients.py +++ b/testapp/exam/xmlrpc_clients.py @@ -26,6 +26,7 @@ class CodeServerProxy(object): "C": "run_c_code", "C++": "run_cplus_code", "java": "run_java_code", + "scilab": "run_scilab_code", } def run_code(self, answer, test_code, user_dir, language): diff --git a/testapp/scilab_files/test_add.sce b/testapp/scilab_files/test_add.sce new file mode 100644 index 0000000..a317cdb --- /dev/null +++ b/testapp/scilab_files/test_add.sce @@ -0,0 +1,29 @@ +mode(-1) +exec("function.sci",-1); +i = 0 +p = add(3,5); +correct = (p == 8); +if correct then + i=i+1 +end +disp("Input submitted 3 and 5") +disp("Expected output 8 got " + string(p)) +p = add(22,-20); +correct = (p==2); +if correct then + i=i+1 +end +disp("Input submitted 22 and -20") +disp("Expected output 2 got " + string(p)) +p =add(91,0); +correct = (p==91); +if correct then + i=i+1 +end +disp("Input submitted 91 and 0") +disp("Expected output 91 got " + string(p)) +if i==3 then + exit(5); +else + exit(3); +end -- cgit From b46364ad0173f59544cb42b97c42136392f805e6 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 13 Mar 2014 15:54:23 +0530 Subject: Code to test scilab in test_server.py included timeout with scilab command to prevent scilab from running scilab infinitely. --- testapp/code_server.py | 2 +- testapp/test_server.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) (limited to 'testapp') diff --git a/testapp/code_server.py b/testapp/code_server.py index c0b92b1..7d9ebd2 100755 --- a/testapp/code_server.py +++ b/testapp/code_server.py @@ -713,7 +713,7 @@ class CodeServer(object): success = False try: cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) - cmd += ' | scilab-cli -nb' + cmd += ' | timeout 8 scilab-cli -nb' ret = self._run_command(cmd, shell=True, stdout=subprocess.PIPE, diff --git a/testapp/test_server.py b/testapp/test_server.py index 2a17739..31be855 100644 --- a/testapp/test_server.py +++ b/testapp/test_server.py @@ -16,7 +16,6 @@ def check_result(result, check='correct answer'): assert result[0], result[1] assert check in result[1].lower(), result[1] - def test_python(): """Test if server runs Python code as expected.""" src = 'while True: pass' @@ -208,6 +207,40 @@ def test_java(): '/tmp', language="java") check_result(result, 'error') +def test_scilab(): + """Test if server runs java code as expected.""" + src = """ + funcprot(0) +function[c]=add(a,b) + c=a+b; +endfunction + """ + result = code_server.run_code(src, 'scilab_files/test_add.sce', + '/tmp', language="scilab") + check_result(result, 'correct answer') + + src = """ + funcprot(0) +function[c]=add(a,b) + c=a+b; +dis( +endfunction + """ + result = code_server.run_code(src, 'scilab_files/test_add.sce', + '/tmp', language="scilab") + check_result(result, 'error') + + src = """ + funcprot(0) +function[c]=add(a,b) + c=a + while(1==1) + end +endfunction + """ + result = code_server.run_code(src, 'scilab_files/test_add.sce', + '/tmp', language="scilab") + check_result(result, 'error') def test_bash(): """Test if server runs Bash code as expected.""" @@ -257,3 +290,4 @@ if __name__ == '__main__': test_c() test_cpp() test_java() + test_scilab() -- cgit From 8f80359989640e77797fedf9f28f981058f47e12 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 13 Mar 2014 16:36:57 +0530 Subject: test condition added to test_server.py --- testapp/test_server.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'testapp') diff --git a/testapp/test_server.py b/testapp/test_server.py index 31be855..b639620 100644 --- a/testapp/test_server.py +++ b/testapp/test_server.py @@ -219,6 +219,16 @@ endfunction '/tmp', language="scilab") check_result(result, 'correct answer') + src = """ + funcprot(0) +function[c]=add(a,b) + c=a-b; +endfunction + """ + result = code_server.run_code(src, 'scilab_files/test_add.sce', + '/tmp', language="scilab") + check_result(result, 'correct answer') + src = """ funcprot(0) function[c]=add(a,b) -- cgit From 7481ac647eb08b29636041ce717fe7ac512a7562 Mon Sep 17 00:00:00 2001 From: prathamesh Date: Thu, 5 Jun 2014 18:44:08 +0530 Subject: Fixed indentation. --- testapp/exam/forms.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'testapp') diff --git a/testapp/exam/forms.py b/testapp/exam/forms.py index 26d9a2d..b606224 100644 --- a/testapp/exam/forms.py +++ b/testapp/exam/forms.py @@ -13,13 +13,13 @@ from string import letters, punctuation, digits import datetime QUESTION_TYPE_CHOICES = ( - ("python", "Python"), - ("bash", "Bash"), - ("mcq", "MCQ"), - ("C", "C Language"), - ("C++", "C++ Language"), - ("java", "Java Language"), - ("scilab", "Scilab"), + ("python", "Python"), + ("bash", "Bash"), + ("mcq", "MCQ"), + ("C", "C Language"), + ("C++", "C++ Language"), + ("java", "Java Language"), + ("scilab", "Scilab"), ) UNAME_CHARS = letters + "._" + digits -- cgit From 83b3d80a2cdc2a342a521fe71cad72e081d6df5f Mon Sep 17 00:00:00 2001 From: prathamesh Date: Fri, 6 Jun 2014 00:16:05 +0530 Subject: minor change --- testapp/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'testapp') diff --git a/testapp/test_server.py b/testapp/test_server.py index 95dc7b5..d22a022 100644 --- a/testapp/test_server.py +++ b/testapp/test_server.py @@ -207,7 +207,7 @@ def test_java(): check_result(result, 'error') def test_scilab(): - """Test if server runs java code as expected.""" + """Test if server runs scilab code as expected.""" src = """ funcprot(0) function[c]=add(a,b) -- cgit