Autochecks scilab function.

A method is created that checks the scilab function submitted by the
student.
The method removes the terminating commands in scilab.
If errors are present then it returns error,
else provides input to the function and checks for the
expected output.
If function returns correct output for all the inputs then
the function is graded correct, else throws the error message.

5 files changed, 159 insertions, 1 deletions

diff --git a/testapp/code_server.py b/testapp/code_server.py
index 8b3f8f1..c0b92b1 100755
--- a/testapp/code_server.py
+++ b/testapp/code_server.py
@@ -28,7 +28,7 @@ from os.path import isdir, dirname, abspath, join, isfile
 import signal
 from multiprocessing import Process, Queue
 import subprocess
-
+import re
 # Local imports.
 from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT
 
@@ -659,6 +659,132 @@ class CodeServer(object):
             if ord(c) is not 26 and ord(c) is not 0:
                 stripped = stripped + c
         return ''.join(stripped)
+ 
+    def run_scilab_code(self, answer, test_code, in_dir=None):
+        """Tests given Scilab function (`answer`) with the `test_code`
+        supplied. If the optional `in_dir` keyword argument is supplied
+        it changes the directory to that directory (it does not change
+        it back to the original when done). This function also timesout
+        when the function takes more than SERVER_TIMEOUT seconds to run
+        to prevent runaway code.
+
+        The testcode is a path to the reference code.
+        The reference code will call the function submitted by the student.
+        The reference code will check for the expected output.
+
+        If the path's start with a "/" then we assume they are absolute paths.
+        If not, we assume they are relative paths w.r.t. the location of this
+        code_server script.
+
+        Returns
+        -------
+
+        A tuple: (success, error message).
+
+        """
+        if in_dir is not None and isdir(in_dir):
+            os.chdir(in_dir)
+        
+        # Removes all the commands that terminates scilab
+        answer,i = self._remove_scilab_exit(answer.lstrip())
+
+        # Throw message if there are commmands that terminates scilab
+        add_err=""
+        if i > 0:
+            add_err = "Please do not use exit, quit and abort commands in your\
+                        code.\n Otherwise your code will not be evaluated\
+                        correctly.\n"
+
+        # The file extension should be .sci
+        submit_f = open('function.sci','w')
+        submit_f.write(answer)
+        submit_f.close()
+        submit_path = abspath(submit_f.name)
+
+        ref_path = test_code.strip()
+        if not ref_path.startswith('/'):
+            ref_path = join(MY_DIR, ref_path)
+
+        # Add a new signal handler for the execution of this code.
+        old_handler = signal.signal(signal.SIGALRM, timeout_handler)
+        signal.alarm(SERVER_TIMEOUT)
+
+        # Do whatever testing needed.
+        success = False
+        try:
+            cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path)
+            cmd += ' | scilab-cli -nb'
+            ret = self._run_command(cmd,
+                                    shell=True,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+            proc, stdout, stderr = ret
+
+            # Get only the error.
+            stderr = self._get_error(stdout)
+            if stderr is None:
+                # Clean output
+                stdout = self._strip_output(stdout)
+                if proc.returncode == 5:
+                    success, err = True, "Correct answer"
+                else:
+                    err = add_err + stdout
+            else:
+                err = add_err + stderr
+        except TimeoutException:
+            err = self.timeout_msg
+        except:
+            type, value = sys.exc_info()[:2]
+            err = "Error: {0}".format(repr(value))
+        finally:
+            # Set back any original signal handler.
+            signal.signal(signal.SIGALRM, old_handler)
+
+        # Delete the created file.
+        os.remove(submit_path)
+
+        # Cancel the signal if any, see signal.alarm documentation.
+        signal.alarm(0)
+
+        # Put us back into the server pool queue since we are free now.
+        self.queue.put(self.port)
+
+        return success, err
+
+    def _remove_scilab_exit(self, string):
+        """
+            Removes exit, quit and abort from the scilab code
+        """
+        new_string = ""
+        i=0
+        for line in string.splitlines():
+            new_line = re.sub(r"exit.*$","",line)
+            new_line = re.sub(r"quit.*$","",new_line)
+            new_line = re.sub(r"abort.*$","",new_line)
+            if line != new_line:
+                i=i+1
+            new_string = new_string +'\n'+ new_line
+        return new_string, i
+
+    def _get_error(self, string):
+        """
+            Fetches only the error from the string.
+            Returns None if no error.
+        """
+        obj =  re.search("!.+\n.+",string);
+        if obj:         
+            return obj.group()
+        return None
+
+    def _strip_output(self, out):
+        """
+            Cleans whitespace from the output
+        """
+        strip_out = "Message"
+        for l in out.split('\n'):
+            if l.strip():
+                strip_out = strip_out+"\n"+l.strip()
+        return strip_out
 
     def run(self):
         """Run XMLRPC server, serving our methods.
diff --git a/testapp/exam/forms.py b/testapp/exam/forms.py
index 917bea7..26d9a2d 100644
--- a/testapp/exam/forms.py
+++ b/testapp/exam/forms.py
@@ -19,6 +19,7 @@ QUESTION_TYPE_CHOICES = (
         ("C", "C Language"),
         ("C++", "C++ Language"),
         ("java", "Java Language"),
+        ("scilab", "Scilab"),
                         )
 
 UNAME_CHARS = letters + "._" + digits
diff --git a/testapp/exam/models.py b/testapp/exam/models.py
index babde0f..8718968 100644
--- a/testapp/exam/models.py
+++ b/testapp/exam/models.py
@@ -21,6 +21,7 @@ QUESTION_TYPE_CHOICES = (
         ("C", "C Language"),
         ("C++", "C++ Language"),
         ("java", "Java Language"),
+        ("scilab", "Scilab"),
                         )
 
 ################################################################################
diff --git a/testapp/exam/xmlrpc_clients.py b/testapp/exam/xmlrpc_clients.py
index cc21e62..14ebf27 100644
--- a/testapp/exam/xmlrpc_clients.py
+++ b/testapp/exam/xmlrpc_clients.py
@@ -26,6 +26,7 @@ class CodeServerProxy(object):
                         "C": "run_c_code",
                         "C++": "run_cplus_code",
                         "java": "run_java_code",
+                        "scilab": "run_scilab_code",
                         }
 
     def run_code(self, answer, test_code, user_dir, language):
diff --git a/testapp/scilab_files/test_add.sce b/testapp/scilab_files/test_add.sce
new file mode 100644
index 0000000..a317cdb
--- /dev/null
+++ b/testapp/scilab_files/test_add.sce
@@ -0,0 +1,29 @@
+mode(-1)
+exec("function.sci",-1);
+i = 0
+p = add(3,5);
+correct = (p == 8);
+if correct then
+ i=i+1
+end
+disp("Input submitted 3 and 5")
+disp("Expected output 8 got " + string(p))
+p = add(22,-20);
+correct = (p==2);
+if correct then
+ i=i+1
+end
+disp("Input submitted 22 and -20")
+disp("Expected output 2 got " + string(p))
+p =add(91,0);
+correct = (p==91);
+if correct then
+ i=i+1
+end
+disp("Input submitted 91 and 0")
+disp("Expected output 91 got " + string(p))
+if i==3 then
+ exit(5);
+else
+ exit(3);
+end