diff options
author | Prabhu Ramachandran | 2015-05-12 20:20:43 +0530 |
---|---|---|
committer | Prabhu Ramachandran | 2015-05-12 20:20:43 +0530 |
commit | a022e0145ec8fb1622d58c2e2281c016b1d45b01 (patch) | |
tree | 1c0c3f2e8605d6f36405c57cbe5de9a895a47958 | |
parent | cd9f2542d09db0e4a352dd410f626f27e23c37e4 (diff) | |
parent | 5b23647de575fd90552807260a4b8e0a96ab6afe (diff) | |
download | online_test-a022e0145ec8fb1622d58c2e2281c016b1d45b01.tar.gz online_test-a022e0145ec8fb1622d58c2e2281c016b1d45b01.tar.bz2 online_test-a022e0145ec8fb1622d58c2e2281c016b1d45b01.zip |
Merge pull request #41 from ankitjavalkar/code-server-redesign-mymaster2
Code server redesign
-rw-r--r-- | testapp/docs/sample_questions.py | 84 | ||||
-rw-r--r-- | testapp/docs/sample_questions.xml | 43 | ||||
-rw-r--r-- | testapp/exam/admin.py | 3 | ||||
-rw-r--r-- | testapp/exam/bash_code_evaluator.py | 122 | ||||
-rw-r--r-- | testapp/exam/bash_files/sample.args (renamed from testapp/docs/sample.args) | 0 | ||||
-rwxr-xr-x | testapp/exam/bash_files/sample.sh (renamed from testapp/docs/sample.sh) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main.cpp (renamed from testapp/c_cpp_files/main.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main2.c (renamed from testapp/c_cpp_files/main2.c) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_array_check.cpp (renamed from testapp/c_cpp_files/main_array_check.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_array_check_all.cpp (renamed from testapp/c_cpp_files/main_array_check_all.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_array_sum.cpp (renamed from testapp/c_cpp_files/main_array_sum.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_blackJack.cpp (renamed from testapp/c_cpp_files/main_blackJack.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_check_digit.cpp (renamed from testapp/c_cpp_files/main_check_digit.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_count667.cpp (renamed from testapp/c_cpp_files/main_count667.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_count7.cpp (renamed from testapp/c_cpp_files/main_count7.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_fact.cpp (renamed from testapp/c_cpp_files/main_fact.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_greatest.cpp (renamed from testapp/c_cpp_files/main_greatest.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_hello_name.c (renamed from testapp/c_cpp_files/main_hello_name.c) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_lessThan9.cpp (renamed from testapp/c_cpp_files/main_lessThan9.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_mean.cpp (renamed from testapp/c_cpp_files/main_mean.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_palindrome.cpp (renamed from testapp/c_cpp_files/main_palindrome.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_roundTo10.cpp (renamed from testapp/c_cpp_files/main_roundTo10.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_specialSum.cpp (renamed from testapp/c_cpp_files/main_specialSum.cpp) | 0 | ||||
-rwxr-xr-x | testapp/exam/c_cpp_files/main_within.cpp (renamed from testapp/c_cpp_files/main_within.cpp) | 0 | ||||
-rw-r--r-- | testapp/exam/code_evaluator.py | 206 | ||||
-rwxr-xr-x | testapp/exam/code_server.py | 752 | ||||
-rw-r--r-- | testapp/exam/cpp_code_evaluator.py | 125 | ||||
-rw-r--r-- | testapp/exam/forms.py | 46 | ||||
-rw-r--r-- | testapp/exam/java_code_evaluator.py | 127 | ||||
-rw-r--r-- | testapp/exam/java_files/main_array_sum.java (renamed from testapp/java_files/main_array_sum.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_fact.java (renamed from testapp/java_files/main_fact.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_great.java (renamed from testapp/java_files/main_great.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_hello_name.java (renamed from testapp/java_files/main_hello_name.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_lastDigit.java (renamed from testapp/java_files/main_lastDigit.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_moreThan30.java (renamed from testapp/java_files/main_moreThan30.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_palindrome.java (renamed from testapp/java_files/main_palindrome.java) | 0 | ||||
-rw-r--r-- | testapp/exam/java_files/main_square.java (renamed from testapp/java_files/main_square.java) | 0 | ||||
-rw-r--r-- | testapp/exam/language_registry.py | 36 | ||||
-rw-r--r-- | testapp/exam/models.py | 63 | ||||
-rw-r--r-- | testapp/exam/python_code_evaluator.py | 61 | ||||
-rw-r--r-- | testapp/exam/scilab_code_evaluator.py | 95 | ||||
-rw-r--r-- | testapp/exam/scilab_files/test_add.sce (renamed from testapp/scilab_files/test_add.sce) | 0 | ||||
-rw-r--r-- | testapp/exam/settings.py | 8 | ||||
-rw-r--r-- | testapp/exam/static/exam/js/add_question.js | 9 | ||||
-rw-r--r-- | testapp/exam/templates/exam/add_question.html | 21 | ||||
-rw-r--r-- | testapp/exam/templates/exam/edit_question.html | 36 | ||||
-rw-r--r-- | testapp/exam/tests.py | 55 | ||||
-rw-r--r-- | testapp/exam/views.py | 186 | ||||
-rw-r--r-- | testapp/exam/xmlrpc_clients.py | 29 | ||||
-rw-r--r-- | testapp/test_server.py | 302 | ||||
-rw-r--r-- | testapp/tests/__init__.py | 0 | ||||
-rw-r--r-- | testapp/tests/test_bash_evaluation.py | 41 | ||||
-rw-r--r-- | testapp/tests/test_c_cpp_evaluation.py | 77 | ||||
-rw-r--r-- | testapp/tests/test_code_evaluation.py | 24 | ||||
-rw-r--r-- | testapp/tests/test_java_evaluation.py | 41 | ||||
-rw-r--r-- | testapp/tests/test_python_evaluation.py | 54 | ||||
-rw-r--r-- | testapp/tests/test_scilab_evaluation.py | 39 |
57 files changed, 1408 insertions, 1277 deletions
diff --git a/testapp/docs/sample_questions.py b/testapp/docs/sample_questions.py deleted file mode 100644 index 60f32cb..0000000 --- a/testapp/docs/sample_questions.py +++ /dev/null @@ -1,84 +0,0 @@ -from datetime import date - -questions = [ -[Question( - summary='Factorial', - points=2, - language='python', - type='code', - description=''' -Write a function called <code>fact</code> which takes a single integer argument -(say <code>n</code>) and returns the factorial of the number. -For example:<br/> -<code>fact(3) -> 6</code> -''', - test=''' -assert fact(0) == 1 -assert fact(5) == 120 -''', - snippet="def fact(num):" - ), -#Add tags here as a list of string. -['Python','function','factorial'], -], - -[Question( - summary='Simple function', - points=1, - language='python', - type='code', - description='''Create a simple function called <code>sqr</code> which takes a single -argument and returns the square of the argument. For example: <br/> -<code>sqr(3) -> 9</code>.''', - test=''' -import math -assert sqr(3) == 9 -assert abs(sqr(math.sqrt(2)) - 2.0) < 1e-14 - ''', - snippet="def sqr(num):" - ), -#Add tags here as a list of string. -['Python','function'], -], - -[Question( - summary='Bash addition', - points=2, - language='bash', - type='code', - description='''Write a shell script which takes two arguments on the - command line and prints the sum of the two on the output.''', - test='''\ -docs/sample.sh -docs/sample.args -''', - snippet="#!/bin/bash" - ), -#Add tags here as a list of string. -[''], -], - -[Question( - summary='Size of integer in Python', - points=0.5, - language='python', - type='mcq', - description='''What is the largest integer value that can be represented -in Python?''', - options='''No Limit -2**32 -2**32 - 1 -None of the above -''', - test = "No Limit" - ), -#Add tags here as a list of string. -['mcq'], -], - -] #list of questions ends here - -quiz = Quiz(start_date=date.today(), - duration=10, - description='Basic Python Quiz 1' - ) diff --git a/testapp/docs/sample_questions.xml b/testapp/docs/sample_questions.xml deleted file mode 100644 index 53c76f8..0000000 --- a/testapp/docs/sample_questions.xml +++ /dev/null @@ -1,43 +0,0 @@ -<question_bank> - -<question> -<summary> -Factorial -</summary> -<description> -Write a function called "fact" which takes a single integer argument (say "n") -and returns the factorial of the number. -For example fact(3) -> 6 -</description> -<points>2</points> -<type>python</type> -<test> -assert fact(0) == 1 -assert fact(5) == 120 -</test> -<options> -</options> -</question> - -<question> -<summary> -Simple function -</summary> -<description> -Create a simple function called "sqr" which takes a single argument and -returns the square of the argument -For example sqr(3) -> 9. -</description> -<points>1</points> -<type>python</type> -<test> -import math -assert sqr(3) == 9 -assert abs(sqr(math.sqrt(2)) - 2.0) < 1e-14 -</test> -<options> -</options> -</question> - - -</question_bank> diff --git a/testapp/exam/admin.py b/testapp/exam/admin.py index 060859a..86a10af 100644 --- a/testapp/exam/admin.py +++ b/testapp/exam/admin.py @@ -1,5 +1,6 @@ -from testapp.exam.models import Question, Quiz +from testapp.exam.models import Question, Quiz, TestCase from django.contrib import admin admin.site.register(Question) +admin.site.register(TestCase) admin.site.register(Quiz) diff --git a/testapp/exam/bash_code_evaluator.py b/testapp/exam/bash_code_evaluator.py new file mode 100644 index 0000000..23c0ae5 --- /dev/null +++ b/testapp/exam/bash_code_evaluator.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from code_evaluator import CodeEvaluator + + +class BashCodeEvaluator(CodeEvaluator): + """Tests the Bash code obtained from Code Server""" + def __init__(self, test_case_data, test, language, user_answer, + ref_code_path=None, in_dir=None): + super(BashCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, + ref_code_path, in_dir) + self.submit_path = self.create_submit_code_file('submit.sh') + self.test_case_args = self._setup() + + # Private Protocol ########## + def _setup(self): + super(BashCodeEvaluator, self)._setup() + + self._set_file_as_executable(self.submit_path) + get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') + get_ref_path = get_ref_path.strip() + get_test_case_path = get_test_case_path.strip() + ref_path, test_case_path = self._set_test_code_file_path(get_ref_path, + get_test_case_path) + + return ref_path, self.submit_path, test_case_path + + def _teardown(self): + # Delete the created file. + super(BashCodeEvaluator, self)._teardown() + os.remove(self.submit_path) + + def _check_code(self, ref_path, submit_path, + test_case_path=None): + """ Function validates student script using instructor script as + reference. Test cases can optionally be provided. The first argument + ref_path, is the path to instructor script, it is assumed to + have executable permission. The second argument submit_path, is + the path to the student script, it is assumed to have executable + permission. The Third optional argument is the path to test the + scripts. Each line in this file is a test case and each test case is + passed to the script as standard arguments. + + Returns + -------- + + returns (True, "Correct answer") : If the student script passes all + test cases/have same output, when compared to the instructor script + + returns (False, error_msg): If the student script fails a single + test/have dissimilar output, when compared to the instructor script. + + Returns (False, error_msg): If mandatory arguments are not files or if + the required permissions are not given to the file(s). + + """ + if not isfile(ref_path): + return False, "No file at %s or Incorrect path" % ref_path + if not isfile(submit_path): + return False, "No file at %s or Incorrect path" % submit_path + if not os.access(ref_path, os.X_OK): + return False, "Script %s is not executable" % ref_path + if not os.access(submit_path, os.X_OK): + return False, "Script %s is not executable" % submit_path + + success = False + + if test_case_path is None or "": + ret = self.run_command(ref_path, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, inst_stdout, inst_stderr = ret + ret = self.run_command(submit_path, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdnt_stdout, stdnt_stderr = ret + if inst_stdout == stdnt_stdout: + return True, "Correct answer" + else: + err = "Error: expected %s, got %s" % (inst_stderr, + stdnt_stderr) + return False, err + else: + if not isfile(test_case_path): + return False, "No test case at %s" % test_case_path + if not os.access(ref_path, os.R_OK): + return False, "Test script %s, not readable" % test_case_path + # valid_answer is True, so that we can stop once a test case fails + valid_answer = True + # loop_count has to be greater than or equal to one. + # Useful for caching things like empty test files,etc. + loop_count = 0 + test_cases = open(test_case_path).readlines() + num_lines = len(test_cases) + for test_case in test_cases: + loop_count += 1 + if valid_answer: + args = [ref_path] + [x for x in test_case.split()] + ret = self.run_command(args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, inst_stdout, inst_stderr = ret + args = [submit_path]+[x for x in test_case.split()] + ret = self.run_command(args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdnt_stdout, stdnt_stderr = ret + valid_answer = inst_stdout == stdnt_stdout + if valid_answer and (num_lines == loop_count): + return True, "Correct answer" + else: + err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, + stdnt_stdout+stdnt_stderr) + return False, err + diff --git a/testapp/docs/sample.args b/testapp/exam/bash_files/sample.args index 4d9f00d..4d9f00d 100644 --- a/testapp/docs/sample.args +++ b/testapp/exam/bash_files/sample.args diff --git a/testapp/docs/sample.sh b/testapp/exam/bash_files/sample.sh index e935cb3..e935cb3 100755 --- a/testapp/docs/sample.sh +++ b/testapp/exam/bash_files/sample.sh diff --git a/testapp/c_cpp_files/main.cpp b/testapp/exam/c_cpp_files/main.cpp index ebe1f08..ebe1f08 100755 --- a/testapp/c_cpp_files/main.cpp +++ b/testapp/exam/c_cpp_files/main.cpp diff --git a/testapp/c_cpp_files/main2.c b/testapp/exam/c_cpp_files/main2.c index ccd1768..ccd1768 100755 --- a/testapp/c_cpp_files/main2.c +++ b/testapp/exam/c_cpp_files/main2.c diff --git a/testapp/c_cpp_files/main_array_check.cpp b/testapp/exam/c_cpp_files/main_array_check.cpp index ea34fdd..ea34fdd 100755 --- a/testapp/c_cpp_files/main_array_check.cpp +++ b/testapp/exam/c_cpp_files/main_array_check.cpp diff --git a/testapp/c_cpp_files/main_array_check_all.cpp b/testapp/exam/c_cpp_files/main_array_check_all.cpp index 140578e..140578e 100755 --- a/testapp/c_cpp_files/main_array_check_all.cpp +++ b/testapp/exam/c_cpp_files/main_array_check_all.cpp diff --git a/testapp/c_cpp_files/main_array_sum.cpp b/testapp/exam/c_cpp_files/main_array_sum.cpp index 55b2ebf..55b2ebf 100755 --- a/testapp/c_cpp_files/main_array_sum.cpp +++ b/testapp/exam/c_cpp_files/main_array_sum.cpp diff --git a/testapp/c_cpp_files/main_blackJack.cpp b/testapp/exam/c_cpp_files/main_blackJack.cpp index cc54e78..cc54e78 100755 --- a/testapp/c_cpp_files/main_blackJack.cpp +++ b/testapp/exam/c_cpp_files/main_blackJack.cpp diff --git a/testapp/c_cpp_files/main_check_digit.cpp b/testapp/exam/c_cpp_files/main_check_digit.cpp index d3bf3d6..d3bf3d6 100755 --- a/testapp/c_cpp_files/main_check_digit.cpp +++ b/testapp/exam/c_cpp_files/main_check_digit.cpp diff --git a/testapp/c_cpp_files/main_count667.cpp b/testapp/exam/c_cpp_files/main_count667.cpp index f146e8c..f146e8c 100755 --- a/testapp/c_cpp_files/main_count667.cpp +++ b/testapp/exam/c_cpp_files/main_count667.cpp diff --git a/testapp/c_cpp_files/main_count7.cpp b/testapp/exam/c_cpp_files/main_count7.cpp index 982e930..982e930 100755 --- a/testapp/c_cpp_files/main_count7.cpp +++ b/testapp/exam/c_cpp_files/main_count7.cpp diff --git a/testapp/c_cpp_files/main_fact.cpp b/testapp/exam/c_cpp_files/main_fact.cpp index a4ff230..a4ff230 100755 --- a/testapp/c_cpp_files/main_fact.cpp +++ b/testapp/exam/c_cpp_files/main_fact.cpp diff --git a/testapp/c_cpp_files/main_greatest.cpp b/testapp/exam/c_cpp_files/main_greatest.cpp index 6d0a7c2..6d0a7c2 100755 --- a/testapp/c_cpp_files/main_greatest.cpp +++ b/testapp/exam/c_cpp_files/main_greatest.cpp diff --git a/testapp/c_cpp_files/main_hello_name.c b/testapp/exam/c_cpp_files/main_hello_name.c index 71b83a2..71b83a2 100755 --- a/testapp/c_cpp_files/main_hello_name.c +++ b/testapp/exam/c_cpp_files/main_hello_name.c diff --git a/testapp/c_cpp_files/main_lessThan9.cpp b/testapp/exam/c_cpp_files/main_lessThan9.cpp index 722b4bb..722b4bb 100755 --- a/testapp/c_cpp_files/main_lessThan9.cpp +++ b/testapp/exam/c_cpp_files/main_lessThan9.cpp diff --git a/testapp/c_cpp_files/main_mean.cpp b/testapp/exam/c_cpp_files/main_mean.cpp index 21a4b1a..21a4b1a 100755 --- a/testapp/c_cpp_files/main_mean.cpp +++ b/testapp/exam/c_cpp_files/main_mean.cpp diff --git a/testapp/c_cpp_files/main_palindrome.cpp b/testapp/exam/c_cpp_files/main_palindrome.cpp index 0e66928..0e66928 100755 --- a/testapp/c_cpp_files/main_palindrome.cpp +++ b/testapp/exam/c_cpp_files/main_palindrome.cpp diff --git a/testapp/c_cpp_files/main_roundTo10.cpp b/testapp/exam/c_cpp_files/main_roundTo10.cpp index 12c961d..12c961d 100755 --- a/testapp/c_cpp_files/main_roundTo10.cpp +++ b/testapp/exam/c_cpp_files/main_roundTo10.cpp diff --git a/testapp/c_cpp_files/main_specialSum.cpp b/testapp/exam/c_cpp_files/main_specialSum.cpp index d614536..d614536 100755 --- a/testapp/c_cpp_files/main_specialSum.cpp +++ b/testapp/exam/c_cpp_files/main_specialSum.cpp diff --git a/testapp/c_cpp_files/main_within.cpp b/testapp/exam/c_cpp_files/main_within.cpp index 50f9ad0..50f9ad0 100755 --- a/testapp/c_cpp_files/main_within.cpp +++ b/testapp/exam/c_cpp_files/main_within.cpp diff --git a/testapp/exam/code_evaluator.py b/testapp/exam/code_evaluator.py new file mode 100644 index 0000000..2a57257 --- /dev/null +++ b/testapp/exam/code_evaluator.py @@ -0,0 +1,206 @@ +import sys +from SimpleXMLRPCServer import SimpleXMLRPCServer +import pwd +import os +import stat +from os.path import isdir, dirname, abspath, join, isfile +import signal +from multiprocessing import Process, Queue +import subprocess +import re +import json +# Local imports. +from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT + + +MY_DIR = abspath(dirname(__file__)) + + +# Raised when the code times-out. +# c.f. http://pguides.net/python/timeout-a-function +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + """A handler for the ALARM signal.""" + raise TimeoutException('Code took too long to run.') + + +def create_signal_handler(): + """Add a new signal handler for the execution of this code.""" + prev_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + return prev_handler + + +def set_original_signal_handler(old_handler=None): + """Set back any original signal handler.""" + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + return + else: + raise Exception("Signal Handler: object cannot be NoneType") + + +def delete_signal_handler(): + signal.alarm(0) + return + + +class CodeEvaluator(object): + """Tests the code obtained from Code Server""" + def __init__(self, test_case_data, test, language, user_answer, + ref_code_path=None, in_dir=None): + msg = 'Code took more than %s seconds to run. You probably '\ + 'have an infinite loop in your code.' % SERVER_TIMEOUT + self.timeout_msg = msg + self.test_case_data = test_case_data + self.language = language.lower() + self.user_answer = user_answer + self.ref_code_path = ref_code_path + self.test = test + self.in_dir = in_dir + self.test_case_args = None + + # Public Protocol ########## + @classmethod + def from_json(cls, language, json_data, in_dir): + json_data = json.loads(json_data) + test_case_data = json_data.get("test_case_data") + user_answer = json_data.get("user_answer") + ref_code_path = json_data.get("ref_code_path") + test = json_data.get("test") + + instance = cls(test_case_data, test, language, user_answer, ref_code_path, + in_dir) + return instance + + def evaluate(self): + """Evaluates given code with the test cases based on + given arguments in test_case_data. + + The ref_code_path is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + If the optional `in_dir` keyword argument is supplied it changes the + directory to that directory (it does not change it back to the original + when done). + + Returns + ------- + + A tuple: (success, error message). + """ + + self._setup() + success, err = self._evaluate(self.test_case_args) + self._teardown() + + result = {'success': success, 'error': err} + return result + + # Private Protocol ########## + def _setup(self): + self._change_dir(self.in_dir) + + def _evaluate(self, args): + # Add a new signal handler for the execution of this code. + prev_handler = create_signal_handler() + success = False + args = args or [] + + # Do whatever testing needed. + try: + success, err = self._check_code(*args) + + except TimeoutException: + err = self.timeout_msg + except: + _type, value = sys.exc_info()[:2] + err = "Error: {0}".format(repr(value)) + finally: + # Set back any original signal handler. + set_original_signal_handler(prev_handler) + + return success, err + + def _teardown(self): + # Cancel the signal + delete_signal_handler() + + def _check_code(self): + raise NotImplementedError("check_code method not implemented") + + def create_submit_code_file(self, file_name): + """ Write the code (`answer`) to a file and set the file path""" + submit_f = open(file_name, 'w') + submit_f.write(self.user_answer.lstrip()) + submit_f.close() + submit_path = abspath(submit_f.name) + + return submit_path + + def _set_file_as_executable(self, fname): + os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + + def _set_test_code_file_path(self, ref_path=None, test_case_path=None): + if ref_path and not ref_path.startswith('/'): + ref_path = join(MY_DIR, ref_path) + + if test_case_path and not test_case_path.startswith('/'): + test_case_path = join(MY_DIR, test_case_path) + + return ref_path, test_case_path + + def _run_command(self, cmd_args, *args, **kw): + """Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stdout and stderr. + """ + try: + proc = subprocess.Popen(cmd_args, *args, **kw) + stdout, stderr = proc.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc.kill() + # Re-raise exception. + raise + return proc, stdout, stderr + + def _compile_command(self, cmd, *args, **kw): + """Compiles C/C++/java code and returns errors if any. + Run a command in a subprocess while blocking, the process is killed + if it takes more than 2 seconds to run. Return the Popen object, the + stderr. + """ + try: + proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc_compile.communicate() + except TimeoutException: + # Runaway code, so kill it. + proc_compile.kill() + # Re-raise exception. + raise + return proc_compile, err + + def _change_dir(self, in_dir): + if in_dir is not None and isdir(in_dir): + os.chdir(in_dir) + + def _remove_null_substitute_char(self, string): + """Returns a string without any null and substitute characters""" + stripped = "" + for c in string: + if ord(c) is not 26 and ord(c) is not 0: + stripped = stripped + c + return ''.join(stripped) diff --git a/testapp/exam/code_server.py b/testapp/exam/code_server.py index 792197d..580379f 100755 --- a/testapp/exam/code_server.py +++ b/testapp/exam/code_server.py @@ -19,7 +19,6 @@ settings.py:SERVER_POOL_PORT. This port exposes a `get_server_port` function that returns an available server. """ import sys -import traceback from SimpleXMLRPCServer import SimpleXMLRPCServer import pwd import os @@ -29,12 +28,16 @@ import signal from multiprocessing import Process, Queue import subprocess import re +import json # Local imports. from settings import SERVER_PORTS, SERVER_TIMEOUT, SERVER_POOL_PORT +from language_registry import set_registry + MY_DIR = abspath(dirname(__file__)) +# Private Protocol ########## def run_as_nobody(): """Runs the current process as nobody.""" # Set the effective uid and to that of nobody. @@ -43,17 +46,6 @@ def run_as_nobody(): os.seteuid(nobody.pw_uid) -# Raised when the code times-out. -# c.f. http://pguides.net/python/timeout-a-function -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - """A handler for the ALARM signal.""" - raise TimeoutException('Code took too long to run.') - - ############################################################################### # `CodeServer` class. ############################################################################### @@ -64,736 +56,38 @@ class CodeServer(object): def __init__(self, port, queue): self.port = port self.queue = queue - msg = 'Code took more than %s seconds to run. You probably '\ - 'have an infinite loop in your code.' % SERVER_TIMEOUT - self.timeout_msg = msg - - def run_python_code(self, answer, test_code, in_dir=None): - """Tests given Python function (`answer`) with the `test_code` - supplied. If the optional `in_dir` keyword argument is supplied - it changes the directory to that directory (it does not change - it back to the original when done). This function also timesout - when the function takes more than SERVER_TIMEOUT seconds to run - to prevent runaway code. - Returns - ------- - - A tuple: (success, error message). - - """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - success = False - tb = None - try: - submitted = compile(answer, '<string>', mode='exec') - g = {} - exec submitted in g - _tests = compile(test_code, '<string>', mode='exec') - exec _tests in g - except TimeoutException: - err = self.timeout_msg - except AssertionError: - type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(test_code).splitlines()[lineno-1] - err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - else: - success = True - err = 'Correct answer' - finally: - del tb - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return success, err - - def run_bash_code(self, answer, test_code, in_dir=None): - """Tests given Bash code (`answer`) with the `test_code` supplied. - - The testcode should typically contain two lines, the first is a path to - the reference script we are to compare against. The second is a path - to the arguments to be supplied to the reference and submitted script. - The output of these will be compared for correctness. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - - """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - def _set_exec(fname): - os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP - | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) - submit_f = open('submit.sh', 'w') - submit_f.write(answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - _set_exec(submit_path) - - ref_path, test_case_path = test_code.strip().splitlines() - if not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - if not test_case_path.startswith('/'): - test_case_path = join(MY_DIR, test_case_path) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - # Do whatever testing needed. - success = False - try: - success, err = self.check_bash_script(ref_path, submit_path, - test_case_path) - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Delete the created file. - os.remove(submit_path) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return success, err - - def _run_command(self, cmd_args, *args, **kw): - """Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stdout and stderr. - """ - try: - proc = subprocess.Popen(cmd_args, *args, **kw) - stdout, stderr = proc.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc.kill() - # Re-raise exception. - raise - return proc, stdout, stderr - - def check_bash_script(self, ref_script_path, submit_script_path, - test_case_path=None): - """ Function validates student script using instructor script as - reference. Test cases can optionally be provided. The first argument - ref_script_path, is the path to instructor script, it is assumed to - have executable permission. The second argument submit_script_path, is - the path to the student script, it is assumed to have executable - permission. The Third optional argument is the path to test the - scripts. Each line in this file is a test case and each test case is - passed to the script as standard arguments. - - Returns - -------- - - returns (True, "Correct answer") : If the student script passes all - test cases/have same output, when compared to the instructor script - - returns (False, error_msg): If the student script fails a single - test/have dissimilar output, when compared to the instructor script. - - Returns (False, error_msg): If mandatory arguments are not files or if - the required permissions are not given to the file(s). - - """ - if not isfile(ref_script_path): - return False, "No file at %s" % ref_script_path - if not isfile(submit_script_path): - return False, 'No file at %s' % submit_script_path - if not os.access(ref_script_path, os.X_OK): - return False, 'Script %s is not executable' % ref_script_path - if not os.access(submit_script_path, os.X_OK): - return False, 'Script %s is not executable' % submit_script_path - - if test_case_path is None: - ret = self._run_command(ref_script_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, inst_stdout, inst_stderr = ret - ret = self._run_command(submit_script_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdnt_stdout, stdnt_stderr = ret - if inst_stdout == stdnt_stdout: - return True, 'Correct answer' - else: - err = "Error: expected %s, got %s" % (inst_stderr, - stdnt_stderr) - return False, err - else: - if not isfile(test_case_path): - return False, "No test case at %s" % test_case_path - if not os.access(ref_script_path, os.R_OK): - return False, "Test script %s, not readable" % test_case_path - valid_answer = True # We initially make it one, so that we can - # stop once a test case fails - loop_count = 0 # Loop count has to be greater than or - # equal to one. - # Useful for caching things like empty - # test files,etc. - test_cases = open(test_case_path).readlines() - num_lines = len(test_cases) - for test_case in test_cases: - loop_count += 1 - if valid_answer: - args = [ref_script_path] + [x for x in test_case.split()] - ret = self._run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, inst_stdout, inst_stderr = ret - args = [submit_script_path]+[x for x in test_case.split()] - ret = self._run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdnt_stdout, stdnt_stderr = ret - valid_answer = inst_stdout == stdnt_stdout - if valid_answer and (num_lines == loop_count): - return True, "Correct answer" - else: - err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, - stdnt_stdout+stdnt_stderr) - return False, err - - def run_c_code(self, answer, test_code, in_dir=None): - """Tests given C code (`answer`) with the `test_code` supplied. - - The testcode is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - - """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - # File extension must be .c - submit_f = open('submit.c', 'w') - submit_f.write(answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - - ref_path = test_code.strip() - if not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - # Do whatever testing needed. - success = False - try: - success, err = self._check_c_cpp_code(ref_path, submit_path) - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Delete the created file. - os.remove(submit_path) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return success, err - - def _compile_command(self, cmd, *args, **kw): - """Compiles C/C++/java code and returns errors if any. - Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stderr. - """ - try: - proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc_compile.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc_compile.kill() - # Re-raise exception. - raise - return proc_compile, err - - def _check_c_cpp_code(self, ref_code_path, submit_code_path): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - if not isfile(ref_code_path): - return False, "No file at %s" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s' % submit_code_path - - success = False - output_path = os.getcwd() + '/output' - compile_command = "g++ %s -c -o %s" % (submit_code_path, output_path) - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - executable = os.getcwd() + '/executable' - compile_main = "g++ %s %s -o %s" % (ref_code_path, output_path, - executable) - ret = self._compile_command(compile_main) - proc, main_err = ret - if main_err == '': - args = [executable] - ret = self._run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" - else: - err = stdout + "\n" + stderr - os.remove(executable) - else: - err = "Error:" - try: - error_lines = main_err.splitlines() - for e in error_lines: - err = err + "\n" + e.split(":", 1)[1] - except: - err = err + "\n" + main_err - os.remove(output_path) - else: - err = "Compilation Error:" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + stdnt_stderr - return success, err - - def run_cplus_code(self, answer, test_code, in_dir=None): - """Tests given C++ code (`answer`) with the `test_code` supplied. - - The testcode is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - - """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - # The file extension must be .cpp - submit_f = open('submitstd.cpp', 'w') - submit_f.write(answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - - ref_path = test_code.strip() - if not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - # Do whatever testing needed. - success = False - try: - success, err = self._check_c_cpp_code(ref_path, submit_path) - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Delete the created file. - os.remove(submit_path) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return success, err - - def run_java_code(self, answer, test_code, in_dir=None): - """Tests given java code (`answer`) with the `test_code` supplied. - - The testcode is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message). - - """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - # The file extension must be .java - # The class name and file name must be same in java - submit_f = open('Test.java', 'w') - submit_f.write(answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) - - ref_path = test_code.strip() - if not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - # Do whatever testing needed. - success = False - try: - success, err = self._check_java_code(ref_path, submit_path) - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Delete the created file. - os.remove(submit_path) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) - - # Put us back into the server pool queue since we are free now. - self.queue.put(self.port) - - return success, err - - def _check_java_code(self, ref_code_path, submit_code_path): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - if not isfile(ref_code_path): - return False, "No file at %s" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s' % submit_code_path - - success = False - compile_command = "javac %s" % (submit_code_path) - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret - stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - student_directory = os.getcwd() + '/' - student_file_name = "Test" - compile_main = "javac %s -classpath %s -d %s" % (ref_code_path, - student_directory, - student_directory) - ret = self._compile_command(compile_main) - proc, main_err = ret - main_err = self._remove_null_substitute_char(main_err) - - if main_err == '': - main_file_name = (ref_code_path.split('/')[-1]).split('.')[0] - run_command = "java -cp %s %s" % (student_directory, - main_file_name) - ret = self._run_command(run_command, - stdin=None, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" - else: - err = stdout + "\n" + stderr - success = False - os.remove("%s%s.class" % (student_directory, main_file_name)) - else: - err = "Error:\n" - try: - error_lines = main_err.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + main_err - os.remove("%s%s.class" % (student_directory, student_file_name)) - else: - err = "Compilation Error:\n" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + stdnt_stderr - return success, err - - def _remove_null_substitute_char(self, string): - """Returns a string without any null and substitute characters""" - stripped = "" - for c in string: - if ord(c) is not 26 and ord(c) is not 0: - stripped = stripped + c - return ''.join(stripped) - - def run_scilab_code(self, answer, test_code, in_dir=None): - """Tests given Scilab function (`answer`) with the `test_code` - supplied. If the optional `in_dir` keyword argument is supplied - it changes the directory to that directory (it does not change - it back to the original when done). This function also timesout - when the function takes more than SERVER_TIMEOUT seconds to run - to prevent runaway code. - - The testcode is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - Returns - ------- - - A tuple: (success, error message). + # Public Protocol ########## + def check_code(self, language, json_data, in_dir=None): + """Calls relevant EvaluateCode class based on language to check the + answer code """ - if in_dir is not None and isdir(in_dir): - os.chdir(in_dir) - - # Removes all the commands that terminates scilab - answer,i = self._remove_scilab_exit(answer.lstrip()) - - # Throw message if there are commmands that terminates scilab - add_err="" - if i > 0: - add_err = "Please do not use exit, quit and abort commands in your\ - code.\n Otherwise your code will not be evaluated\ - correctly.\n" - - # The file extension should be .sci - submit_f = open('function.sci','w') - submit_f.write(answer) - submit_f.close() - submit_path = abspath(submit_f.name) - - ref_path = test_code.strip() - if not ref_path.startswith('/'): - ref_path = join(MY_DIR, ref_path) - - # Add a new signal handler for the execution of this code. - old_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - - # Do whatever testing needed. - success = False - try: - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) - cmd += ' | timeout 8 scilab-cli -nb' - ret = self._run_command(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - - # Get only the error. - stderr = self._get_error(stdout) - if stderr is None: - # Clean output - stdout = self._strip_output(stdout) - if proc.returncode == 5: - success, err = True, "Correct answer" - else: - err = add_err + stdout - else: - err = add_err + stderr - except TimeoutException: - err = self.timeout_msg - except: - type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) - finally: - # Set back any original signal handler. - signal.signal(signal.SIGALRM, old_handler) - - # Delete the created file. - os.remove(submit_path) - - # Cancel the signal if any, see signal.alarm documentation. - signal.alarm(0) + code_evaluator = self._create_evaluator_instance(language, json_data, + in_dir) + result = code_evaluator.evaluate() # Put us back into the server pool queue since we are free now. self.queue.put(self.port) - return success, err - - def _remove_scilab_exit(self, string): - """ - Removes exit, quit and abort from the scilab code - """ - new_string = "" - i=0 - for line in string.splitlines(): - new_line = re.sub(r"exit.*$","",line) - new_line = re.sub(r"quit.*$","",new_line) - new_line = re.sub(r"abort.*$","",new_line) - if line != new_line: - i=i+1 - new_string = new_string +'\n'+ new_line - return new_string, i - - def _get_error(self, string): - """ - Fetches only the error from the string. - Returns None if no error. - """ - obj = re.search("!.+\n.+",string); - if obj: - return obj.group() - return None - - def _strip_output(self, out): - """ - Cleans whitespace from the output - """ - strip_out = "Message" - for l in out.split('\n'): - if l.strip(): - strip_out = strip_out+"\n"+l.strip() - return strip_out + return json.dumps(result) def run(self): - """Run XMLRPC server, serving our methods. - """ + """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("localhost", self.port)) self.server = server server.register_instance(self) self.queue.put(self.port) server.serve_forever() + # Private Protocol ########## + def _create_evaluator_instance(self, language, json_data, in_dir): + """Create instance of relevant EvaluateCode class based on language""" + set_registry() + registry = get_registry() + cls = registry.get_class(language) + instance = cls.from_json(language, json_data, in_dir) + return instance + ############################################################################### # `ServerPool` class. @@ -825,6 +119,8 @@ class ServerPool(object): p.start() self.servers = servers + # Public Protocol ########## + def get_server_port(self): """Get available server port from ones in the pool. This will block till it gets an available server. diff --git a/testapp/exam/cpp_code_evaluator.py b/testapp/exam/cpp_code_evaluator.py new file mode 100644 index 0000000..15e2b13 --- /dev/null +++ b/testapp/exam/cpp_code_evaluator.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from code_evaluator import CodeEvaluator + + +class CppCodeEvaluator(CodeEvaluator): + """Tests the C code obtained from Code Server""" + def __init__(self, test_case_data, test, language, user_answer, + ref_code_path=None, in_dir=None): + super(CppCodeEvaluator, self).__init__(test_case_data, test, language, + user_answer, ref_code_path, + in_dir) + self.submit_path = self.create_submit_code_file('submit.c') + self.test_case_args = self._setup() + + # Private Protocol ########## + def _setup(self): + super(CppCodeEvaluator, self)._setup() + + get_ref_path = self.ref_code_path + ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + + # Set file paths + c_user_output_path = os.getcwd() + '/output' + c_ref_output_path = os.getcwd() + '/executable' + + # Set command variables + compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, + c_user_output_path) + compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, + c_user_output_path, + c_ref_output_path) + run_command_args = [c_ref_output_path] + remove_user_output = c_user_output_path + remove_ref_output = c_ref_output_path + + return (ref_path, self.submit_path, compile_command, compile_main, + run_command_args, remove_user_output, remove_ref_output) + + def _teardown(self): + # Delete the created file. + super(CppCodeEvaluator, self)._teardown() + os.remove(self.submit_path) + + def _check_code(self, ref_code_path, submit_code_path, compile_command, + compile_main, run_command_args, remove_user_output, + remove_ref_output): + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + + """ + if not isfile(ref_code_path): + return False, "No file at %s or Incorrect path" % ref_code_path + if not isfile(submit_code_path): + return False, 'No file at %s or Incorrect path' % submit_code_path + + success = False + ret = self._compile_command(compile_command) + proc, stdnt_stderr = ret + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + ret = self._compile_command(compile_main) + proc, main_err = ret + main_err = self._remove_null_substitute_char(main_err) + + if main_err == '': + ret = self._run_command(run_command_args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" + else: + err = stdout + "\n" + stderr + os.remove(remove_ref_output) + else: + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(remove_user_output) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/testapp/exam/forms.py b/testapp/exam/forms.py index 1f12a3b..93584a6 100644 --- a/testapp/exam/forms.py +++ b/testapp/exam/forms.py @@ -1,5 +1,5 @@ from django import forms -from testapp.exam.models import Profile, Quiz, Question +from exam.models import Profile, Quiz, Question, TestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User @@ -8,6 +8,7 @@ from taggit.forms import TagField from taggit_autocomplete_modified.managers import TaggableManagerAutocomplete from taggit_autocomplete_modified.widgets import TagAutocomplete from taggit_autocomplete_modified import settings +from django.forms.models import inlineformset_factory from string import letters, punctuation, digits import datetime @@ -177,7 +178,7 @@ class QuizForm(forms.Form): new_quiz.save() -class QuestionForm(forms.Form): +class QuestionForm(forms.ModelForm): """Creates a form to add or edit a Question. It has the related fields and functions required.""" @@ -186,8 +187,8 @@ class QuestionForm(forms.Form): description = forms.CharField(widget=forms.Textarea\ (attrs={'cols': 40, 'rows': 1})) points = forms.FloatField() - test = forms.CharField(widget=forms.Textarea\ - (attrs={'cols': 40, 'rows': 1})) + solution = forms.CharField(widget=forms.Textarea\ + (attrs={'cols': 40, 'rows': 1}), required=False) options = forms.CharField(widget=forms.Textarea\ (attrs={'cols': 40, 'rows': 1}), required=False) language = forms.CharField(max_length=20, widget=forms.Select\ @@ -198,29 +199,37 @@ class QuestionForm(forms.Form): tags = TagField(widget=TagAutocomplete(), required=False) snippet = forms.CharField(widget=forms.Textarea\ (attrs={'cols': 40, 'rows': 1}), required=False) - - def save(self): - summary = self.cleaned_data["summary"] - description = self.cleaned_data["description"] - points = self.cleaned_data['points'] - test = self.cleaned_data["test"] - options = self.cleaned_data['options'] - language = self.cleaned_data['language'] - type = self.cleaned_data["type"] - active = self.cleaned_data["active"] - snippet = self.cleaned_data["snippet"] + ref_code_path = forms.CharField(widget=forms.Textarea\ + (attrs={'cols': 40, 'rows': 1}), required=False) + + def save(self, commit=True): + summary = self.cleaned_data.get("summary") + description = self.cleaned_data.get("description") + points = self.cleaned_data.get("points") + options = self.cleaned_data.get("options") + language = self.cleaned_data.get("language") + type = self.cleaned_data.get("type") + active = self.cleaned_data.get("active") + snippet = self.cleaned_data.get("snippet") new_question = Question() new_question.summary = summary new_question.description = description new_question.points = points - new_question.test = test + # new_question.test = test new_question.options = options new_question.language = language new_question.type = type new_question.active = active new_question.snippet = snippet - new_question.save() + new_question = super(QuestionForm, self).save(commit=False) + if commit: + new_question.save() + + return new_question + + class Meta: + model = Question class RandomQuestionForm(forms.Form): @@ -229,3 +238,6 @@ class RandomQuestionForm(forms.Form): marks = forms.CharField(max_length=8, widget=forms.Select\ (choices=(('select', 'Select Marks'),))) shuffle_questions = forms.BooleanField(required=False) + +TestCaseFormSet = inlineformset_factory(Question, TestCase,\ + can_order=False, can_delete=False, extra=1) diff --git a/testapp/exam/java_code_evaluator.py b/testapp/exam/java_code_evaluator.py new file mode 100644 index 0000000..08ae208 --- /dev/null +++ b/testapp/exam/java_code_evaluator.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +import traceback +import pwd +import os +from os.path import join, isfile +import subprocess +import importlib + +# local imports +from code_evaluator import CodeEvaluator + + +class JavaCodeEvaluator(CodeEvaluator): + """Tests the Java code obtained from Code Server""" + def __init__(self, test_case_data, test, language, user_answer, + ref_code_path=None, in_dir=None): + super(JavaCodeEvaluator, self).__init__(test_case_data, test, + language, user_answer, + ref_code_path, in_dir) + self.submit_path = self.create_submit_code_file('Test.java') + self.test_case_args = self._setup() + + # Private Protocol ########## + def _setup(self): + super(JavaCodeEvaluator, self)._setup() + + ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) + + # Set file paths + java_student_directory = os.getcwd() + '/' + java_ref_file_name = (ref_path.split('/')[-1]).split('.')[0] + + # Set command variables + compile_command = 'javac {0}'.format(self.submit_path), + compile_main = ('javac {0} -classpath ' + '{1} -d {2}').format(ref_path, + java_student_directory, + java_student_directory) + run_command_args = "java -cp {0} {1}".format(java_student_directory, + java_ref_file_name) + remove_user_output = "{0}{1}.class".format(java_student_directory, + 'Test') + remove_ref_output = "{0}{1}.class".format(java_student_directory, + java_ref_file_name) + + return (ref_path, self.submit_path, compile_command, compile_main, + run_command_args, remove_user_output, remove_ref_output) + + def _teardown(self): + # Delete the created file. + super(JavaCodeEvaluator, self)._teardown() + os.remove(self.submit_path) + + def _check_code(self, ref_code_path, submit_code_path, compile_command, + compile_main, run_command_args, remove_user_output, + remove_ref_output): + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + + """ + if not isfile(ref_code_path): + return False, "No file at %s or Incorrect path" % ref_code_path + if not isfile(submit_code_path): + return False, 'No file at %s or Incorrect path' % submit_code_path + + success = False + ret = self._compile_command(compile_command) + proc, stdnt_stderr = ret + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + ret = self._compile_command(compile_main) + proc, main_err = ret + main_err = self._remove_null_substitute_char(main_err) + + if main_err == '': + ret = self._run_command(run_command_args, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" + else: + err = stdout + "\n" + stderr + os.remove(remove_ref_output) + else: + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(remove_user_output) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/testapp/java_files/main_array_sum.java b/testapp/exam/java_files/main_array_sum.java index 5eae299..5eae299 100644 --- a/testapp/java_files/main_array_sum.java +++ b/testapp/exam/java_files/main_array_sum.java diff --git a/testapp/java_files/main_fact.java b/testapp/exam/java_files/main_fact.java index 325dab6..325dab6 100644 --- a/testapp/java_files/main_fact.java +++ b/testapp/exam/java_files/main_fact.java diff --git a/testapp/java_files/main_great.java b/testapp/exam/java_files/main_great.java index 4bfcb1f..4bfcb1f 100644 --- a/testapp/java_files/main_great.java +++ b/testapp/exam/java_files/main_great.java diff --git a/testapp/java_files/main_hello_name.java b/testapp/exam/java_files/main_hello_name.java index 84bb282..84bb282 100644 --- a/testapp/java_files/main_hello_name.java +++ b/testapp/exam/java_files/main_hello_name.java diff --git a/testapp/java_files/main_lastDigit.java b/testapp/exam/java_files/main_lastDigit.java index 05439e2..05439e2 100644 --- a/testapp/java_files/main_lastDigit.java +++ b/testapp/exam/java_files/main_lastDigit.java diff --git a/testapp/java_files/main_moreThan30.java b/testapp/exam/java_files/main_moreThan30.java index 7da31cb..7da31cb 100644 --- a/testapp/java_files/main_moreThan30.java +++ b/testapp/exam/java_files/main_moreThan30.java diff --git a/testapp/java_files/main_palindrome.java b/testapp/exam/java_files/main_palindrome.java index c0745f9..c0745f9 100644 --- a/testapp/java_files/main_palindrome.java +++ b/testapp/exam/java_files/main_palindrome.java diff --git a/testapp/java_files/main_square.java b/testapp/exam/java_files/main_square.java index 5cb8c35..5cb8c35 100644 --- a/testapp/java_files/main_square.java +++ b/testapp/exam/java_files/main_square.java diff --git a/testapp/exam/language_registry.py b/testapp/exam/language_registry.py new file mode 100644 index 0000000..76a23d7 --- /dev/null +++ b/testapp/exam/language_registry.py @@ -0,0 +1,36 @@ +from settings import code_evaluators +import importlib + +registry = None + +def set_registry(): + global registry + registry = _LanguageRegistry() + +def get_registry(): + return registry + +class _LanguageRegistry(object): + def __init__(self): + self._register = {} + for language, module in code_evaluators.iteritems(): + self._register[language] = None + + # Public Protocol ########## + def get_class(self, language): + """ Get the code evaluator class for the given language """ + if not self._register.get(language): + self._register[language] = code_evaluators.get(language) + + cls = self._register[language] + module_name, class_name = cls.rsplit(".", 1) + # load the module, will raise ImportError if module cannot be loaded + get_module = importlib.import_module(module_name) + # get the class, will raise AttributeError if class cannot be found + get_class = getattr(get_module, class_name) + return get_class + + def register(self, language, class_name): + """ Register a new code evaluator class for language""" + self._register[language] = class_name + diff --git a/testapp/exam/models.py b/testapp/exam/models.py index 72fb51b..c5043dc 100644 --- a/testapp/exam/models.py +++ b/testapp/exam/models.py @@ -1,4 +1,5 @@ import datetime +import json from random import sample, shuffle from django.db import models from django.contrib.auth.models import User @@ -19,8 +20,8 @@ class Profile(models.Model): languages = ( ("python", "Python"), ("bash", "Bash"), - ("C", "C Language"), - ("C++", "C++ Language"), + ("c", "C Language"), + ("cpp", "C++ Language"), ("java", "Java Language"), ("scilab", "Scilab"), ) @@ -59,9 +60,13 @@ class Question(models.Model): # Number of points for the question. points = models.FloatField(default=1.0) - # Test cases for the question in the form of code that is run. + # Answer for MCQs. test = models.TextField(blank=True) + # Test cases file paths (comma seperated for reference code path and test case code path) + # Applicable for CPP, C, Java and Scilab + ref_code_path = models.TextField(blank=True) + # Any multiple choice options. Place one option per line. options = models.TextField(blank=True) @@ -82,6 +87,41 @@ class Question(models.Model): # Tags for the Question. tags = TaggableManager() + def consolidate_answer_data(self, test_cases, user_answer): + test_case_data_dict = [] + question_info_dict = {} + + for test_case in test_cases: + kw_args_dict = {} + pos_args_list = [] + + test_case_data = {} + test_case_data['test_id'] = test_case.id + test_case_data['func_name'] = test_case.func_name + test_case_data['expected_answer'] = test_case.expected_answer + + if test_case.kw_args: + for args in test_case.kw_args.split(","): + arg_name, arg_value = args.split("=") + kw_args_dict[arg_name.strip()] = arg_value.strip() + + if test_case.pos_args: + for args in test_case.pos_args.split(","): + pos_args_list.append(args.strip()) + + test_case_data['kw_args'] = kw_args_dict + test_case_data['pos_args'] = pos_args_list + test_case_data_dict.append(test_case_data) + + # question_info_dict['language'] = self.language + question_info_dict['id'] = self.id + question_info_dict['user_answer'] = user_answer + question_info_dict['test_parameter'] = test_case_data_dict + question_info_dict['ref_code_path'] = self.ref_code_path + question_info_dict['test'] = self.test + + return json.dumps(question_info_dict) + def __unicode__(self): return self.summary @@ -396,3 +436,20 @@ class AssignmentUpload(models.Model): user = models.ForeignKey(Profile) assignmentQuestion = models.ForeignKey(Question) assignmentFile = models.FileField(upload_to=get_assignment_dir) + + +################################################################################ +class TestCase(models.Model): + question = models.ForeignKey(Question, blank=True, null = True) + + # Test case function name + func_name = models.CharField(blank=True, null = True, max_length=200) + + # Test case Keyword arguments in dict form + kw_args = models.TextField(blank=True, null = True) + + # Test case Positional arguments in list form + pos_args = models.TextField(blank=True, null = True) + + # Test case Expected answer in list form + expected_answer = models.TextField(blank=True, null = True) diff --git a/testapp/exam/python_code_evaluator.py b/testapp/exam/python_code_evaluator.py new file mode 100644 index 0000000..0c473cf --- /dev/null +++ b/testapp/exam/python_code_evaluator.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib + +# local imports +from code_evaluator import CodeEvaluator + + +class PythonCodeEvaluator(CodeEvaluator): + """Tests the Python code obtained from Code Server""" + # Private Protocol ########## + def _check_code(self): + success = False + + try: + tb = None + test_code = self._create_test_case() + submitted = compile(self.user_answer, '<string>', mode='exec') + g = {} + exec submitted in g + _tests = compile(test_code, '<string>', mode='exec') + exec _tests in g + except AssertionError: + type, value, tb = sys.exc_info() + info = traceback.extract_tb(tb) + fname, lineno, func, text = info[-1] + text = str(test_code).splitlines()[lineno-1] + err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + else: + success = True + err = 'Correct answer' + + del tb + return success, err + + def _create_test_case(self): + """ + Create assert based test cases in python + """ + test_code = "" + if self.test: + return self.test + elif self.test_case_data: + for test_case in self.test_case_data: + pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \ + if test_case.get('pos_args') else "" + kw_args = ", ".join(str(k+"="+a) for k, a + in test_case.get('kw_args').iteritems()) \ + if test_case.get('kw_args') else "" + args = pos_args + ", " + kw_args if pos_args and kw_args \ + else pos_args or kw_args + function_name = test_case.get('func_name') + expected_answer = test_case.get('expected_answer') + + tcode = "assert {0}({1}) == {2}".format(function_name, args, + expected_answer) + test_code += tcode + "\n" + return test_code diff --git a/testapp/exam/scilab_code_evaluator.py b/testapp/exam/scilab_code_evaluator.py new file mode 100644 index 0000000..53640cc --- /dev/null +++ b/testapp/exam/scilab_code_evaluator.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +import traceback +import os +from os.path import join, isfile +import subprocess +import re +import importlib + +# local imports +from code_evaluator import CodeEvaluator + + +class ScilabCodeEvaluator(CodeEvaluator): + """Tests the Scilab code obtained from Code Server""" + def __init__(self, test_case_data, test, language, user_answer, + ref_code_path=None, in_dir=None): + super(ScilabCodeEvaluator, self).__init__(test_case_data, test, + language, user_answer, + ref_code_path, in_dir) + self.submit_path = self.create_submit_code_file('function.sci') + self.test_case_args = self._setup() + + # Private Protocol ########## + def _setup(self): + super(ScilabCodeEvaluator, self)._setup() + + ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) + + return ref_path, # Return as a tuple + + def _teardown(self): + # Delete the created file. + super(ScilabCodeEvaluator, self)._teardown() + os.remove(self.submit_path) + + def _check_code(self, ref_path): + success = False + + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) + cmd += ' | timeout 8 scilab-cli -nb' + ret = self._run_command(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + + # Get only the error. + stderr = self._get_error(stdout) + if stderr is None: + # Clean output + stdout = self._strip_output(stdout) + if proc.returncode == 5: + success, err = True, "Correct answer" + else: + err = add_err + stdout + else: + err = add_err + stderr + + return success, err + + def _remove_scilab_exit(self, string): + """ + Removes exit, quit and abort from the scilab code + """ + new_string = "" + i = 0 + for line in string.splitlines(): + new_line = re.sub(r"exit.*$", "", line) + new_line = re.sub(r"quit.*$", "", new_line) + new_line = re.sub(r"abort.*$", "", new_line) + if line != new_line: + i = i + 1 + new_string = new_string + '\n' + new_line + return new_string, i + + def _get_error(self, string): + """ + Fetches only the error from the string. + Returns None if no error. + """ + obj = re.search("!.+\n.+", string) + if obj: + return obj.group() + return None + + def _strip_output(self, out): + """ + Cleans whitespace from the output + """ + strip_out = "Message" + for l in out.split('\n'): + if l.strip(): + strip_out = strip_out+"\n"+l.strip() + return strip_out + diff --git a/testapp/scilab_files/test_add.sce b/testapp/exam/scilab_files/test_add.sce index a317cdb..a317cdb 100644 --- a/testapp/scilab_files/test_add.sce +++ b/testapp/exam/scilab_files/test_add.sce diff --git a/testapp/exam/settings.py b/testapp/exam/settings.py index 682516f..93f90a9 100644 --- a/testapp/exam/settings.py +++ b/testapp/exam/settings.py @@ -18,3 +18,11 @@ SERVER_TIMEOUT = 2 # reason set this to the root you have to serve at. In the above example # host.org/foo/exam set URL_ROOT='/foo' URL_ROOT = '' + +code_evaluators = {"python": "python_code_evaluator.PythonCodeEvaluator", + "c": "c_cpp_code_evaluator.CCPPCodeEvaluator", + "cpp": "c_cpp_code_evaluator.CCPPCodeEvaluator", + "java": "java_evaluator.JavaCodeEvaluator", + "bash": "bash_evaluator.BashCodeEvaluator", + "scilab": "scilab_evaluator.ScilabCodeEvaluator", + } diff --git a/testapp/exam/static/exam/js/add_question.js b/testapp/exam/static/exam/js/add_question.js index 267cdb2..946c139 100644 --- a/testapp/exam/static/exam/js/add_question.js +++ b/testapp/exam/static/exam/js/add_question.js @@ -153,8 +153,7 @@ function textareaformat() if(value == 'mcq' || value == 'mcc') { document.getElementById('id_options').style.visibility='visible'; - document.getElementById('label_option').innerHTML="Options :" - + document.getElementById('label_option').innerHTML="Options :"; } else { @@ -168,7 +167,6 @@ function textareaformat() { document.getElementById('id_options').style.visibility='visible'; document.getElementById('label_option').innerHTML="Options :" - } else { @@ -189,8 +187,9 @@ function autosubmit() if(type.value == 'select') { type.style.border = 'solid red'; - return false; - } + return false; + } + if (type.value == 'mcq' || type.value == 'mcc') { diff --git a/testapp/exam/templates/exam/add_question.html b/testapp/exam/templates/exam/add_question.html index b0b22b1..43f09e1 100644 --- a/testapp/exam/templates/exam/add_question.html +++ b/testapp/exam/templates/exam/add_question.html @@ -27,14 +27,25 @@ <tr><td>Points:<td><button class="btn-mini" type="button" onClick="increase(frm);">+</button>{{ form.points }}<button class="btn-mini" type="button" onClick="decrease(frm);">-</button>{{ form.points.errors }} <tr><td><strong>Rendered: </strong><td><p id='my'></p> <tr><td>Description: <td>{{ form.description}} {{form.description.errors}} - <tr><td>Test: <td>{{ form.test }}{{form.test.errors}} <tr><td>Snippet: <td>{{ form.snippet }}{{ form.snippet.errors }}</td></tD></td></tr> <tr><td>Tags: <td>{{ form.tags }} - <tr><td id='label_option'>Options: <td>{{ form.options }} {{form.options.errors}} - - + <tr><td id='label_option'>Options: <td>{{ form.options }} {{form.options.errors}} + <tr><td id='label_solution'>Test: <td>{{ form.solution }} {{form.solution.errors}} + <tr><td id='label_ref_code_path'>Reference Code Path: <td>{{ form.ref_code_path }} {{form.ref_code_path.errors}} + + <form method="post" action=""> + {% if formset%} + {{ formset.management_form }} + {% for form in formset %} + {{ form }} + {% endfor %} + {% endif %} + </form> </table></center> - <center><button class="btn" type="submit" name="savequestion">Save</button> + <center><button class="btn" type="submit" name="add_test">Add Test Case</button> + <button class="btn" type="submit" name="delete_test">Remove Test Case</button> + </center><br> + <center><button class="btn" type="submit" name="save_question">Save</button> <button class="btn" type="button" name="button" onClick='location.replace("{{URL_ROOT}}/exam/manage/questions/");'>Cancel</button> </center> </form> {% endblock %} diff --git a/testapp/exam/templates/exam/edit_question.html b/testapp/exam/templates/exam/edit_question.html index b28cc3e..6deca4a 100644 --- a/testapp/exam/templates/exam/edit_question.html +++ b/testapp/exam/templates/exam/edit_question.html @@ -21,29 +21,37 @@ <table> - {% for form in forms %} + {% for question, test in data_list %} - <tr><td height=10><a id='a{{forloop.counter}}' onClick="data('contentDiv{{forloop.counter}}','myContent{{forloop.counter}}','a{{forloop.counter}}','{{form.summary.value}}');" style='cursor:pointer;'>{{form.summary.value}}</a> - + <tr><td height=10><a id='a{{forloop.counter}}' onClick="data('contentDiv{{forloop.counter}}','myContent{{forloop.counter}}','a{{forloop.counter}}','{{question.summary.value}}');" style='cursor:pointer;'>{{question.summary.value}}</a> + <div id="contentDiv{{forloop.counter}}" style="display:none;"> <div id="myContent{{forloop.counter}}" style="display: none;"> - - <center><table class=span1> - <tr><td><b>Summary:</b> <td>{{ form.summary }}{{ form.summary.errors }} - <tr><td><b> Language: </b><td> {{form.language}}{{form.language.errors}} - <tr><td><b> Active: </b><td> {{ form.active }}{{form.active.errors}} Type: {{ form.type }}{{form.type.errors}} - <tr><td><b>Points:<td><button class="btn-mini" name={{forloop.counter}} type="button" onClick="increase(frm,{{forloop.counter}});">+</button>{{ form.points }}<button class="btn-mini" type="button" onClick="decrease(frm,{{forloop.counter}});">-</button>{{ form.points.errors }} + + <center><table class=span1> + <tr><td><b>Summary:</b> <td>{{ question.summary }}{{ question.summary.errors }} + <tr><td><b> Language: </b><td> {{question.language}}{{question.language.errors}} + <tr><td><b> Active: </b><td> {{ question.active }}{{question.active.errors}} Type: {{ question.type }}{{question.type.errors}} + <tr><td><b>Points:<td><button class="btn-mini" name={{forloop.counter}} type="button" onClick="increase(frm,{{forloop.counter}});">+</button>{{ question.points }}<button class="btn-mini" type="button" onClick="decrease(frm,{{forloop.counter}});">-</button>{{ question.points.errors }} <tr><td><strong>Rendered: </strong><td><p id='my{{forloop.counter}}'></p> - <tr><td><b>Description: <td>{{ form.description }} {{form.description.errors}} - <tr><td><b>Test: <td>{{ form.test }}{{form.test.errors}} - <tr><td><b>Snippet: <td>{{ form.snippet }}{{ form.snippet.errors }}</td></b></td></tr> - <tr><td><b>Tags: </b><td>{{ form.tags }} - <tr><td id='label_option{{forloop.counter}}'><b>Options:<td>{{ form.options }} {{form.options.errors}} {{form.options.helptext}} + <tr><td><b>Description: <td>{{ question.description }} + {{question.description.errors}} <tr><td><b>Test: <td> + {{ question.test }}{{question.test.errors}} + <tr><td><b>Snippet: <td>{{ question.snippet }}{{ question.snippet.errors }} + </td></b></td></tr> + <tr><td><b>Tags: </b><td>{{ question.tags }} + <tr><td id='label_option{{forloop.counter}}'><b>Options:<td>{{ question.options }} + {{question.options.errors}} {{question.options.helptext}} + </table></center> + <center><table class=span1> + {{ test }} </table></center> </div> </div> {% endfor %} </table></center> + + {% for i in data %} <input type=hidden name='questions' value="{{ i }}" /> {% endfor %} diff --git a/testapp/exam/tests.py b/testapp/exam/tests.py index d76e4f8..ff48c25 100644 --- a/testapp/exam/tests.py +++ b/testapp/exam/tests.py @@ -1,8 +1,7 @@ from django.utils import unittest from exam.models import User, Profile, Question, Quiz, QuestionPaper,\ - QuestionSet, AnswerPaper, Answer -import datetime - + QuestionSet, AnswerPaper, Answer, TestCase +import datetime, json def setUpModule(): # create user profile @@ -51,12 +50,31 @@ class ProfileTestCases(unittest.TestCase): class QuestionTestCases(unittest.TestCase): def setUp(self): # Single question details + # self.question = Question(summary='Demo question', language='Python', + # type='Code', active=True, + # description='Write a function', points=1.0, + # test='Test Cases', snippet='def myfunc()') self.question = Question(summary='Demo question', language='Python', type='Code', active=True, description='Write a function', points=1.0, - test='Test Cases', snippet='def myfunc()') + snippet='def myfunc()') self.question.save() self.question.tags.add('python', 'function') + self.testcase = TestCase(question=self.question, + func_name='def myfunc', kw_args='a=10,b=11', + pos_args='12,13', expected_answer='15') + answer_data = {"user_answer": "demo_answer", + "test_parameter": [{"func_name": "def myfunc", + "expected_answer": "15", + "test_id": self.testcase.id, + "pos_args": ["12", "13"], + "kw_args": {"a": "10", + "b": "11"} + }], + "id": self.question.id, + "language": "Python"} + self.answer_data_json = json.dumps(answer_data) + self.user_answer = "demo_answer" def test_question(self): """ Test question """ @@ -67,13 +85,40 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(self.question.description, 'Write a function') self.assertEqual(self.question.points, 1.0) self.assertTrue(self.question.active) - self.assertEqual(self.question.test, 'Test Cases') self.assertEqual(self.question.snippet, 'def myfunc()') tag_list = [] for tag in self.question.tags.all(): tag_list.append(tag.name) self.assertEqual(tag_list, ['python', 'function']) + def test_consolidate_answer_data(self): + """ Test consolidate_answer_data function """ + result = self.question.consolidate_answer_data([self.testcase], + self.user_answer) + self.assertEqual(result, self.answer_data_json) + + + +############################################################################### +class TestCaseTestCases(unittest.TestCase): + def setUp(self): + self.question = Question(summary='Demo question', language='Python', + type='Code', active=True, + description='Write a function', points=1.0, + snippet='def myfunc()') + self.question.save() + self.testcase = TestCase(question=self.question, + func_name='def myfunc', kw_args='a=10,b=11', + pos_args='12,13', expected_answer='15') + + def test_testcase(self): + """ Test question """ + self.assertEqual(self.testcase.question, self.question) + self.assertEqual(self.testcase.func_name, 'def myfunc') + self.assertEqual(self.testcase.kw_args, 'a=10,b=11') + self.assertEqual(self.testcase.pos_args, '12,13') + self.assertEqual(self.testcase.expected_answer, '15') + ############################################################################### class QuizTestCases(unittest.TestCase): diff --git a/testapp/exam/views.py b/testapp/exam/views.py index 11aca06..5b7baac 100644 --- a/testapp/exam/views.py +++ b/testapp/exam/views.py @@ -14,12 +14,13 @@ from django.db.models import Sum from django.views.decorators.csrf import csrf_exempt from taggit.models import Tag from itertools import chain +import json # Local imports. from testapp.exam.models import Quiz, Question, QuestionPaper, QuestionSet -from testapp.exam.models import Profile, Answer, AnswerPaper, User +from testapp.exam.models import Profile, Answer, AnswerPaper, User, TestCase from testapp.exam.forms import UserRegisterForm, UserLoginForm, QuizForm,\ - QuestionForm, RandomQuestionForm -from testapp.exam.xmlrpc_clients import code_server + QuestionForm, RandomQuestionForm, TestCaseFormSet +from exam.xmlrpc_clients import code_server from settings import URL_ROOT from testapp.exam.models import AssignmentUpload @@ -281,16 +282,14 @@ def edit_quiz(request): def edit_question(request): - """Edit the list of questions seleted by the user for editing.""" + """Edit the list of questions selected by the user for editing.""" user = request.user if not user.is_authenticated() or not is_moderator(user): raise Http404('You are not allowed to view this page!') - question_list = request.POST.getlist('questions') summary = request.POST.getlist('summary') description = request.POST.getlist('description') points = request.POST.getlist('points') - test = request.POST.getlist('test') options = request.POST.getlist('options') type = request.POST.getlist('type') active = request.POST.getlist('active') @@ -298,14 +297,21 @@ def edit_question(request): snippet = request.POST.getlist('snippet') for j, question_id in enumerate(question_list): question = Question.objects.get(id=question_id) + test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=question) + if test_case_formset.is_valid(): + test_case_instance = test_case_formset.save(commit=False) + for i in test_case_instance: + i.save() + question.summary = summary[j] question.description = description[j] question.points = points[j] - question.test = test[j] question.options = options[j] question.active = active[j] question.language = language[j] question.snippet = snippet[j] + question.ref_code_path = ref_code_path[j] + question.test = test[j] question.type = type[j] question.save() return my_redirect("/exam/manage/questions") @@ -314,6 +320,16 @@ def edit_question(request): def add_question(request, question_id=None): """To add a new question in the database. Create a new question and store it.""" + + def add_or_delete_test_form(post_request, instance): + request_copy = post_request.copy() + if 'add_test' in post_request: + request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) + 1 + elif 'delete_test' in post_request: + request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) - 1 + test_case_formset = TestCaseFormSet(request_copy, prefix='test', instance=instance) + return test_case_formset + user = request.user ci = RequestContext(request) if not user.is_authenticated() or not is_moderator(user): @@ -321,44 +337,88 @@ def add_question(request, question_id=None): if request.method == "POST": form = QuestionForm(request.POST) if form.is_valid(): - data = form.cleaned_data if question_id is None: - form.save() - question = Question.objects.order_by("-id")[0] - tags = form['tags'].data.split(',') - for i in range(0, len(tags)-1): - tag = tags[i].strip() - question.tags.add(tag) - return my_redirect("/exam/manage/questions") + test_case_formset = add_or_delete_test_form(request.POST, form.save(commit=False)) + if 'save_question' in request.POST: + qtn = form.save(commit=False) + test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=qtn) + form.save() + question = Question.objects.order_by("-id")[0] + tags = form['tags'].data.split(',') + for i in range(0, len(tags)-1): + tag = tags[i].strip() + question.tags.add(tag) + if test_case_formset.is_valid(): + test_case_formset.save() + else: + return my_render_to_response('exam/add_question.html', + {'form': form, + 'formset': test_case_formset}, + context_instance=ci) + + return my_redirect("/exam/manage/questions") + + return my_render_to_response('exam/add_question.html', + {'form': form, + 'formset': test_case_formset}, + context_instance=ci) + else: d = Question.objects.get(id=question_id) - d.summary = form['summary'].data - d.description = form['description'].data - d.points = form['points'].data - d.test = form['test'].data - d.options = form['options'].data - d.type = form['type'].data - d.active = form['active'].data - d.language = form['language'].data - d.snippet = form['snippet'].data - d.save() - question = Question.objects.get(id=question_id) - for tag in question.tags.all(): - question.tags.remove(tag) - tags = form['tags'].data.split(',') - for i in range(0, len(tags)-1): - tag = tags[i].strip() - question.tags.add(tag) - return my_redirect("/exam/manage/questions") + test_case_formset = add_or_delete_test_form(request.POST, d) + if 'save_question' in request.POST: + d.summary = form['summary'].data + d.description = form['description'].data + d.points = form['points'].data + d.options = form['options'].data + d.type = form['type'].data + d.active = form['active'].data + d.language = form['language'].data + d.snippet = form['snippet'].data + d.ref_code_path = form['ref_code_path'].data + d.test = form['test'].data + d.save() + question = Question.objects.get(id=question_id) + for tag in question.tags.all(): + question.tags.remove(tag) + tags = form['tags'].data.split(',') + for i in range(0, len(tags)-1): + tag = tags[i].strip() + question.tags.add(tag) + + test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=question) + if test_case_formset.is_valid(): + test_case_instance = test_case_formset.save(commit=False) + for i in test_case_instance: + i.save() + else: + return my_render_to_response('exam/add_question.html', + {'form': form, + 'formset': test_case_formset}, + context_instance=ci) + + + return my_redirect("/exam/manage/questions") + return my_render_to_response('exam/add_question.html', + {'form': form, + 'formset': test_case_formset}, + context_instance=ci) + else: + test_case_formset = add_or_delete_test_form(request.POST, form.save(commit=False)) return my_render_to_response('exam/add_question.html', - {'form': form}, + {'form': form, + 'formset': test_case_formset}, context_instance=ci) else: + form = QuestionForm() + test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) if question_id is None: form = QuestionForm() + test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) return my_render_to_response('exam/add_question.html', - {'form': form}, + {'form': form, + 'formset': test_case_formset}, context_instance=ci) else: d = Question.objects.get(id=question_id) @@ -366,12 +426,13 @@ def add_question(request, question_id=None): form.initial['summary'] = d.summary form.initial['description'] = d.description form.initial['points'] = d.points - form.initial['test'] = d.test form.initial['options'] = d.options form.initial['type'] = d.type form.initial['active'] = d.active form.initial['language'] = d.language form.initial['snippet'] = d.snippet + form.initial['ref_code_path'] = d.ref_code_path + form.initial['test'] = d.test form_tags = d.tags.all() form_tags_split = form_tags.values('name') initial_tags = "" @@ -380,8 +441,13 @@ def add_question(request, question_id=None): if (initial_tags == ","): initial_tags = "" form.initial['tags'] = initial_tags + + test_case_formset = TestCaseFormSet(prefix='test', + instance=d) + return my_render_to_response('exam/add_question.html', - {'form': form}, + {'form': form, + 'formset': test_case_formset}, context_instance=ci) @@ -848,6 +914,10 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): if not user.is_authenticated() or paper.end_time < datetime.datetime.now(): return my_redirect('/exam/login/') question = get_object_or_404(Question, pk=q_id) + q_paper = QuestionPaper.objects.get(id=questionpaper_id) + paper = AnswerPaper.objects.get(user=request.user, question_paper=q_paper) + test_cases = TestCase.objects.filter(question=question) + snippet_code = request.POST.get('snippet') user_code = request.POST.get('answer') skip = request.POST.get('skip', None) @@ -876,7 +946,8 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): assign.save() user_answer = 'ASSIGNMENT UPLOADED' else: - user_answer = snippet_code + "\n" + user_code + user_code = request.POST.get('answer') + user_answer = snippet_code + "\n" + user_code if snippet_code else user_code new_answer = Answer(question=question, answer=user_answer, correct=False) @@ -887,18 +958,20 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): # questions, we obtain the results via XML-RPC with the code executed # safely in a separate process (the code_server.py) running as nobody. if not question.type == 'upload': - correct, success, err_msg = validate_answer(user, user_answer, question) + json_data = question.consolidate_answer_data(test_cases, user_answer) \ + if question.type == 'code' else None + correct, result = validate_answer(user, user_answer, question, json_data) if correct: new_answer.correct = correct new_answer.marks = question.points - new_answer.error = err_msg + new_answer.error = result.get('error') success_msg = True else: - new_answer.error = err_msg + new_answer.error = result.get('error') new_answer.save() time_left = paper.time_left() - if not success: # Should only happen for non-mcq questions. + if not result.get('success'): # Should only happen for non-mcq questions. if time_left == 0: reason = 'Your time is up!' return complete(request, reason, attempt_num, questionpaper_id) @@ -913,8 +986,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): if old_answer: old_answer[0].answer = user_code old_answer[0].save() - context = {'question': question, 'questions': questions, - 'error_message': err_msg, + context = {'question': question, 'error_message': result.get('error'), 'paper': paper, 'last_attempt': user_code, 'quiz_name': paper.question_paper.quiz.description, 'time_left': time_left, 'to_attempt': to_attempt, @@ -933,7 +1005,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): questionpaper_id, success_msg) -def validate_answer(user, user_answer, question): +def validate_answer(user, user_answer, question, json_data=None): """ Checks whether the answer submitted by the user is right or wrong. If right then returns correct = True, success and @@ -942,9 +1014,9 @@ def validate_answer(user, user_answer, question): only one attempt are allowed for them. For code questions success is True only if the answer is correct. """ - success = True + + result = {'success': True, 'error': 'Incorrect answer'} correct = False - message = 'Incorrect answer' if user_answer is not None: if question.type == 'mcq': @@ -958,11 +1030,12 @@ def validate_answer(user, user_answer, question): message = 'Correct answer' elif question.type == 'code': user_dir = get_user_dir(user) - success, message = code_server.run_code(user_answer, question.test, - user_dir, question.language) - if success: + json_result = code_server.run_code(question.language, json_data, user_dir) + result = json.loads(json_result) + if result.get('success'): correct = True - return correct, success, message + + return correct, result def quit(request, attempt_num=None, questionpaper_id=None): @@ -1167,18 +1240,20 @@ def show_all_questions(request): data = request.POST.getlist('question') forms = [] + formsets = [] for j in data: d = Question.objects.get(id=j) form = QuestionForm() form.initial['summary'] = d.summary form.initial['description'] = d.description form.initial['points'] = d.points - form.initial['test'] = d.test form.initial['options'] = d.options form.initial['type'] = d.type form.initial['active'] = d.active form.initial['language'] = d.language form.initial['snippet'] = d.snippet + form.initial['ref_code_path'] = d.ref_code_path + form.initial['test'] = d.test form_tags = d.tags.all() form_tags_split = form_tags.values('name') initial_tags = "" @@ -1188,8 +1263,13 @@ def show_all_questions(request): initial_tags = "" form.initial['tags'] = initial_tags forms.append(form) + test_case_formset = TestCaseFormSet(prefix='test', instance=d) + formsets.append(test_case_formset) + data_list = zip(forms, formsets) + return my_render_to_response('exam/edit_question.html', - {'forms': forms, 'data': data}, + {'data': data, + 'data_list': data_list}, context_instance=ci) else: questions = Question.objects.all() diff --git a/testapp/exam/xmlrpc_clients.py b/testapp/exam/xmlrpc_clients.py index 14ebf27..8f5642e 100644 --- a/testapp/exam/xmlrpc_clients.py +++ b/testapp/exam/xmlrpc_clients.py @@ -21,15 +21,8 @@ class CodeServerProxy(object): def __init__(self): pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT) self.pool_server = ServerProxy(pool_url) - self.methods = {"python": 'run_python_code', - "bash": 'run_bash_code', - "C": "run_c_code", - "C++": "run_cplus_code", - "java": "run_java_code", - "scilab": "run_scilab_code", - } - def run_code(self, answer, test_code, user_dir, language): + def run_code(self, language, json_data, user_dir): """Tests given code (`answer`) with the `test_code` supplied. If the optional `in_dir` keyword argument is supplied it changes the directory to that directory (it does not change it back to the original when @@ -38,26 +31,28 @@ class CodeServerProxy(object): Parameters ---------- - answer : str - The user's answer for the question. + json_data contains; + user_answer : str + The user's answer for the question. test_code : str The test code to check the user code with. - user_dir : str (directory) - The directory to run the tests inside. language : str The programming language to use. + user_dir : str (directory) + The directory to run the tests inside. + + Returns ------- - A tuple: (success, error message). + A json string of a dict: {success: success, err: error message}. """ - method_name = self.methods[language] + try: server = self._get_server() - method = getattr(server, method_name) - result = method(answer, test_code, user_dir) + result = server.check_code(language, json_data, user_dir) except ConnectionError: - result = [False, 'Unable to connect to any code servers!'] + result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'}) return result def _get_server(self): diff --git a/testapp/test_server.py b/testapp/test_server.py deleted file mode 100644 index d22a022..0000000 --- a/testapp/test_server.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Simple test suite for the code server. Running this requires that one start -up the code server as:: - - $ sudo ./code_server.py - -""" -from exam.xmlrpc_clients import code_server - - -def check_result(result, check='correct answer'): - if check != 'correct answer': - assert result[0] == False - else: - assert result[0] == True - if "unable to connect" in result[1].lower(): - assert result[0], result[1] - assert check in result[1].lower(), result[1] - -def test_python(): - """Test if server runs Python code as expected.""" - src = 'while True: pass' - result = code_server.run_code(src, '', '/tmp', language="python") - check_result(result, 'more than ') - src = 'x = 1' - result = code_server.run_code(src, 'assert x == 1', '/tmp', - language="python") - check_result(result, 'correct answer') - - result = code_server.run_code(src, 'assert x == 0', '/tmp', - language="python") - check_result(result, 'assertionerror') - - src = 'abracadabra' - result = code_server.run_code(src, 'assert x == 0', '/tmp', - language="python") - check_result(result, 'nameerror') - - -def test_c(): - """Test if server runs c code as expected.""" - src = """ - #include<stdiol.h> - int ad(int a, int b) - {return a+b;} - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C") - check_result(result, 'error') - - src = """ - int add(int a, int b) - {return a+b} - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C") - check_result(result, 'compilation error') - - src = """ - int add(int a, int b) - {while(1>0){} - return a+b;} - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C") - check_result(result, 'more than') - - src = """ - int add(int a, int b) - {return a+b;} - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C") - check_result(result, 'correct answer') - - src = """ - #include<stdio.h> - int add(int a, int b) - {printf("All Correct");} - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C") - check_result(result, 'incorrect') - - -def test_cpp(): - """Test if server runs c code as expected.""" - src = """ - int add(int a, int b) - { - return a+b - } - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C++") - check_result(result, 'error') - - src = """ - int add(int a, int b) - { - return a+b; - } - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C++") - check_result(result, 'correct answer') - - src = """ - int dd(int a, int b) - { - return a+b; - } - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C++") - check_result(result, 'error') - - src = """ - int add(int a, int b) - { - while(0==0) - {} - return a+b; - } - """ - result = code_server.run_code(src, 'c_cpp_files/main.cpp', - '/tmp', language="C++") - check_result(result, 'more than') - - -def test_java(): - """Test if server runs java code as expected.""" - src = """ - class Test - { - int square_num(int a) - { - return a*a; - } - } - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'correct answer') - - src = """ - class Test - { - int square_num(int a) - { - return b*b; - } - } - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'error') - - src = """ - class Test - { - int square_nu(int a) - { - return a*a; - } - } - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'error') - - src = """ - class Test - { - int square_num(int a) - { - while(0==0) - {} - } - } - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'more than') - - src = """ - class Test - { - int square_num(int a) - { - return a+b - } - } - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'error') - - src = """ - class Test - { - int square_num(int a) - { - return a+b - """ - result = code_server.run_code(src, 'java_files/main_square.java', - '/tmp', language="java") - check_result(result, 'error') - -def test_scilab(): - """Test if server runs scilab code as expected.""" - src = """ - funcprot(0) -function[c]=add(a,b) - c=a+b; -endfunction - """ - result = code_server.run_code(src, 'scilab_files/test_add.sce', - '/tmp', language="scilab") - check_result(result, 'correct answer') - - src = """ - funcprot(0) -function[c]=add(a,b) - c=a-b; -endfunction - """ - result = code_server.run_code(src, 'scilab_files/test_add.sce', - '/tmp', language="scilab") - check_result(result, 'correct answer') - - src = """ - funcprot(0) -function[c]=add(a,b) - c=a+b; -dis( -endfunction - """ - result = code_server.run_code(src, 'scilab_files/test_add.sce', - '/tmp', language="scilab") - check_result(result, 'error') - - src = """ - funcprot(0) -function[c]=add(a,b) - c=a - while(1==1) - end -endfunction - """ - result = code_server.run_code(src, 'scilab_files/test_add.sce', - '/tmp', language="scilab") - check_result(result, 'error') - -def test_bash(): - """Test if server runs Bash code as expected.""" - src = """ -#!/bin/bash - [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 )) - """ - result = code_server.run_code(src, 'docs/sample.sh\ndocs/sample.args', - '/tmp', language="bash") - check_result(result) - - src = """ -#!/bin/bash - [[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 )) - """ - result = code_server.run_code(src, 'docs/sample.sh\ndocs/sample.args', - '/tmp', language="bash") - check_result(result, 'error') - - src = """\ -#!/bin/bash - while [ 1 ] ; do echo "" > /dev/null ; done - """ - result = code_server.run_code(src, 'docs/sample.sh\ndocs/sample.args', - '/tmp', language="bash") - check_result(result, 'more than ') - - src = ''' -#!/bin/bash - while [ 1 ] ; do echo "" > /dev/null - ''' - result = code_server.run_code(src, 'docs/sample.sh\ndocs/sample.args', - '/tmp', language="bash") - check_result(result, 'error') - - src = '''# Enter your code here. -#!/bin/bash - while [ 1 ] ; do echo "" > /dev/null - ''' - result = code_server.run_code(src, 'docs/sample.sh\ndocs/sample.args', - '/tmp', language="bash") - check_result(result, 'oserror') - -if __name__ == '__main__': - test_python() - test_bash() - test_c() - test_cpp() - test_java() - test_scilab() diff --git a/testapp/tests/__init__.py b/testapp/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/testapp/tests/__init__.py diff --git a/testapp/tests/test_bash_evaluation.py b/testapp/tests/test_bash_evaluation.py new file mode 100644 index 0000000..3ae3b0b --- /dev/null +++ b/testapp/tests/test_bash_evaluation.py @@ -0,0 +1,41 @@ +import unittest +import os +from exam.bash_code_evaluator import BashCodeEvaluator +from exam.settings import SERVER_TIMEOUT + +class BashEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "bash" + self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + self.test = None + + def test_correct_answer(self): + user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" + get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/testapp/tests/test_c_cpp_evaluation.py b/testapp/tests/test_c_cpp_evaluation.py new file mode 100644 index 0000000..b820963 --- /dev/null +++ b/testapp/tests/test_c_cpp_evaluation.py @@ -0,0 +1,77 @@ +import unittest +import os +from exam.cpp_code_evaluator import CppCodeEvaluator +from exam.settings import SERVER_TIMEOUT + +class CEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "C" + self.ref_code_path = "c_cpp_files/main.cpp" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + self.test = None + + def test_correct_answer(self): + user_answer = "int add(int a, int b)\n{return a+b;}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_compilation_error(self): + user_answer = "int add(int a, int b)\n{return a+b}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "int add(int a, int b)\n{while(1>0){}}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + +############################################################################### +class CppEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "CPP" + self.ref_code_path = "c_cpp_files/main.cpp" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + self.test = None + + def test_correct_answer(self): + user_answer = "int add(int a, int b)\n{return a+b;}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_compilation_error(self): + user_answer = "int add(int a, int b)\n{return a+b}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "int add(int a, int b)\n{while(1>0){}}" + get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/testapp/tests/test_code_evaluation.py b/testapp/tests/test_code_evaluation.py new file mode 100644 index 0000000..fa2b1fa --- /dev/null +++ b/testapp/tests/test_code_evaluation.py @@ -0,0 +1,24 @@ +import unittest +import os +from exam import python_code_evaluator +from exam.language_registry import _LanguageRegistry, set_registry, get_registry +from exam.settings import SERVER_TIMEOUT + + +class RegistryTestCase(unittest.TestCase): + def setUp(self): + set_registry() + self.registry_object = get_registry() + self.language_registry = _LanguageRegistry() + + def test_set_register(self): + class_name = getattr(python_code_evaluator, 'PythonCodeEvaluator') + self.registry_object.register("python", "exam.python_code_evaluator.PythonCodeEvaluator") + self.assertEquals(self.registry_object.get_class("python"), class_name) + + def tearDown(self): + self.registry_object = None + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/testapp/tests/test_java_evaluation.py b/testapp/tests/test_java_evaluation.py new file mode 100644 index 0000000..d86d7b3 --- /dev/null +++ b/testapp/tests/test_java_evaluation.py @@ -0,0 +1,41 @@ +import unittest +import os +from exam.java_code_evaluator import JavaCodeEvaluator +from exam.settings import SERVER_TIMEOUT + +class JavaEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "java" + self.ref_code_path = "java_files/main_square.java" + self.in_dir = "/tmp" + self.test_case_data = [] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + self.test = None + + def test_correct_answer(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" + get_class = JavaCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" + get_class = JavaCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" + get_class = JavaCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/testapp/tests/test_python_evaluation.py b/testapp/tests/test_python_evaluation.py new file mode 100644 index 0000000..57e111c --- /dev/null +++ b/testapp/tests/test_python_evaluation.py @@ -0,0 +1,54 @@ + +import unittest +import os +from exam.python_code_evaluator import PythonCodeEvaluator +from exam.settings import SERVER_TIMEOUT + +class PythonEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "Python" + self.test = None + self.test_case_data = [{"func_name": "add", + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} + }] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = "def add(a, b):\n\treturn a + b""" + get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_class.evaluate() + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_incorrect_answer(self): + user_answer = "def add(a, b):\n\treturn a - b" + test_case_data = [{"func_name": "add", + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} + }] + get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_class.evaluate() + self.assertFalse(result.get("success")) + self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") + + def test_infinite_loop(self): + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass""" + test_case_data = [{"func_name": "add", + "expected_answer": "5", + "test_id": u'null', + "pos_args": ["3", "2"], + "kw_args": {} + }] + get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) + result = get_class.evaluate() + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/testapp/tests/test_scilab_evaluation.py b/testapp/tests/test_scilab_evaluation.py new file mode 100644 index 0000000..072fff6 --- /dev/null +++ b/testapp/tests/test_scilab_evaluation.py @@ -0,0 +1,39 @@ +import unittest +import os +from exam.scilab_code_evaluator import ScilabCodeEvaluator +from exam.settings import SERVER_TIMEOUT + +class ScilabEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.language = "scilab" + self.ref_code_path = "scilab_files/test_add.sce" + self.in_dir = "/tmp" + self.test_case_data = [] + self.test = None + + def test_correct_answer(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" + get_class = ScilabCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_correct_answer_2(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction" + get_class = ScilabCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertTrue(result.get("success")) + self.assertEqual(result.get("error"), "Correct answer") + + def test_error(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\t c=a+b;\ndis(\tendfunction" + get_class = ScilabCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + result = get_class.evaluate() + + self.assertFalse(result.get("success")) + self.assertTrue("Error" in result.get("error")) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file |