From bf5b4e7607bae0b81ceeb99e8bf5d750433e92e8 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 20 Dec 2016 12:42:44 +0530 Subject: Fix errors and rename resources - code_evaluator module and class renamed to grader - Test cases fixed - Comments removed - weight variable renamed to mark --- yaksh/admin.py | 4 +- yaksh/base_evaluator.py | 2 +- yaksh/bash_stdio_evaluator.py | 2 +- yaksh/code_evaluator.py | 178 ------------------------ yaksh/code_server.py | 12 +- yaksh/cpp_stdio_evaluator.py | 2 +- yaksh/evaluator_tests/test_bash_evaluation.py | 38 ++--- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 82 +++++------ yaksh/evaluator_tests/test_code_evaluation.py | 4 +- yaksh/evaluator_tests/test_java_evaluation.py | 83 +++++------ yaksh/evaluator_tests/test_python_evaluation.py | 80 +++++------ yaksh/evaluator_tests/test_scilab_evaluation.py | 30 ++-- yaksh/forms.py | 4 +- yaksh/grader.py | 178 ++++++++++++++++++++++++ yaksh/java_stdio_evaluator.py | 2 +- yaksh/language_registry.py | 7 +- yaksh/models.py | 2 +- yaksh/python_assertion_evaluator.py | 2 +- yaksh/python_stdio_evaluator.py | 2 +- yaksh/scilab_code_evaluator.py | 8 +- yaksh/settings.py | 30 +--- yaksh/test_models.py | 4 +- yaksh/test_views.py | 2 +- yaksh/views.py | 2 +- 24 files changed, 362 insertions(+), 398 deletions(-) delete mode 100644 yaksh/code_evaluator.py create mode 100644 yaksh/grader.py diff --git a/yaksh/admin.py b/yaksh/admin.py index 58af7b2..c525ba3 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -1,11 +1,11 @@ from yaksh.models import Question, Quiz, QuestionPaper -from yaksh.models import TestCase, StandardTestCase, StdioBasedTestCase, Course, AnswerPaper +from yaksh.models import TestCase, StandardTestCase, StdIOBasedTestCase, Course, AnswerPaper from django.contrib import admin admin.site.register(Question) admin.site.register(TestCase) admin.site.register(StandardTestCase) -admin.site.register(StdioBasedTestCase) +admin.site.register(StdIOBasedTestCase) admin.site.register(Course) admin.site.register(Quiz) admin.site.register(QuestionPaper) diff --git a/yaksh/base_evaluator.py b/yaksh/base_evaluator.py index b290ba4..ce1647f 100644 --- a/yaksh/base_evaluator.py +++ b/yaksh/base_evaluator.py @@ -10,7 +10,7 @@ import stat # Local imports -from .code_evaluator import MY_DIR, TimeoutException +from .grader import MY_DIR, TimeoutException class BaseEvaluator(object): """Base Evaluator class containing generic attributes and callable methods""" diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 63bf3dc..50ee0d6 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class BashStdioEvaluator(StdIOEvaluator): +class BashStdIOEvaluator(StdIOEvaluator): """Evaluates Bash StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py deleted file mode 100644 index 095099b..0000000 --- a/yaksh/code_evaluator.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python -from __future__ import unicode_literals -import sys -import pwd -import os -import stat -import contextlib -from os.path import isdir, dirname, abspath, join, isfile, exists -import signal -import traceback -from multiprocessing import Process, Queue -import subprocess -import re - -try: - from SimpleXMLRPCServer import SimpleXMLRPCServer -except ImportError: - # The above import will not work on Python-3.x. - from xmlrpc.server import SimpleXMLRPCServer - -# Local imports -from .settings import SERVER_TIMEOUT -from .language_registry import create_evaluator_instance - - -MY_DIR = abspath(dirname(__file__)) - -registry = None - -# Raised when the code times-out. -# c.f. http://pguides.net/python/timeout-a-function -class TimeoutException(Exception): - pass - -@contextlib.contextmanager -def change_dir(path): - cur_dir = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(cur_dir) - - -def timeout_handler(signum, frame): - """A handler for the ALARM signal.""" - raise TimeoutException('Code took too long to run.') - - -def create_signal_handler(): - """Add a new signal handler for the execution of this code.""" - prev_handler = signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(SERVER_TIMEOUT) - return prev_handler - - -def set_original_signal_handler(old_handler=None): - """Set back any original signal handler.""" - if old_handler is not None: - signal.signal(signal.SIGALRM, old_handler) - return - else: - raise Exception("Signal Handler: object cannot be NoneType") - - -def delete_signal_handler(): - signal.alarm(0) - return - - -class CodeEvaluator(object): - """Tests the code obtained from Code Server""" - def __init__(self, in_dir=None): - msg = 'Code took more than %s seconds to run. You probably '\ - 'have an infinite loop in your code.' % SERVER_TIMEOUT - self.timeout_msg = msg - self.in_dir = in_dir if in_dir else MY_DIR - - - def evaluate(self, kwargs): #language, test_case_type, - """Evaluates given code with the test cases based on - given arguments in test_case_data. - - The ref_code_path is a path to the reference code. - The reference code will call the function submitted by the student. - The reference code will check for the expected output. - - If the path's start with a "/" then we assume they are absolute paths. - If not, we assume they are relative paths w.r.t. the location of this - code_server script. - - If the optional `in_dir` keyword argument is supplied it changes the - directory to that directory (it does not change it back to the original - when done). - - Returns - ------- - - A tuple: (success, error message, weight). - """ - - self.setup() - test_case_instances = self.get_evaluator_objects(kwargs) - with change_dir(self.in_dir): - success, error, weight = self.safe_evaluate(test_case_instances) - self.teardown() - - result = {'success': success, 'error': error, 'weight': weight} - return result - - # Private Protocol ########## - def setup(self): - if self.in_dir: - if not os.path.exists(self.in_dir): - os.makedirs(self.in_dir) - - def get_evaluator_objects(self, kwargs): - metadata = kwargs.get('metadata') - test_case_data = kwargs.get('test_case_data') - test_case_instances = [] - - for test_case in test_case_data: - test_case_instance = create_evaluator_instance(metadata, test_case) - test_case_instances.append(test_case_instance) - - return test_case_instances - - - def safe_evaluate(self, test_case_instances): - """ - Handles code evaluation along with compilation, signal handling - and Exception handling - """ - # Add a new signal handler for the execution of this code. - prev_handler = create_signal_handler() - success = False - test_case_success_status = [False] * len(test_case_instances) - error = "" - weight = 0.0 - - # Do whatever testing needed. - try: - # Run evaluator selection registry here - for idx, test_case_instance in enumerate(test_case_instances): - test_case_success = False - test_case_instance.compile_code() - test_case_success, err, mark_fraction = test_case_instance.check_code() - if test_case_success: - weight += mark_fraction - - error += err + "\n" - test_case_success_status[idx] = test_case_success - - success = all(test_case_success_status) - - for test_case_instance in test_case_instances: - test_case_instance.teardown() - - except TimeoutException: - error = self.timeout_msg - except OSError: - msg = traceback.format_exc(limit=0) - error = "Error: {0}".format(msg) - except Exception as e: - exc_type, exc_value, exc_tb = sys.exc_info() - tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) - if len(tb_list) > 2: - del tb_list[1:3] - error = "Error: {0}".format("".join(tb_list)) - finally: - # Set back any original signal handler. - set_original_signal_handler(prev_handler) - - return success, error, weight - - def teardown(self): - # Cancel the signal - delete_signal_handler() diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 815eb55..4db5810 100644 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -54,7 +54,7 @@ from tornado.web import Application, RequestHandler # Local imports from .settings import SERVER_PORTS, SERVER_POOL_PORT from .language_registry import create_evaluator_instance -from .code_evaluator import CodeEvaluator +from .grader import Grader MY_DIR = abspath(dirname(__file__)) @@ -85,19 +85,15 @@ class CodeServer(object): """Calls relevant EvaluateCode class based on language to check the answer code """ - data = self.unpack_json_to_python_obj(json_data) - code_eval_instance = CodeEvaluator(in_dir) - result = code_eval_instance.evaluate(data) #language, test_case_type, + data = json.loads(json_data) + grader = Grader(in_dir) + result = grader.evaluate(data) # Put us back into the server pool queue since we are free now. self.queue.put(self.port) return json.dumps(result) - def unpack_json_to_python_obj(self, json_data): - data = json.loads(json_data) - return data - def run(self): """Run XMLRPC server, serving our methods.""" server = SimpleXMLRPCServer(("0.0.0.0", self.port)) diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 6d4b55d..c318a82 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class CppStdioEvaluator(StdIOEvaluator): +class CppStdIOEvaluator(StdIOEvaluator): """Evaluates C StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 142d7f0..06a56e4 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -3,9 +3,9 @@ import unittest import os import shutil import tempfile -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.bash_code_evaluator import BashCodeEvaluator -from yaksh.bash_stdio_evaluator import BashStdioEvaluator +from yaksh.bash_stdio_evaluator import BashStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent @@ -44,8 +44,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") @@ -63,8 +63,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) @@ -82,8 +82,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -107,13 +107,13 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class BashStdioEvaluationTestCases(unittest.TestCase): +class BashStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " @@ -144,8 +144,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -174,8 +174,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -202,8 +202,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -229,8 +229,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 4dd6a2f..dc6fdc9 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -6,9 +6,9 @@ import tempfile from textwrap import dedent # Local import -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.cpp_code_evaluator import CppCodeEvaluator -from yaksh.cpp_stdio_evaluator import CppStdioEvaluator +from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -44,8 +44,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") @@ -62,8 +62,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -82,8 +82,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -100,8 +100,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -134,13 +134,13 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") -class CppStdioEvaluationTestCases(unittest.TestCase): +class CppStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', @@ -171,8 +171,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -202,8 +202,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -231,8 +231,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -254,8 +254,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -279,8 +279,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -302,8 +302,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -330,8 +330,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -355,8 +355,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -387,8 +387,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -417,8 +417,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -441,8 +441,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -467,8 +467,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -491,8 +491,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -520,8 +520,8 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index f664200..cb783b0 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -14,7 +14,7 @@ class RegistryTestCase(unittest.TestCase): ".PythonAssertionEvaluator" ) stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdioEvaluator" + "PythonStdIOEvaluator" ) code_evaluators['python'] = \ {"standardtestcase": assertion_evaluator_path, @@ -29,7 +29,7 @@ class RegistryTestCase(unittest.TestCase): ".PythonAssertionEvaluator" ) stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdioEvaluator" + "PythonStdIOEvaluator" ) class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator' diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 246a3e5..36eb6a5 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -3,13 +3,14 @@ import unittest import os import shutil import tempfile -from yaksh import code_evaluator -from yaksh.code_evaluator import CodeEvaluator -from yaksh.java_code_evaluator import JavaCodeEvaluator -from yaksh.java_stdio_evaluator import JavaStdioEvaluator -from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent +# Local Import +from yaksh import grader as gd +from yaksh.grader import Grader +from yaksh.java_code_evaluator import JavaCodeEvaluator +from yaksh.java_stdio_evaluator import JavaStdIOEvaluator + class JavaAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -23,14 +24,15 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): } ] self.in_dir = tmp_in_dir_path - code_evaluator.SERVER_TIMEOUT = 9 + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(code_evaluator.SERVER_TIMEOUT) - self.file_paths = None + " your code.").format(gd.SERVER_TIMEOUT) + def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) @@ -46,8 +48,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -64,8 +66,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get('success')) lines_of_error = len(result.get('error').splitlines()) @@ -85,8 +87,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) @@ -103,8 +105,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -145,13 +147,13 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class JavaStdioEvaluationTestCases(unittest.TestCase): +class JavaStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) @@ -162,14 +164,14 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - code_evaluator.SERVER_TIMEOUT = 4 + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(code_evaluator.SERVER_TIMEOUT) - self.file_paths = None + " your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) @@ -193,8 +195,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -225,8 +227,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -251,8 +253,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -275,14 +277,13 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): - user_answer = dedent(""" class Test {public static void main(String[] args){ @@ -300,8 +301,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -329,8 +330,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -360,8 +361,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -401,8 +402,8 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 29336d6..e638049 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -6,9 +6,9 @@ import shutil from textwrap import dedent # Local import -from yaksh.code_evaluator import CodeEvaluator +from yaksh.grader import Grader from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdio_evaluator import PythonStdioEvaluator +from yaksh.python_stdio_evaluator import PythonStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT class PythonAssertionEvaluationTestCases(unittest.TestCase): @@ -44,8 +44,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -65,8 +65,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -98,8 +98,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -125,8 +125,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -157,8 +157,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -191,8 +191,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -222,8 +222,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -254,8 +254,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -286,8 +286,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -321,8 +321,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -352,8 +352,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertIn("Correct answer", result.get('error')) @@ -389,8 +389,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -430,8 +430,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -471,8 +471,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -505,8 +505,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -537,8 +537,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -567,8 +567,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -599,8 +599,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), "Correct answer\n") @@ -628,8 +628,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): } # When - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), timeout_msg) diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index e879cda..0275ee8 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -4,10 +4,9 @@ import os import shutil import tempfile -from yaksh import code_evaluator -from yaksh.code_evaluator import CodeEvaluator +from yaksh import grader as gd +from yaksh.grader import Grader from yaksh.scilab_code_evaluator import ScilabCodeEvaluator -from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -17,14 +16,14 @@ class ScilabEvaluationTestCases(unittest.TestCase): "weight": 0.0 }] self.in_dir = tmp_in_dir_path + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) - code_evaluator.SERVER_TIMEOUT = 9 - self.file_paths = None + " in your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): - code_evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 shutil.rmtree(self.in_dir) def test_correct_answer(self): @@ -40,8 +39,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -59,8 +58,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertTrue('error' in result.get("error")) @@ -79,8 +78,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -88,7 +87,6 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(lines_of_error > 1) def test_infinite_loop(self): - code_evaluator.SERVER_TIMEOUT = 4 user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") kwargs = { @@ -101,8 +99,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data, } - evaluator = CodeEvaluator(self.in_dir) - result = evaluator.evaluate(kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) diff --git a/yaksh/forms.py b/yaksh/forms.py index 1931fad..6fbaf5d 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,6 +1,6 @@ from django import forms from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course,\ - QuestionPaper, StandardTestCase, StdioBasedTestCase + QuestionPaper, StandardTestCase, StdIOBasedTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -37,7 +37,7 @@ question_types = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - ("stdiobasedtestcase", "Stdio Based Testcase"), + ("stdiobasedtestcase", "StdIO Based Testcase"), ("mcqtestcase", "MCQ Testcase"), ) diff --git a/yaksh/grader.py b/yaksh/grader.py new file mode 100644 index 0000000..ef349e0 --- /dev/null +++ b/yaksh/grader.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +from __future__ import unicode_literals +import sys +import pwd +import os +import stat +import contextlib +from os.path import isdir, dirname, abspath, join, isfile, exists +import signal +import traceback +from multiprocessing import Process, Queue +import subprocess +import re + +try: + from SimpleXMLRPCServer import SimpleXMLRPCServer +except ImportError: + # The above import will not work on Python-3.x. + from xmlrpc.server import SimpleXMLRPCServer + +# Local imports +from .settings import SERVER_TIMEOUT +from .language_registry import create_evaluator_instance + + +MY_DIR = abspath(dirname(__file__)) + +registry = None + +# Raised when the code times-out. +# c.f. http://pguides.net/python/timeout-a-function +class TimeoutException(Exception): + pass + +@contextlib.contextmanager +def change_dir(path): + cur_dir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(cur_dir) + + +def timeout_handler(signum, frame): + """A handler for the ALARM signal.""" + raise TimeoutException('Code took too long to run.') + + +def create_signal_handler(): + """Add a new signal handler for the execution of this code.""" + prev_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(SERVER_TIMEOUT) + return prev_handler + + +def set_original_signal_handler(old_handler=None): + """Set back any original signal handler.""" + if old_handler is not None: + signal.signal(signal.SIGALRM, old_handler) + return + else: + raise Exception("Signal Handler: object cannot be NoneType") + + +def delete_signal_handler(): + signal.alarm(0) + return + + +class Grader(object): + """Tests the code obtained from Code Server""" + def __init__(self, in_dir=None): + msg = 'Code took more than %s seconds to run. You probably '\ + 'have an infinite loop in your code.' % SERVER_TIMEOUT + self.timeout_msg = msg + self.in_dir = in_dir if in_dir else MY_DIR + + + def evaluate(self, kwargs): #language, test_case_type, + """Evaluates given code with the test cases based on + given arguments in test_case_data. + + The ref_code_path is a path to the reference code. + The reference code will call the function submitted by the student. + The reference code will check for the expected output. + + If the path's start with a "/" then we assume they are absolute paths. + If not, we assume they are relative paths w.r.t. the location of this + code_server script. + + If the optional `in_dir` keyword argument is supplied it changes the + directory to that directory (it does not change it back to the original + when done). + + Returns + ------- + + A tuple: (success, error message, weight). + """ + + self.setup() + test_case_instances = self.get_evaluator_objects(kwargs) + with change_dir(self.in_dir): + success, error, mark = self.safe_evaluate(test_case_instances) + self.teardown() + + result = {'success': success, 'error': error, 'weight': mark} + return result + + # Private Protocol ########## + def setup(self): + if self.in_dir: + if not os.path.exists(self.in_dir): + os.makedirs(self.in_dir) + + def get_evaluator_objects(self, kwargs): + metadata = kwargs.get('metadata') + test_case_data = kwargs.get('test_case_data') + test_case_instances = [] + + for test_case in test_case_data: + test_case_instance = create_evaluator_instance(metadata, test_case) + test_case_instances.append(test_case_instance) + + return test_case_instances + + + def safe_evaluate(self, test_case_instances): + """ + Handles code evaluation along with compilation, signal handling + and Exception handling + """ + # Add a new signal handler for the execution of this code. + prev_handler = create_signal_handler() + success = False + test_case_success_status = [False] * len(test_case_instances) + error = "" + weight = 0.0 + + # Do whatever testing needed. + try: + # Run evaluator selection registry here + for idx, test_case_instance in enumerate(test_case_instances): + test_case_success = False + test_case_instance.compile_code() + test_case_success, err, mark_fraction = test_case_instance.check_code() + if test_case_success: + weight += mark_fraction + + error += err + "\n" + test_case_success_status[idx] = test_case_success + + success = all(test_case_success_status) + + for test_case_instance in test_case_instances: + test_case_instance.teardown() + + except TimeoutException: + error = self.timeout_msg + except OSError: + msg = traceback.format_exc(limit=0) + error = "Error: {0}".format(msg) + except Exception as e: + exc_type, exc_value, exc_tb = sys.exc_info() + tb_list = traceback.format_exception(exc_type, exc_value, exc_tb) + if len(tb_list) > 2: + del tb_list[1:3] + error = "Error: {0}".format("".join(tb_list)) + finally: + # Set back any original signal handler. + set_original_signal_handler(prev_handler) + + return success, error, weight + + def teardown(self): + # Cancel the signal + delete_signal_handler() diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index a5f5540..a854847 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -9,7 +9,7 @@ from .stdio_evaluator import StdIOEvaluator from .file_utils import copy_files, delete_files -class JavaStdioEvaluator(StdIOEvaluator): +class JavaStdIOEvaluator(StdIOEvaluator): """Evaluates Java StdIO based code""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 8d3aad2..994e9ed 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -14,15 +14,10 @@ def get_registry(): registry = _LanguageRegistry() return registry -# def unpack_json_to_python_obj(json_data): -# data = json.loads(json_data) -# return data - -def create_evaluator_instance(metadata, test_case): #create_evaluator_instance +def create_evaluator_instance(metadata, test_case): """Create instance of relevant EvaluateCode class based on language""" registry = get_registry() cls = registry.get_class(metadata.get('language'), test_case.get('test_case_type')) - # instance = cls(in_dir) instance = cls(metadata, test_case) return instance diff --git a/yaksh/models.py b/yaksh/models.py index 9ba85ba..6e1744c 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1160,7 +1160,7 @@ class StandardTestCase(TestCase): ) -class StdioBasedTestCase(TestCase): +class StdIOBasedTestCase(TestCase): expected_input = models.CharField(max_length=100, blank=True) expected_output = models.CharField(max_length=100) weight = models.IntegerField(default=1.0) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 0b49cb0..4d44838 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -9,7 +9,7 @@ import importlib # Local imports from .file_utils import copy_files, delete_files from .base_evaluator import BaseEvaluator -from .code_evaluator import TimeoutException +from .grader import TimeoutException class PythonAssertionEvaluator(BaseEvaluator): diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 07bd59c..da0c954 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -28,7 +28,7 @@ def redirect_stdout(): sys.stdout = old_target # restore to the previous value -class PythonStdioEvaluator(BaseEvaluator): +class PythonStdIOEvaluator(BaseEvaluator): """Tests the Python code obtained from Code Server""" def __init__(self, metadata, test_case_data): self.files = [] diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 97e40a8..cc3c401 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -26,12 +26,6 @@ class ScilabCodeEvaluator(BaseEvaluator): self.test_case = test_case_data.get('test_case') self.weight = test_case_data.get('weight') - # def setup(self): - # super(ScilabCodeEvaluator, self).setup() - # self.files = [] - # self.submit_code_path = \ - # self.create_submit_code_file('function.sci') - def teardown(self): # Delete the created file. os.remove(self.submit_code_path) @@ -61,7 +55,7 @@ class ScilabCodeEvaluator(BaseEvaluator): cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format( clean_ref_path ) - cmd += ' | timeout 8 scilab-cli -nb' + cmd += ' | scilab-cli -nb' ret = self._run_command(cmd, shell=True, stdout=subprocess.PIPE, diff --git a/yaksh/settings.py b/yaksh/settings.py index e18d310..0e432cf 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -19,40 +19,20 @@ SERVER_TIMEOUT = 4 # host.org/foo/exam set URL_ROOT='/foo' URL_ROOT = '' -# code_evaluators = { -# "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", -# "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" -# }, -# "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", -# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" -# }, -# "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", -# "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" -# }, -# "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", -# "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, - -# "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", -# "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" -# }, - -# "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, -# } - code_evaluators = { "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdioEvaluator" + "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator" }, "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" }, "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdioEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" }, "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", - "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdioEvaluator"}, + "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator"}, "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", - "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdioEvaluator" + "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator" }, "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/test_models.py b/yaksh/test_models.py index 0db1b82..317c832 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -1,7 +1,7 @@ import unittest from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdioBasedTestCase, FileUpload, McqTestCase + StdIOBasedTestCase, FileUpload, McqTestCase import json from datetime import datetime, timedelta from django.utils import timezone @@ -879,7 +879,7 @@ class TestCaseTestCases(unittest.TestCase): test_case='assert myfunc(12, 13) == 15', type='standardtestcase' ) - self.stdout_based_testcase = StdioBasedTestCase( + self.stdout_based_testcase = StdIOBasedTestCase( question=self.question2, expected_output='Hello World', type='standardtestcase' diff --git a/yaksh/test_views.py b/yaksh/test_views.py index 30ebcaa..2419591 100644 --- a/yaksh/test_views.py +++ b/yaksh/test_views.py @@ -9,7 +9,7 @@ from django.utils import timezone from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdioBasedTestCase, has_profile + StdIOBasedTestCase, has_profile class TestProfile(TestCase): diff --git a/yaksh/views.py b/yaksh/views.py index 0d77426..89274df 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -25,7 +25,7 @@ import six # Local imports. from yaksh.models import get_model_class, Quiz, Question, QuestionPaper, QuestionSet, Course from yaksh.models import Profile, Answer, AnswerPaper, User, TestCase, FileUpload,\ - has_profile, StandardTestCase, McqTestCase, StdioBasedTestCase, HookTestCase + has_profile, StandardTestCase, McqTestCase, StdIOBasedTestCase, HookTestCase from yaksh.forms import UserRegisterForm, UserLoginForm, QuizForm,\ QuestionForm, RandomQuestionForm,\ QuestionFilterForm, CourseForm, ProfileForm, UploadFileForm,\ -- cgit