summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/code_evaluator.py63
-rwxr-xr-xyaksh/code_server.py26
-rw-r--r--yaksh/forms.py2
-rw-r--r--yaksh/language_registry.py27
-rw-r--r--yaksh/models.py44
-rw-r--r--yaksh/python_code_evaluator.py74
-rw-r--r--yaksh/python_stdout_evaluator.py47
-rw-r--r--yaksh/settings.py5
-rw-r--r--yaksh/templates/yaksh/add_question.html1
-rw-r--r--yaksh/tester/python/verifier.py121
-rw-r--r--yaksh/views.py2
-rw-r--r--yaksh/xmlrpc_clients.py4
12 files changed, 317 insertions, 99 deletions
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index f877952..7e2a729 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -8,7 +8,7 @@ import signal
from multiprocessing import Process, Queue
import subprocess
import re
-import json
+# import json
# Local imports.
from settings import SERVER_TIMEOUT
@@ -50,33 +50,39 @@ def delete_signal_handler():
class CodeEvaluator(object):
"""Tests the code obtained from Code Server"""
- def __init__(self, test_case_data, test, language, user_answer,
- ref_code_path=None, in_dir=None):
+ # def __init__(self, test_case_data, test, language, user_answer,
+ # ref_code_path=None, in_dir=None):
+ def __init__(self, in_dir, **kwargs):
+
+
msg = 'Code took more than %s seconds to run. You probably '\
'have an infinite loop in your code.' % SERVER_TIMEOUT
self.timeout_msg = msg
- self.test_case_data = test_case_data
- self.language = language.lower()
- self.user_answer = user_answer
- self.ref_code_path = ref_code_path
- self.test = test
- self.in_dir = in_dir
- self.test_case_args = None
+ # self.test_case_data = test_case_data
+ # self.language = language.lower() #@@@remove
+ # self.user_answer = user_answer #@@@specific to check-code
+ # self.ref_code_path = ref_code_path #@@@specific to check-code
+ # self.test = test #@@@specific to check-code
+ self.in_dir = in_dir #@@@Common for all, no change
+ self.test_case_args = None #@@@no change
# Public Protocol ##########
- @classmethod
- def from_json(cls, language, json_data, in_dir):
- json_data = json.loads(json_data)
- test_case_data = json_data.get("test_case_data")
- user_answer = json_data.get("user_answer")
- ref_code_path = json_data.get("ref_code_path")
- test = json_data.get("test")
-
- instance = cls(test_case_data, test, language, user_answer, ref_code_path,
- in_dir)
- return instance
-
- def evaluate(self):
+ # @classmethod
+ # def from_json(cls, language, json_data, in_dir):
+ # json_data = json.loads(json_data)
+ # # test_case_data = json_data.get("test_case_data")
+ # user_answer = json_data.get("user_answer")
+ # ref_code_path = json_data.get("ref_code_path")
+ # test = json_data.get("test")
+
+ # # instance = cls(test_case_data, test, language, user_answer, ref_code_path,
+ # # in_dir)
+ # instance = cls(test, language, user_answer, ref_code_path,
+ # in_dir)
+ # return instance
+
+ # def evaluate(self):
+ def evaluate(self, **kwargs):
"""Evaluates given code with the test cases based on
given arguments in test_case_data.
@@ -99,7 +105,8 @@ class CodeEvaluator(object):
"""
self.setup()
- success, err = self.safe_evaluate(self.test_case_args)
+ # success, err = self.safe_evaluate(self.test_case_args)
+ success, err = self.safe_evaluate(**kwargs)
self.teardown()
result = {'success': success, 'error': err}
@@ -109,15 +116,17 @@ class CodeEvaluator(object):
def setup(self):
self._change_dir(self.in_dir)
- def safe_evaluate(self, args):
+ # def safe_evaluate(self, args):
+ def safe_evaluate(self, **kwargs):
# Add a new signal handler for the execution of this code.
prev_handler = create_signal_handler()
success = False
- args = args or []
+ # args = args or []
# Do whatever testing needed.
try:
- success, err = self.check_code(*args)
+ # success, err = self.check_code(*args)
+ success, err = self.check_code(**kwargs)
except TimeoutException:
err = self.timeout_msg
diff --git a/yaksh/code_server.py b/yaksh/code_server.py
index 2762f12..7951ac8 100755
--- a/yaksh/code_server.py
+++ b/yaksh/code_server.py
@@ -31,7 +31,7 @@ import re
import json
# Local imports.
from settings import SERVER_PORTS, SERVER_POOL_PORT
-from language_registry import get_registry, registry
+from language_registry import get_registry, create_evaluator_instance, unpack_json
MY_DIR = abspath(dirname(__file__))
@@ -58,13 +58,14 @@ class CodeServer(object):
self.queue = queue
# Public Protocol ##########
- def check_code(self, language, json_data, in_dir=None):
+ def check_code(self, language, test_case_type, json_data, in_dir=None):
"""Calls relevant EvaluateCode class based on language to check the
answer code
"""
- code_evaluator = self._create_evaluator_instance(language, json_data,
+ code_evaluator = create_evaluator_instance(language, test_case_type, json_data,
in_dir)
- result = code_evaluator.evaluate()
+ data = unpack_json(json_data) #@@@ def should be here
+ result = code_evaluator.evaluate(**data)
# Put us back into the server pool queue since we are free now.
self.queue.put(self.port)
@@ -79,15 +80,14 @@ class CodeServer(object):
self.queue.put(self.port)
server.serve_forever()
- # Private Protocol ##########
- def _create_evaluator_instance(self, language, json_data, in_dir):
- """Create instance of relevant EvaluateCode class based on language"""
- # set_registry()
- registry1 = get_registry()
- print registry
- cls = registry1.get_class(language)
- instance = cls.from_json(language, json_data, in_dir)
- return instance
+ # # Private Protocol ##########
+ # def _create_evaluator_instance(self, language, json_data, in_dir):
+ # """Create instance of relevant EvaluateCode class based on language"""
+ # # set_registry()
+ # registry = get_registry()
+ # cls = registry.get_class(language)
+ # instance = cls.from_json(language, json_data, in_dir)
+ # return instance
###############################################################################
diff --git a/yaksh/forms.py b/yaksh/forms.py
index 5c8dafa..5959dc4 100644
--- a/yaksh/forms.py
+++ b/yaksh/forms.py
@@ -31,7 +31,7 @@ question_types = (
test_case_types = (
("assert_based", "Assertion Based Testcase"),
# ("argument_based", "Multiple Correct Choices"),
- # ("stdout_based", "Code"),
+ ("stdout_based", "Stdout Based Testcase"),
)
UNAME_CHARS = letters + "._" + digits
diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py
index ee311ec..512e2f5 100644
--- a/yaksh/language_registry.py
+++ b/yaksh/language_registry.py
@@ -1,21 +1,31 @@
from settings import code_evaluators
import importlib
+import json
registry = None
# def set_registry():
# global registry
# registry = _LanguageRegistry()
-
-def _set_registry():
+
+def get_registry(): #@@@get_evaluator_registry
global registry
if registry is None:
registry = _LanguageRegistry()
return registry
-
-def get_registry():
- registry = _set_registry()
- return registry
+
+def unpack_json(json_data):
+ data = json.loads(json_data)
+ return data
+
+def create_evaluator_instance(language, test_case_type, json_data, in_dir):
+ """Create instance of relevant EvaluateCode class based on language"""
+ # set_registry()
+ registry = get_registry()
+ cls = registry.get_class(language, test_case_type) #@@@get_evaluator_for_language
+ instance = cls(in_dir)
+ # instance = cls.from_json(language, json_data, in_dir)
+ return instance
class _LanguageRegistry(object):
def __init__(self):
@@ -24,12 +34,13 @@ class _LanguageRegistry(object):
self._register[language] = None
# Public Protocol ##########
- def get_class(self, language):
+ def get_class(self, language, test_case_type):
""" Get the code evaluator class for the given language """
if not self._register.get(language):
self._register[language] = code_evaluators.get(language)
- cls = self._register[language]
+ test_case_register = self._register[language]
+ cls = test_case_register.get(test_case_type)
module_name, class_name = cls.rsplit(".", 1)
# load the module, will raise ImportError if module cannot be loaded
get_module = importlib.import_module(module_name)
diff --git a/yaksh/models.py b/yaksh/models.py
index c4f3561..6fa96bf 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -33,7 +33,7 @@ enrollment_methods = (
test_case_types = (
("assert_based", "Assertion Based Testcase"),
# ("argument_based", "Multiple Correct Choices"),
- # ("stdout_based", "Code"),
+ ("stdout_based", "Stdout Based Testcase"),
)
attempts = [(i, i) for i in range(1, 6)]
@@ -170,6 +170,7 @@ class Question(models.Model):
# The type of evaluator
test_case_type = models.CharField(max_length=24, choices=test_case_types)
+
# Is this question active or not. If it is inactive it will not be used
# when creating a QuestionPaper.
active = models.BooleanField(default=True)
@@ -183,38 +184,39 @@ class Question(models.Model):
# user for particular question
user = models.ForeignKey(User, related_name="user")
- def consolidate_answer_data(self, test_cases, user_answer):
+ def consolidate_answer_data(self, user_answer):
test_case_data_dict = []
question_info_dict = {}
- for test_case in test_cases:
- kw_args_dict = {}
- pos_args_list = []
+ # for test_case in test_cases:
+ # kw_args_dict = {}
+ # pos_args_list = []
- test_case_data = {}
- test_case_data['test_id'] = test_case.id
- test_case_data['func_name'] = test_case.func_name
- test_case_data['expected_answer'] = test_case.expected_answer
+ # test_case_data = {}
+ # test_case_data['test_id'] = test_case.id
+ # test_case_data['func_name'] = test_case.func_name
+ # test_case_data['expected_answer'] = test_case.expected_answer
- if test_case.kw_args:
- for args in test_case.kw_args.split(","):
- arg_name, arg_value = args.split("=")
- kw_args_dict[arg_name.strip()] = arg_value.strip()
+ # if test_case.kw_args:
+ # for args in test_case.kw_args.split(","):
+ # arg_name, arg_value = args.split("=")
+ # kw_args_dict[arg_name.strip()] = arg_value.strip()
- if test_case.pos_args:
- for args in test_case.pos_args.split(","):
- pos_args_list.append(args.strip())
+ # if test_case.pos_args:
+ # for args in test_case.pos_args.split(","):
+ # pos_args_list.append(args.strip())
- test_case_data['kw_args'] = kw_args_dict
- test_case_data['pos_args'] = pos_args_list
- test_case_data_dict.append(test_case_data)
+ # test_case_data['kw_args'] = kw_args_dict
+ # test_case_data['pos_args'] = pos_args_list
+ # test_case_data_dict.append(test_case_data)
# question_info_dict['language'] = self.language
- question_info_dict['id'] = self.id
+ # question_info_dict['id'] = self.id
question_info_dict['user_answer'] = user_answer
- question_info_dict['test_parameter'] = test_case_data_dict
+ # question_info_dict['test_parameter'] = test_case_data_dict
question_info_dict['ref_code_path'] = self.ref_code_path
question_info_dict['test'] = self.test
+ # question_info_dict['test_case_type'] = self.test_case_type
return json.dumps(question_info_dict)
diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py
index 3835b44..5722b2d 100644
--- a/yaksh/python_code_evaluator.py
+++ b/yaksh/python_code_evaluator.py
@@ -12,13 +12,13 @@ from code_evaluator import CodeEvaluator, TimeoutException
class PythonCodeEvaluator(CodeEvaluator):
"""Tests the Python code obtained from Code Server"""
- def check_code(self):
+ def check_code(self, test, user_answer, ref_code_path):
success = False
try:
tb = None
- test_code = self._create_test_case()
- submitted = compile(self.user_answer, '<string>', mode='exec')
+ test_code = test
+ submitted = compile(user_answer, '<string>', mode='exec')
g = {}
exec submitted in g
_tests = compile(test_code, '<string>', mode='exec')
@@ -40,26 +40,50 @@ class PythonCodeEvaluator(CodeEvaluator):
del tb
return success, err
- def _create_test_case(self):
- """
- Create assert based test cases in python
- """
- test_code = ""
- if self.test:
- return self.test
- elif self.test_case_data:
- for test_case in self.test_case_data:
- pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \
- if test_case.get('pos_args') else ""
- kw_args = ", ".join(str(k+"="+a) for k, a
- in test_case.get('kw_args').iteritems()) \
- if test_case.get('kw_args') else ""
- args = pos_args + ", " + kw_args if pos_args and kw_args \
- else pos_args or kw_args
- function_name = test_case.get('func_name')
- expected_answer = test_case.get('expected_answer')
+ # def check_code(self):
+ # success = False
- tcode = "assert {0}({1}) == {2}".format(function_name, args,
- expected_answer)
- test_code += tcode + "\n"
- return test_code
+ # try:
+ # tb = None
+ # test_code = self._create_test_case()
+ # submitted = compile(self.user_answer, '<string>', mode='exec')
+ # g = {}
+ # exec submitted in g
+ # _tests = compile(test_code, '<string>', mode='exec')
+ # exec _tests in g
+ # except AssertionError:
+ # type, value, tb = sys.exc_info()
+ # info = traceback.extract_tb(tb)
+ # fname, lineno, func, text = info[-1]
+ # text = str(test_code).splitlines()[lineno-1]
+ # err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
+ # else:
+ # success = True
+ # err = 'Correct answer'
+
+ # del tb
+ # return success, err
+
+ # def _create_test_case(self):
+ # """
+ # Create assert based test cases in python
+ # """
+ # test_code = ""
+ # if self.test:
+ # return self.test
+ # elif self.test_case_data:
+ # for test_case in self.test_case_data:
+ # pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \
+ # if test_case.get('pos_args') else ""
+ # kw_args = ", ".join(str(k+"="+a) for k, a
+ # in test_case.get('kw_args').iteritems()) \
+ # if test_case.get('kw_args') else ""
+ # args = pos_args + ", " + kw_args if pos_args and kw_args \
+ # else pos_args or kw_args
+ # function_name = test_case.get('func_name')
+ # expected_answer = test_case.get('expected_answer')
+
+ # tcode = "assert {0}({1}) == {2}".format(function_name, args,
+ # expected_answer)
+ # test_code += tcode + "\n"
+ # return test_code
diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py
new file mode 100644
index 0000000..89d3424
--- /dev/null
+++ b/yaksh/python_stdout_evaluator.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+import sys
+import traceback
+import os
+from os.path import join
+import importlib
+from contextlib import contextmanager
+
+# local imports
+from code_evaluator import CodeEvaluator
+
+
+@contextmanager
+def redirect_stdout():
+ from StringIO import StringIO
+ new_target = StringIO()
+
+ old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+ try:
+ yield new_target # run some code with the replaced stdout
+ finally:
+ sys.stdout = old_target # restore to the previous value
+
+
+class PythonStdoutEvaluator(CodeEvaluator):
+ """Tests the Python code obtained from Code Server"""
+
+ def check_code(self, test, user_answer, ref_code_path):
+ success = False
+
+ try:
+ tb = None
+ test_code = test
+ submitted = compile(user_answer, '<string>', mode='exec')
+ with redirect_stdout() as output_buffer:
+ g = {}
+ exec submitted in g
+ raw_output_value = output_buffer.getvalue()
+ output_value = raw_output_value.encode('string_escape').strip()
+ if output_value == str(test_code):
+ success = True
+ err = 'Correct answer'
+ else:
+ raise ValueError("Incorrect Answer")
+
+ del tb
+ return success, err \ No newline at end of file
diff --git a/yaksh/settings.py b/yaksh/settings.py
index 63bd875..f8b240d 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -20,7 +20,10 @@ SERVER_TIMEOUT = 2
URL_ROOT = ''
code_evaluators = {
- "python": "python_code_evaluator.PythonCodeEvaluator",
+ "python": {"assert_based": "python_code_evaluator.PythonCodeEvaluator",
+ "argument_based": "python_argument_based_evaluator.PythonCodeEvaluator",
+ "stdout_based": "python_stdout_evaluator.PythonStdoutEvaluator"
+ },
"c": "cpp_code_evaluator.CppCodeEvaluator",
"cpp": "cpp_code_evaluator.CppCodeEvaluator",
"java": "java_code_evaluator.JavaCodeEvaluator",
diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html
index 1b1d28d..88d8f03 100644
--- a/yaksh/templates/yaksh/add_question.html
+++ b/yaksh/templates/yaksh/add_question.html
@@ -30,6 +30,7 @@
<tr><td id='label_option'>Options: <td>{{ form.options }} {{form.options.errors}}
<tr><td id='label_solution'>Test: <td>{{ form.test }} {{form.test.errors}}
<tr><td id='label_ref_code_path'>Reference Code Path: <td>{{ form.ref_code_path }} {{form.ref_code_path.errors}}
+ <tr><td> test_case_type: <td> {{ form.test_case_type }}{{ form.test_case_type.errors }}
<form method="post" action="">
{% if formset%}
diff --git a/yaksh/tester/python/verifier.py b/yaksh/tester/python/verifier.py
new file mode 100644
index 0000000..102dcb9
--- /dev/null
+++ b/yaksh/tester/python/verifier.py
@@ -0,0 +1,121 @@
+import sys
+from .utils import import_by_path
+from contextlib import contextmanager
+
+
+@contextmanager
+def redirect_stdout():
+ from StringIO import StringIO
+ new_target = StringIO()
+
+ old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+ try:
+ yield new_target # run some code with the replaced stdout
+ finally:
+ sys.stdout = old_target # restore to the previous value
+
+# def redirect_stdout():
+# # import sys
+# from StringIO import StringIO
+# oldout,olderr = sys.stdout, sys.stderr
+# try:
+# out = StringIO()
+# err = StringIO()
+# # sys.stdout,sys.stderr = out, err
+# yield out, err
+# finally:
+# sys.stdout,sys.stderr = oldout, olderr
+# out = out.getvalue()
+# err = err.getvalue()
+
+TESTER_BACKEND = {
+ "python": "PythonPrintTesterBackend" #@@@rename to test-case-creator, this file should be backend.py
+}
+
+class TesterException(Exception):
+ """ Parental class for all tester exceptions """
+ pass
+
+class UnknownBackendException(TesterException):
+ """ Exception thrown if tester backend is not recognized. """
+ pass
+
+
+def detect_backend(language):
+ """
+ Detect the right backend for a test case.
+ """
+ backend_name = TESTER_BACKEND.get(language)
+ # backend = import_by_path(backend_name)
+ backend = PythonTesterBackend() #@@@
+ return backend
+
+class PythonPrintTesterBackend(object):
+ def test_code(self, submitted, reference_output):
+ """
+ create a test command
+ """
+ with redirect_stdout() as output_buffer:
+ g = {}
+ exec submitted in g
+
+ # return_buffer = out.encode('string_escape')
+ raw_output_value = output_buffer.getvalue()
+ output_value = raw_output_value.encode('string_escape').strip()
+ if output_value == str(reference_output):
+ return True
+ else:
+ raise ValueError("Incorrect Answer", output_value, reference_output)
+
+
+class PythonTesterBackend(object):
+ # def __init__(self, test_case):
+ # self._test_case = test_case
+ def create(self): #@@@ test()
+ """
+ create a test command
+ """
+ test_code = "assert {0}({1}) == {2}".format(self.test_case_parameters['function_name'], self.test_case_parameters['args'],
+ self.test_case_parameters['expected_answer'])
+ return test_code
+
+ def pack(self, test_case):
+ kw_args_dict = {}
+ pos_args_list = []
+ test_case_data = {}
+ test_case_data['test_id'] = test_case.id
+ test_case_data['func_name'] = test_case.func_name
+ test_case_data['expected_answer'] = test_case.expected_answer
+
+ if test_case.kw_args:
+ for args in test_case.kw_args.split(","):
+ arg_name, arg_value = args.split("=")
+ kw_args_dict[arg_name.strip()] = arg_value.strip()
+
+ if test_case.pos_args:
+ for args in test_case.pos_args.split(","):
+ pos_args_list.append(args.strip())
+
+ test_case_data['kw_args'] = kw_args_dict
+ test_case_data['pos_args'] = pos_args_list
+
+ return test_case_data
+
+ def unpack(self, test_case_data):
+ pos_args = ", ".join(str(i) for i in test_case_data.get('pos_args')) \
+ if test_case_data.get('pos_args') else ""
+ kw_args = ", ".join(str(k+"="+a) for k, a
+ in test_case_data.get('kw_args').iteritems()) \
+ if test_case_data.get('kw_args') else ""
+ args = pos_args + ", " + kw_args if pos_args and kw_args \
+ else pos_args or kw_args
+ function_name = test_case_data.get('func_name')
+ expected_answer = test_case_data.get('expected_answer')
+
+ self.test_case_parameters = {
+ 'args': args,
+ 'function_name': function_name,
+ 'expected_answer': expected_answer
+ }
+
+ return self.test_case_parameters \ No newline at end of file
diff --git a/yaksh/views.py b/yaksh/views.py
index f540351..520f396 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -513,7 +513,7 @@ def validate_answer(user, user_answer, question, json_data=None):
message = 'Correct answer'
elif question.type == 'code':
user_dir = get_user_dir(user)
- json_result = code_server.run_code(question.language, json_data, user_dir)
+ json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir)
result = json.loads(json_result)
if result.get('success'):
correct = True
diff --git a/yaksh/xmlrpc_clients.py b/yaksh/xmlrpc_clients.py
index 3a3c0c6..7124550 100644
--- a/yaksh/xmlrpc_clients.py
+++ b/yaksh/xmlrpc_clients.py
@@ -23,7 +23,7 @@ class CodeServerProxy(object):
pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT)
self.pool_server = ServerProxy(pool_url)
- def run_code(self, language, json_data, user_dir):
+ def run_code(self, language, test_case_type, json_data, user_dir):
"""Tests given code (`answer`) with the `test_code` supplied. If the
optional `in_dir` keyword argument is supplied it changes the directory
to that directory (it does not change it back to the original when
@@ -51,7 +51,7 @@ class CodeServerProxy(object):
try:
server = self._get_server()
- result = server.check_code(language, json_data, user_dir)
+ result = server.check_code(language, test_case_type, json_data, user_dir)
except ConnectionError:
result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'})
return result
generated by cgit (git 2.34.1) at 2024-12-21 07:22:33 +0000