Add a python standard out evaluator

12 files changed, 317 insertions, 99 deletions

diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index f877952..7e2a729 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -8,7 +8,7 @@ import signal
 from multiprocessing import Process, Queue
 import subprocess
 import re
-import json
+# import json
 # Local imports.
 from settings import SERVER_TIMEOUT
 
@@ -50,33 +50,39 @@ def delete_signal_handler():
 
 class CodeEvaluator(object):
     """Tests the code obtained from Code Server"""
-    def __init__(self, test_case_data, test, language, user_answer, 
-                     ref_code_path=None, in_dir=None):
+    # def __init__(self, test_case_data, test, language, user_answer, 
+    #                  ref_code_path=None, in_dir=None):
+    def __init__(self, in_dir, **kwargs):
+
+
         msg = 'Code took more than %s seconds to run. You probably '\
               'have an infinite loop in your code.' % SERVER_TIMEOUT
         self.timeout_msg = msg
-        self.test_case_data = test_case_data
-        self.language = language.lower()
-        self.user_answer = user_answer
-        self.ref_code_path = ref_code_path
-        self.test = test
-        self.in_dir = in_dir
-        self.test_case_args = None
+        # self.test_case_data = test_case_data
+        # self.language = language.lower()    #@@@remove
+        # self.user_answer = user_answer      #@@@specific to check-code
+        # self.ref_code_path = ref_code_path  #@@@specific to check-code
+        # self.test = test                    #@@@specific to check-code
+        self.in_dir = in_dir                #@@@Common for all, no change
+        self.test_case_args = None          #@@@no change
 
     # Public Protocol ##########
-    @classmethod
-    def from_json(cls, language, json_data, in_dir):
-        json_data = json.loads(json_data)
-        test_case_data = json_data.get("test_case_data")
-        user_answer = json_data.get("user_answer")
-        ref_code_path = json_data.get("ref_code_path")
-        test = json_data.get("test") 
-
-        instance = cls(test_case_data, test, language, user_answer, ref_code_path,
-                         in_dir)
-        return instance
-
-    def evaluate(self):
+    # @classmethod
+    # def from_json(cls, language, json_data, in_dir):
+    #     json_data = json.loads(json_data)
+    #     # test_case_data = json_data.get("test_case_data")
+    #     user_answer = json_data.get("user_answer")
+    #     ref_code_path = json_data.get("ref_code_path")
+    #     test = json_data.get("test") 
+
+    #     # instance = cls(test_case_data, test, language, user_answer, ref_code_path,
+    #     #                  in_dir)
+    #     instance = cls(test, language, user_answer, ref_code_path,
+    #                      in_dir)
+    #     return instance
+
+    # def evaluate(self):
+    def evaluate(self, **kwargs):
         """Evaluates given code with the test cases based on
         given arguments in test_case_data.
 
@@ -99,7 +105,8 @@ class CodeEvaluator(object):
         """
 
         self.setup()
-        success, err = self.safe_evaluate(self.test_case_args)
+        # success, err = self.safe_evaluate(self.test_case_args)
+        success, err = self.safe_evaluate(**kwargs)
         self.teardown()
 
         result = {'success': success, 'error': err}
@@ -109,15 +116,17 @@ class CodeEvaluator(object):
     def setup(self):
         self._change_dir(self.in_dir)
 
-    def safe_evaluate(self, args):
+    # def safe_evaluate(self, args):
+    def safe_evaluate(self, **kwargs):
         # Add a new signal handler for the execution of this code.
         prev_handler = create_signal_handler()
         success = False
-        args = args or []
+        # args = args or []
 
         # Do whatever testing needed.
         try:
-            success, err = self.check_code(*args)
+            # success, err = self.check_code(*args)
+            success, err = self.check_code(**kwargs)
 
         except TimeoutException:
             err = self.timeout_msg
diff --git a/yaksh/code_server.py b/yaksh/code_server.py
index 2762f12..7951ac8 100755
--- a/yaksh/code_server.py
+++ b/yaksh/code_server.py
@@ -31,7 +31,7 @@ import re
 import json
 # Local imports.
 from settings import SERVER_PORTS, SERVER_POOL_PORT
-from language_registry import get_registry, registry
+from language_registry import get_registry, create_evaluator_instance, unpack_json
 
 
 MY_DIR = abspath(dirname(__file__))
@@ -58,13 +58,14 @@ class CodeServer(object):
         self.queue = queue
 
     # Public Protocol ##########
-    def check_code(self, language, json_data, in_dir=None):
+    def check_code(self, language, test_case_type, json_data, in_dir=None):
         """Calls relevant EvaluateCode class based on language to check the
          answer code
         """
-        code_evaluator = self._create_evaluator_instance(language, json_data,
+        code_evaluator = create_evaluator_instance(language, test_case_type, json_data,
                                                              in_dir)
-        result = code_evaluator.evaluate()
+        data = unpack_json(json_data) #@@@ def should be here
+        result = code_evaluator.evaluate(**data)
 
         # Put us back into the server pool queue since we are free now.
         self.queue.put(self.port)
@@ -79,15 +80,14 @@ class CodeServer(object):
         self.queue.put(self.port)
         server.serve_forever()
 
-    # Private Protocol ##########
-    def _create_evaluator_instance(self, language, json_data, in_dir):
-        """Create instance of relevant EvaluateCode class based on language"""
-        # set_registry()
-        registry1 = get_registry()
-        print registry
-        cls = registry1.get_class(language)
-        instance = cls.from_json(language, json_data, in_dir)
-        return instance
+    # # Private Protocol ##########
+    # def _create_evaluator_instance(self, language, json_data, in_dir):
+    #     """Create instance of relevant EvaluateCode class based on language"""
+    #     # set_registry()
+    #     registry = get_registry()
+    #     cls = registry.get_class(language)
+    #     instance = cls.from_json(language, json_data, in_dir)
+    #     return instance
 
 
 ###############################################################################
diff --git a/yaksh/forms.py b/yaksh/forms.py
index 5c8dafa..5959dc4 100644
--- a/yaksh/forms.py
+++ b/yaksh/forms.py
@@ -31,7 +31,7 @@ question_types = (
 test_case_types = (
         ("assert_based", "Assertion Based Testcase"),
         # ("argument_based", "Multiple Correct Choices"),
-        # ("stdout_based", "Code"),
+        ("stdout_based", "Stdout Based Testcase"),
     )
 
 UNAME_CHARS = letters + "._" + digits
diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py
index ee311ec..512e2f5 100644
--- a/yaksh/language_registry.py
+++ b/yaksh/language_registry.py
@@ -1,21 +1,31 @@
 from settings import code_evaluators
 import importlib
+import json
 
 registry = None
 
 # def set_registry():
 #     global registry
 #     registry = _LanguageRegistry()
-
-def _set_registry():
+    
+def get_registry(): #@@@get_evaluator_registry
     global registry
     if registry is None:
         registry = _LanguageRegistry()
     return registry
-    
-def get_registry():
-    registry = _set_registry()
-    return registry
+
+def unpack_json(json_data):
+    data = json.loads(json_data)
+    return data
+
+def create_evaluator_instance(language, test_case_type, json_data, in_dir):
+    """Create instance of relevant EvaluateCode class based on language"""
+    # set_registry()
+    registry = get_registry()
+    cls = registry.get_class(language, test_case_type) #@@@get_evaluator_for_language
+    instance = cls(in_dir)
+    # instance = cls.from_json(language, json_data, in_dir)
+    return instance
 
 class _LanguageRegistry(object):
     def __init__(self):
@@ -24,12 +34,13 @@ class _LanguageRegistry(object):
             self._register[language] = None
 
     # Public Protocol ##########
-    def get_class(self, language):
+    def get_class(self, language, test_case_type):
         """ Get the code evaluator class for the given language """
         if not self._register.get(language):
             self._register[language] = code_evaluators.get(language)
 
-        cls = self._register[language]
+        test_case_register = self._register[language]
+        cls = test_case_register.get(test_case_type)
         module_name, class_name = cls.rsplit(".", 1)
         # load the module, will raise ImportError if module cannot be loaded
         get_module = importlib.import_module(module_name)
diff --git a/yaksh/models.py b/yaksh/models.py
index c4f3561..6fa96bf 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -33,7 +33,7 @@ enrollment_methods = (
 test_case_types = (
         ("assert_based", "Assertion Based Testcase"),
         # ("argument_based", "Multiple Correct Choices"),
-        # ("stdout_based", "Code"),
+        ("stdout_based", "Stdout Based Testcase"),
     )
 
 attempts = [(i, i) for i in range(1, 6)]
@@ -170,6 +170,7 @@ class Question(models.Model):
 
     # The type of evaluator
     test_case_type = models.CharField(max_length=24, choices=test_case_types)
+    
     # Is this question active or not. If it is inactive it will not be used
     # when creating a QuestionPaper.
     active = models.BooleanField(default=True)
@@ -183,38 +184,39 @@ class Question(models.Model):
     # user for particular question
     user = models.ForeignKey(User, related_name="user")
 
-    def consolidate_answer_data(self, test_cases, user_answer):
+    def consolidate_answer_data(self, user_answer):
         test_case_data_dict = []
         question_info_dict = {}
 
-        for test_case in test_cases:
-            kw_args_dict = {}
-            pos_args_list = []
+        # for test_case in test_cases:
+        #     kw_args_dict = {}
+        #     pos_args_list = []
 
-            test_case_data = {}
-            test_case_data['test_id'] = test_case.id
-            test_case_data['func_name'] = test_case.func_name
-            test_case_data['expected_answer'] = test_case.expected_answer
+            # test_case_data = {}
+            # test_case_data['test_id'] = test_case.id
+            # test_case_data['func_name'] = test_case.func_name
+            # test_case_data['expected_answer'] = test_case.expected_answer
 
-            if test_case.kw_args:
-                for args in test_case.kw_args.split(","):
-                    arg_name, arg_value = args.split("=")
-                    kw_args_dict[arg_name.strip()] = arg_value.strip()
+            # if test_case.kw_args:
+            #     for args in test_case.kw_args.split(","):
+            #         arg_name, arg_value = args.split("=")
+            #         kw_args_dict[arg_name.strip()] = arg_value.strip()
 
-            if test_case.pos_args:
-                for args in test_case.pos_args.split(","):
-                    pos_args_list.append(args.strip())
+            # if test_case.pos_args:
+            #     for args in test_case.pos_args.split(","):
+            #         pos_args_list.append(args.strip())
 
-            test_case_data['kw_args'] = kw_args_dict
-            test_case_data['pos_args'] = pos_args_list
-            test_case_data_dict.append(test_case_data)
+            # test_case_data['kw_args'] = kw_args_dict
+            # test_case_data['pos_args'] = pos_args_list
+            # test_case_data_dict.append(test_case_data)
 
         # question_info_dict['language'] = self.language
-        question_info_dict['id'] = self.id
+        # question_info_dict['id'] = self.id
         question_info_dict['user_answer'] = user_answer
-        question_info_dict['test_parameter'] = test_case_data_dict
+        # question_info_dict['test_parameter'] = test_case_data_dict
         question_info_dict['ref_code_path'] = self.ref_code_path
         question_info_dict['test'] = self.test
+        # question_info_dict['test_case_type'] = self.test_case_type
 
         return json.dumps(question_info_dict)
 
diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py
index 3835b44..5722b2d 100644
--- a/yaksh/python_code_evaluator.py
+++ b/yaksh/python_code_evaluator.py
@@ -12,13 +12,13 @@ from code_evaluator import CodeEvaluator, TimeoutException
 class PythonCodeEvaluator(CodeEvaluator):
     """Tests the Python code obtained from Code Server"""
 
-    def check_code(self):
+    def check_code(self, test, user_answer, ref_code_path):
         success = False
 
         try:
             tb = None
-            test_code = self._create_test_case()
-            submitted = compile(self.user_answer, '<string>', mode='exec')
+            test_code = test
+            submitted = compile(user_answer, '<string>', mode='exec')
             g = {}
             exec submitted in g
             _tests = compile(test_code, '<string>', mode='exec')
@@ -40,26 +40,50 @@ class PythonCodeEvaluator(CodeEvaluator):
         del tb
         return success, err
 
-    def _create_test_case(self):
-        """
-            Create assert based test cases in python
-        """
-        test_code = ""
-        if self.test:
-            return self.test
-        elif self.test_case_data:
-            for test_case in self.test_case_data:
-                pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \
-                                    if test_case.get('pos_args') else ""
-                kw_args = ", ".join(str(k+"="+a) for k, a
-                                 in test_case.get('kw_args').iteritems()) \
-                                if test_case.get('kw_args') else ""
-                args = pos_args + ", " + kw_args if pos_args and kw_args \
-                                                    else pos_args or kw_args
-                function_name = test_case.get('func_name')
-                expected_answer = test_case.get('expected_answer')
+    # def check_code(self):
+    #     success = False
 
-                tcode = "assert {0}({1}) == {2}".format(function_name, args,
-                                             expected_answer)
-                test_code += tcode + "\n"
-            return test_code
+    #     try:
+    #         tb = None
+    #         test_code = self._create_test_case()
+    #         submitted = compile(self.user_answer, '<string>', mode='exec')
+    #         g = {}
+    #         exec submitted in g
+    #         _tests = compile(test_code, '<string>', mode='exec')
+    #         exec _tests in g
+    #     except AssertionError:
+    #         type, value, tb = sys.exc_info()
+    #         info = traceback.extract_tb(tb)
+    #         fname, lineno, func, text = info[-1]
+    #         text = str(test_code).splitlines()[lineno-1]
+    #         err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
+    #     else:
+    #         success = True
+    #         err = 'Correct answer'
+
+    #     del tb
+    #     return success, err
+
+    # def _create_test_case(self):
+    #     """
+    #         Create assert based test cases in python
+    #     """
+    #     test_code = ""
+    #     if self.test:
+    #         return self.test
+    #     elif self.test_case_data:
+    #         for test_case in self.test_case_data:
+    #             pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \
+    #                                 if test_case.get('pos_args') else ""
+    #             kw_args = ", ".join(str(k+"="+a) for k, a
+    #                              in test_case.get('kw_args').iteritems()) \
+    #                             if test_case.get('kw_args') else ""
+    #             args = pos_args + ", " + kw_args if pos_args and kw_args \
+    #                                                 else pos_args or kw_args
+    #             function_name = test_case.get('func_name')
+    #             expected_answer = test_case.get('expected_answer')
+
+    #             tcode = "assert {0}({1}) == {2}".format(function_name, args,
+    #                                          expected_answer)
+    #             test_code += tcode + "\n"
+    #         return test_code
diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py
new file mode 100644
index 0000000..89d3424
--- /dev/null
+++ b/yaksh/python_stdout_evaluator.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+import sys
+import traceback
+import os
+from os.path import join
+import importlib
+from contextlib import contextmanager
+
+# local imports
+from code_evaluator import CodeEvaluator
+
+
+@contextmanager
+def redirect_stdout():
+        from StringIO import StringIO
+        new_target = StringIO()
+
+        old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+        try:
+            yield new_target # run some code with the replaced stdout
+        finally:
+            sys.stdout = old_target # restore to the previous value
+
+
+class PythonStdoutEvaluator(CodeEvaluator):
+    """Tests the Python code obtained from Code Server"""
+
+    def check_code(self, test, user_answer, ref_code_path):
+        success = False
+
+        try:
+            tb = None
+            test_code = test
+            submitted = compile(user_answer, '<string>', mode='exec')
+            with redirect_stdout() as output_buffer: 
+                g = {}
+                exec submitted in g
+            raw_output_value = output_buffer.getvalue()
+            output_value = raw_output_value.encode('string_escape').strip()
+            if output_value == str(test_code):
+                success = True
+                err = 'Correct answer'
+            else:
+                raise ValueError("Incorrect Answer")
+
+        del tb
+        return success, err
\ No newline at end of file
diff --git a/yaksh/settings.py b/yaksh/settings.py
index 63bd875..f8b240d 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -20,7 +20,10 @@ SERVER_TIMEOUT = 2
 URL_ROOT = ''
 
 code_evaluators = {
-            "python": "python_code_evaluator.PythonCodeEvaluator",
+            "python": {"assert_based": "python_code_evaluator.PythonCodeEvaluator",
+            		"argument_based": "python_argument_based_evaluator.PythonCodeEvaluator",
+            		"stdout_based": "python_stdout_evaluator.PythonStdoutEvaluator"
+            	},
             "c": "cpp_code_evaluator.CppCodeEvaluator",
             "cpp": "cpp_code_evaluator.CppCodeEvaluator",
             "java": "java_code_evaluator.JavaCodeEvaluator",
diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html
index 1b1d28d..88d8f03 100644
--- a/yaksh/templates/yaksh/add_question.html
+++ b/yaksh/templates/yaksh/add_question.html
@@ -30,6 +30,7 @@
   <tr><td id='label_option'>Options: <td>{{ form.options }}  {{form.options.errors}}
   <tr><td id='label_solution'>Test: <td>{{ form.test }}  {{form.test.errors}}
   <tr><td id='label_ref_code_path'>Reference Code Path: <td>{{ form.ref_code_path }}  {{form.ref_code_path.errors}}
+  <tr><td> test_case_type: <td> {{ form.test_case_type }}{{ form.test_case_type.errors }}
 
     <form method="post" action="">
       {% if formset%}
diff --git a/yaksh/tester/python/verifier.py b/yaksh/tester/python/verifier.py
new file mode 100644
index 0000000..102dcb9
--- /dev/null
+++ b/yaksh/tester/python/verifier.py
@@ -0,0 +1,121 @@
+import sys
+from .utils import import_by_path
+from contextlib import contextmanager
+
+
+@contextmanager
+def redirect_stdout():
+    from StringIO import StringIO
+    new_target = StringIO()
+
+    old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+    try:
+        yield new_target # run some code with the replaced stdout
+    finally:
+        sys.stdout = old_target # restore to the previous value
+
+# def redirect_stdout():
+#     # import sys
+#     from StringIO import StringIO
+#     oldout,olderr = sys.stdout, sys.stderr
+#     try:
+#         out = StringIO()
+#         err = StringIO()
+#         # sys.stdout,sys.stderr = out, err
+#         yield out, err
+#     finally:
+#         sys.stdout,sys.stderr = oldout, olderr
+#         out = out.getvalue()
+#         err = err.getvalue()
+
+TESTER_BACKEND = {
+    "python": "PythonPrintTesterBackend" #@@@rename to test-case-creator, this file should be backend.py
+}
+
+class TesterException(Exception):
+    """ Parental class for all tester exceptions """
+    pass
+
+class UnknownBackendException(TesterException):
+    """ Exception thrown if tester backend is not recognized. """
+    pass
+
+
+def detect_backend(language):
+    """
+    Detect the right backend for a test case.
+    """
+    backend_name = TESTER_BACKEND.get(language)
+    # backend = import_by_path(backend_name)
+    backend = PythonTesterBackend() #@@@
+    return backend
+
+class PythonPrintTesterBackend(object):
+    def test_code(self, submitted, reference_output):
+        """
+        create a test command
+        """
+        with redirect_stdout() as output_buffer: 
+            g = {}
+            exec submitted in g
+
+        # return_buffer = out.encode('string_escape')
+        raw_output_value = output_buffer.getvalue()
+        output_value = raw_output_value.encode('string_escape').strip()
+        if output_value == str(reference_output):
+            return True
+        else:
+            raise ValueError("Incorrect Answer", output_value, reference_output)
+
+
+class PythonTesterBackend(object):
+    # def __init__(self, test_case):
+    #     self._test_case = test_case
+    def create(self): #@@@ test()
+        """
+        create a test command
+        """
+        test_code = "assert {0}({1}) == {2}".format(self.test_case_parameters['function_name'], self.test_case_parameters['args'],
+                                     self.test_case_parameters['expected_answer'])
+        return test_code
+
+    def pack(self, test_case):
+        kw_args_dict = {}
+        pos_args_list = []
+        test_case_data = {}
+        test_case_data['test_id'] = test_case.id
+        test_case_data['func_name'] = test_case.func_name
+        test_case_data['expected_answer'] = test_case.expected_answer
+
+        if test_case.kw_args:
+            for args in test_case.kw_args.split(","):
+                arg_name, arg_value = args.split("=")
+                kw_args_dict[arg_name.strip()] = arg_value.strip()
+
+        if test_case.pos_args:
+            for args in test_case.pos_args.split(","):
+                pos_args_list.append(args.strip())
+
+        test_case_data['kw_args'] = kw_args_dict
+        test_case_data['pos_args'] = pos_args_list
+
+        return test_case_data
+
+    def unpack(self, test_case_data):
+        pos_args = ", ".join(str(i) for i in test_case_data.get('pos_args')) \
+                            if test_case_data.get('pos_args') else ""
+        kw_args = ", ".join(str(k+"="+a) for k, a
+                         in test_case_data.get('kw_args').iteritems()) \
+                        if test_case_data.get('kw_args') else ""
+        args = pos_args + ", " + kw_args if pos_args and kw_args \
+                                            else pos_args or kw_args
+        function_name = test_case_data.get('func_name')
+        expected_answer = test_case_data.get('expected_answer')
+
+        self.test_case_parameters = {
+            'args': args,
+            'function_name': function_name,
+            'expected_answer': expected_answer
+        }
+
+        return self.test_case_parameters
\ No newline at end of file
diff --git a/yaksh/views.py b/yaksh/views.py
index f540351..520f396 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -513,7 +513,7 @@ def validate_answer(user, user_answer, question, json_data=None):
                 message = 'Correct answer'
         elif question.type == 'code':
             user_dir = get_user_dir(user)
-            json_result = code_server.run_code(question.language, json_data, user_dir)
+            json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir)
             result = json.loads(json_result)
             if result.get('success'):
                 correct = True
diff --git a/yaksh/xmlrpc_clients.py b/yaksh/xmlrpc_clients.py
index 3a3c0c6..7124550 100644
--- a/yaksh/xmlrpc_clients.py
+++ b/yaksh/xmlrpc_clients.py
@@ -23,7 +23,7 @@ class CodeServerProxy(object):
         pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT)
         self.pool_server = ServerProxy(pool_url)
 
-    def run_code(self, language, json_data, user_dir):
+    def run_code(self, language, test_case_type, json_data, user_dir):
         """Tests given code (`answer`) with the `test_code` supplied.  If the
         optional `in_dir` keyword argument is supplied it changes the directory
         to that directory (it does not change it back to the original when
@@ -51,7 +51,7 @@ class CodeServerProxy(object):
 
         try:
             server = self._get_server()
-            result = server.check_code(language, json_data, user_dir)
+            result = server.check_code(language, test_case_type, json_data, user_dir)
         except ConnectionError:
             result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'})
         return result