- Add test cases for multiple python evaluators

- Change name from python_standard_evaluator to python_assertion_evaluator

6 files changed, 149 insertions, 172 deletions

diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index 7e2a729..535daa3 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -52,9 +52,7 @@ class CodeEvaluator(object):
     """Tests the code obtained from Code Server"""
     # def __init__(self, test_case_data, test, language, user_answer, 
     #                  ref_code_path=None, in_dir=None):
-    def __init__(self, in_dir, **kwargs):
-
-
+    def __init__(self, in_dir=None):
         msg = 'Code took more than %s seconds to run. You probably '\
               'have an infinite loop in your code.' % SERVER_TIMEOUT
         self.timeout_msg = msg
@@ -64,7 +62,7 @@ class CodeEvaluator(object):
         # self.ref_code_path = ref_code_path  #@@@specific to check-code
         # self.test = test                    #@@@specific to check-code
         self.in_dir = in_dir                #@@@Common for all, no change
-        self.test_case_args = None          #@@@no change
+        # self.test_case_args = None          #@@@no change
 
     # Public Protocol ##########
     # @classmethod
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index c55f04f..e157cde 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,56 +1,47 @@
 import unittest
 import os
-from yaksh.python_code_evaluator import PythonCodeEvaluator
+from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
+from yaksh.python_stdout_evaluator import PythonStdoutEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 from textwrap import dedent
 
 
-class PythonEvaluationTestCases(unittest.TestCase):
+class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "Python"
-        self.test = None
-        self.test_case_data = [{"func_name": "add",
-                                "expected_answer": "5",
-                                "test_id": u'null',
-                                "pos_args": ["3", "2"],
-                                "kw_args": {}
-                                }]
+        self.test_case_data = ['assert(add(1,2)==3)']
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a + b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        # {u'user_answer': u'def adder(a,b):\r\n    return a', u'test_case_data': [u'assert(adder(1,2)==3)']}
+        user_answer = "def add(a,b):\n\treturn a + b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get('error'), "Correct answer")
 
     def test_incorrect_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a - b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEqual(result.get("error"), "AssertionError  in: assert add(3, 2) == 5")
+        user_answer = "def add(a,b):\n\treturn a - b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), "AssertionError  in: assert(add(1,2)==3)")
 
     def test_infinite_loop(self):
-        user_answer = dedent("""
-        def add(a, b):
-            while True:
-                pass
-        """)
-        timeout_msg = ("Code took more than {0} seconds to run. "
-                       "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEquals(result.get("error"), timeout_msg)
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), self.timeout_msg)
 
     def test_syntax_error(self):
         user_answer = dedent("""
@@ -59,9 +50,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         syntax_error_msg = ["Traceback", "call", "File", "line", "<string>",
                             "SyntaxError", "invalid syntax"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -75,9 +68,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         indent_error_msg = ["Traceback", "call", "File", "line", "<string>",
                             "IndentationError", "indented block"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -87,9 +82,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
     def test_name_error(self):
         user_answer = ""
         name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -103,9 +100,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         recursion_error_msg = ["Traceback", "call", "RuntimeError",
                                 "maximum recursion depth exceeded"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -118,9 +117,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
             return a + b
         """)
         type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -134,14 +135,54 @@ class PythonEvaluationTestCases(unittest.TestCase):
             return int(a) + int(b) + int(c)
         """)
         value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
         for msg in value_error_msg:
             self.assertIn(msg, result.get("error"))
 
+class PythonStdoutEvaluationTestCases(unittest.TestCase):
+    def setUp(self):
+        self.output = ['Hello World\\n']
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
+
+    def test_correct_answer(self):
+        user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertEqual(result.get('error'), "Correct answer")
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        user_answer = "a = 'Goodbye'\nb = 'Name'\nprint '{0} {1}'.format(a, b)"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), "Incorrect Answer")
+
+    def test_infinite_loop(self):
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), 'Incorrect Answer')
+
+>>>>>>> - Add test cases for multiple python evaluators
+
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
new file mode 100644
index 0000000..0615d84
--- /dev/null
+++ b/yaksh/python_assertion_evaluator.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+import sys
+import traceback
+import os
+from os.path import join
+import importlib
+
+# local imports
+from code_evaluator import CodeEvaluator, TimeoutException
+
+
+class PythonAssertionEvaluator(CodeEvaluator):
+    """Tests the Python code obtained from Code Server"""
+
+    # def check_code(self, test, user_answer, ref_code_path):
+    def check_code(self, user_answer, test_case_data):
+        success = False
+        try:
+            tb = None
+            submitted = compile(user_answer, '<string>', mode='exec')
+            g = {}
+            exec submitted in g
+            for test_code in test_case_data:
+                _tests = compile(test_code, '<string>', mode='exec')
+                exec _tests in g
+        except AssertionError:
+            type, value, tb = sys.exc_info()
+            info = traceback.extract_tb(tb)
+            fname, lineno, func, text = info[-1]
+            text = str(test_code).splitlines()[lineno-1]
+            err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
+        except TimeoutException:
+            raise
+        except Exception:
+            err = traceback.format_exc(limit=0)
+        else:
+            success = True
+            err = 'Correct answer'
+
+        del tb
+        return success, err
diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py
deleted file mode 100644
index a131a0e..0000000
--- a/yaksh/python_code_evaluator.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-import sys
-import traceback
-import os
-from os.path import join
-import importlib
-
-# local imports
-from code_evaluator import CodeEvaluator, TimeoutException
-
-
-class PythonCodeEvaluator(CodeEvaluator):
-    """Tests the Python code obtained from Code Server"""
-
-    # def check_code(self, test, user_answer, ref_code_path):
-    def check_code(self, user_answer, test_cases):
-        success = False
-
-        try:
-            tb = None
-            submitted = compile(user_answer, '<string>', mode='exec')
-            g = {}
-            exec submitted in g
-            for test_code in test_cases:
-                _tests = compile(test_code, '<string>', mode='exec')
-                exec _tests in g
-        except AssertionError:
-            type, value, tb = sys.exc_info()
-            info = traceback.extract_tb(tb)
-            fname, lineno, func, text = info[-1]
-            text = str(test_code).splitlines()[lineno-1]
-            err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
-        except TimeoutException:
-            raise
-        except Exception:
-            err = traceback.format_exc(limit=0)
-        else:
-            success = True
-            err = 'Correct answer'
-
-        del tb
-        return success, err
-
-    # def unpack_test_case_data(self, test_case_data):
-    #     test_cases = []
-    #     for t in test_case_data:
-    #         test_case = t.get('test_case')
-    #         test_cases.append(test_case)
-
-    #     return test_cases
-
-    # def check_code(self):
-    #     success = False
-
-    #     try:
-    #         tb = None
-    #         test_code = self._create_test_case()
-    #         submitted = compile(self.user_answer, '<string>', mode='exec')
-    #         g = {}
-    #         exec submitted in g
-    #         _tests = compile(test_code, '<string>', mode='exec')
-    #         exec _tests in g
-    #     except AssertionError:
-    #         type, value, tb = sys.exc_info()
-    #         info = traceback.extract_tb(tb)
-    #         fname, lineno, func, text = info[-1]
-    #         text = str(test_code).splitlines()[lineno-1]
-    #         err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
-    #     else:
-    #         success = True
-    #         err = 'Correct answer'
-
-    #     del tb
-    #     return success, err
-
-    # def _create_test_case(self):
-    #     """
-    #         Create assert based test cases in python
-    #     """
-    #     test_code = ""
-    #     if self.test:
-    #         return self.test
-    #     elif self.test_case_data:
-    #         for test_case in self.test_case_data:
-    #             pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \
-    #                                 if test_case.get('pos_args') else ""
-    #             kw_args = ", ".join(str(k+"="+a) for k, a
-    #                              in test_case.get('kw_args').iteritems()) \
-    #                             if test_case.get('kw_args') else ""
-    #             args = pos_args + ", " + kw_args if pos_args and kw_args \
-    #                                                 else pos_args or kw_args
-    #             function_name = test_case.get('func_name')
-    #             expected_answer = test_case.get('expected_answer')
-
-    #             tcode = "assert {0}({1}) == {2}".format(function_name, args,
-    #                                          expected_answer)
-    #             test_code += tcode + "\n"
-    #         return test_code
diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py
index 28c3372..8f3eb65 100644
--- a/yaksh/python_stdout_evaluator.py
+++ b/yaksh/python_stdout_evaluator.py
@@ -25,29 +25,24 @@ def redirect_stdout():
 class PythonStdoutEvaluator(CodeEvaluator):
     """Tests the Python code obtained from Code Server"""
 
-    def check_code(self, user_answer, test_cases):
+    def check_code(self, user_answer, test_case_data):
         success = False
 
         tb = None
-        expected_output = test_cases[0]
+        expected_output = test_case_data[0]
         submitted = compile(user_answer, '<string>', mode='exec')
         with redirect_stdout() as output_buffer:
             g = {}
             exec submitted in g
         raw_output_value = output_buffer.getvalue()
         output_value = raw_output_value.encode('string_escape').strip()
-        if output_value == str(test_code):
+        if output_value == expected_output:
             success = True
-            err = 'Correct answer'
+            err = "Correct answer"
+
         else:
             success = False
             err = "Incorrect Answer"
 
         del tb
         return success, err
-
-    # def unpack_test_case_data(self, test_case_data):
-    #     for t in test_case_data:
-    #         test_case = t.get('output')
-
-    #     return test_case
diff --git a/yaksh/settings.py b/yaksh/settings.py
index a4e78db..30fab0a 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -20,7 +20,7 @@ SERVER_TIMEOUT = 2
 URL_ROOT = ''
 
 code_evaluators = {
-            "python": {"standardtestcase": "python_code_evaluator.PythonCodeEvaluator",
+            "python": {"standardtestcase": "python_assertion_evaluator.PythonStandardEvaluator",
             		"stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator"
         	},
             "c": "cpp_code_evaluator.CppCodeEvaluator",