- Add test cases for multiple python evaluators

- Change name from python_standard_evaluator to python_assertion_evaluator

1 files changed, 100 insertions, 59 deletions

diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index c55f04f..e157cde 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,56 +1,47 @@
 import unittest
 import os
-from yaksh.python_code_evaluator import PythonCodeEvaluator
+from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
+from yaksh.python_stdout_evaluator import PythonStdoutEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 from textwrap import dedent
 
 
-class PythonEvaluationTestCases(unittest.TestCase):
+class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "Python"
-        self.test = None
-        self.test_case_data = [{"func_name": "add",
-                                "expected_answer": "5",
-                                "test_id": u'null',
-                                "pos_args": ["3", "2"],
-                                "kw_args": {}
-                                }]
+        self.test_case_data = ['assert(add(1,2)==3)']
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a + b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        # {u'user_answer': u'def adder(a,b):\r\n    return a', u'test_case_data': [u'assert(adder(1,2)==3)']}
+        user_answer = "def add(a,b):\n\treturn a + b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get('error'), "Correct answer")
 
     def test_incorrect_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a - b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEqual(result.get("error"), "AssertionError  in: assert add(3, 2) == 5")
+        user_answer = "def add(a,b):\n\treturn a - b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), "AssertionError  in: assert(add(1,2)==3)")
 
     def test_infinite_loop(self):
-        user_answer = dedent("""
-        def add(a, b):
-            while True:
-                pass
-        """)
-        timeout_msg = ("Code took more than {0} seconds to run. "
-                       "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEquals(result.get("error"), timeout_msg)
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), self.timeout_msg)
 
     def test_syntax_error(self):
         user_answer = dedent("""
@@ -59,9 +50,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         syntax_error_msg = ["Traceback", "call", "File", "line", "<string>",
                             "SyntaxError", "invalid syntax"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -75,9 +68,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         indent_error_msg = ["Traceback", "call", "File", "line", "<string>",
                             "IndentationError", "indented block"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -87,9 +82,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
     def test_name_error(self):
         user_answer = ""
         name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -103,9 +100,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
         """)
         recursion_error_msg = ["Traceback", "call", "RuntimeError",
                                 "maximum recursion depth exceeded"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -118,9 +117,11 @@ class PythonEvaluationTestCases(unittest.TestCase):
             return a + b
         """)
         type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -134,14 +135,54 @@ class PythonEvaluationTestCases(unittest.TestCase):
             return int(a) + int(b) + int(c)
         """)
         value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
         for msg in value_error_msg:
             self.assertIn(msg, result.get("error"))
 
+class PythonStdoutEvaluationTestCases(unittest.TestCase):
+    def setUp(self):
+        self.output = ['Hello World\\n']
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
+
+    def test_correct_answer(self):
+        user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertEqual(result.get('error'), "Correct answer")
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        user_answer = "a = 'Goodbye'\nb = 'Name'\nprint '{0} {1}'.format(a, b)"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), "Incorrect Answer")
+
+    def test_infinite_loop(self):
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.output
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), 'Incorrect Answer')
+
+>>>>>>> - Add test cases for multiple python evaluators
+
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()