Merge pull request #96 from ankitjavalkar/code-eval-refactor-clean2

Code evaluator refactor

6 files changed, 322 insertions, 193 deletions

diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index c6a5408..4ff3e0a 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -5,37 +5,48 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class BashEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "bash"
-        self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args"
+        self.test_case_data = [
+            {"test_case": "bash_files/sample.sh,bash_files/sample.args"}
+        ]
         self.in_dir = "/tmp"
-        self.test_case_data = []
         self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        self.test = None
+            "You probably have an infinite loop in your"
+            " code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
-        user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
-        get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
+            " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
+        )
+        get_class = BashCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertTrue(result.get('success'))
+        self.assertEquals(result.get('error'), "Correct answer")
 
     def test_error(self):
-        user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))"
-        get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
+        user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] "
+            "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))")
+        get_class = BashCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get("success"))
         self.assertTrue("Error" in result.get("error"))
 
     def test_infinite_loop(self):
-        user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done"
-        get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
+        user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
+            " do echo "" > /dev/null ; done")
+        get_class = BashCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get("success"))
         self.assertEquals(result.get("error"), self.timeout_msg)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index d01cc9c..71fb843 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -5,71 +5,50 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class CEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "C"
-        self.ref_code_path = "c_cpp_files/main.cpp"
+        self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
         self.in_dir = "/tmp"
-        self.test_case_data = []
         self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        self.test = None
+            "You probably have an infinite loop in your"
+            " code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
         user_answer = "int add(int a, int b)\n{return a+b;}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
-
-    def test_compilation_error(self):
-        user_answer = "int add(int a, int b)\n{return a+b}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
-        self.assertFalse(result.get("success"))
-        self.assertTrue("Compilation Error" in result.get("error"))
-
-    def test_infinite_loop(self):
-        user_answer = "int add(int a, int b)\n{while(1>0){}}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
- 
-        self.assertFalse(result.get("success"))
-        self.assertEquals(result.get("error"), self.timeout_msg)
-
-
-###############################################################################
-class CppEvaluationTestCases(unittest.TestCase):
-    def setUp(self):
-        self.language = "CPP"
-        self.ref_code_path = "c_cpp_files/main.cpp"
-        self.in_dir = "/tmp"
-        self.test_case_data = []
-        self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        self.test = None
-
-    def test_correct_answer(self):
-        user_answer = "int add(int a, int b)\n{return a+b;}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        get_class = CppCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertTrue(result.get('success'))
+        self.assertEquals(result.get('error'), "Correct answer")
+
+    def test_incorrect_answer(self):
+        user_answer = "int add(int a, int b)\n{return a-b;}"
+        get_class = CppCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertIn("Incorrect:", result.get('error'))
+        self.assertTrue(result.get('error').splitlines > 1)
 
     def test_compilation_error(self):
         user_answer = "int add(int a, int b)\n{return a+b}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
+        get_class = CppCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get("success"))
         self.assertTrue("Compilation Error" in result.get("error"))
 
     def test_infinite_loop(self):
         user_answer = "int add(int a, int b)\n{while(1>0){}}"
-        get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
- 
+        get_class = CppCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get("success"))
         self.assertEquals(result.get("error"), self.timeout_msg)
 
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py
index 84701fb..51c0c51 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_code_evaluation.py
@@ -1,20 +1,44 @@
 import unittest
 import os
-from yaksh import python_code_evaluator
-from yaksh.language_registry import _LanguageRegistry, set_registry, get_registry
-from yaksh.settings import SERVER_TIMEOUT
+from yaksh import python_assertion_evaluator
+from yaksh.language_registry import _LanguageRegistry, get_registry
+from yaksh.settings import SERVER_TIMEOUT, code_evaluators
 
 
 class RegistryTestCase(unittest.TestCase):
     def setUp(self):
-        set_registry()
         self.registry_object = get_registry()
         self.language_registry = _LanguageRegistry()
+        assertion_evaluator_path = ("yaksh.python_assertion_evaluator"
+            ".PythonAssertionEvaluator"
+        )
+        stdout_evaluator_path = ("yaksh.python_stdout_evaluator."
+            "PythonStdoutEvaluator"
+        )
+        code_evaluators['python'] = \
+        {"standardtestcase": assertion_evaluator_path,
+        "stdoutbasedtestcase": stdout_evaluator_path
+        }
 
     def test_set_register(self):
-        class_name = getattr(python_code_evaluator, 'PythonCodeEvaluator')
-        self.registry_object.register("python", "yaksh.python_code_evaluator.PythonCodeEvaluator")
-        self.assertEquals(self.registry_object.get_class("python"), class_name)
+        evaluator_class = self.registry_object.get_class("python", 
+            "standardtestcase"
+        )
+        assertion_evaluator_path = ("yaksh.python_assertion_evaluator"
+            ".PythonAssertionEvaluator"
+        )
+        stdout_evaluator_path = ("yaksh.python_stdout_evaluator."
+            "PythonStdoutEvaluator"
+        )
+        class_name = getattr(python_assertion_evaluator, 
+            'PythonAssertionEvaluator'
+        )
+        self.registry_object.register("python", 
+            {"standardtestcase": assertion_evaluator_path,
+                "stdoutbasedtestcase": stdout_evaluator_path
+            }
+        )
+        self.assertEquals(evaluator_class, class_name)
 
     def tearDown(self):
         self.registry_object = None
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index dfa1066..801277f 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -2,52 +2,64 @@ import unittest
 import os
 from yaksh import code_evaluator as evaluator
 from yaksh.java_code_evaluator import JavaCodeEvaluator
+from yaksh.settings import SERVER_TIMEOUT
 
 
 class JavaEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "java"
-        self.ref_code_path = "java_files/main_square.java"
+        self.test_case_data = [
+            {"test_case": "java_files/main_square.java"}
+        ]
         self.in_dir = "/tmp"
-        self.test_case_data = []
         evaluator.SERVER_TIMEOUT = 9
         self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop in "
-                            "your code.").format(evaluator.SERVER_TIMEOUT)
-        self.test = None
+            "You probably have an infinite loop in"
+            " your code.").format(evaluator.SERVER_TIMEOUT)
 
     def tearDown(self):
         evaluator.SERVER_TIMEOUT = 2
 
     def test_correct_answer(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
-        get_class = JavaCodeEvaluator(self.test_case_data, self.test,
-                                         self.language, user_answer,
-                                         self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
+        get_class = JavaCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertEquals(result.get('error'), "Correct answer")
+        self.assertTrue(result.get('success'))
 
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+    def test_incorrect_answer(self):
+        user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
+        get_class = JavaCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertIn("Incorrect:", result.get('error'))
+        self.assertTrue(result.get('error').splitlines > 1)
 
     def test_error(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
-        get_class = JavaCodeEvaluator(self.test_case_data, self.test,
-                                         self.language, user_answer,
-                                         self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
- 
+        get_class = JavaCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs) 
         self.assertFalse(result.get("success"))
         self.assertTrue("Error" in result.get("error"))
 
     def test_infinite_loop(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
-        get_class = JavaCodeEvaluator(self.test_case_data, self.test,
-                                         self.language, user_answer,
-                                         self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
- 
+        get_class = JavaCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs) 
         self.assertFalse(result.get("success"))
         self.assertEquals(result.get("error"), self.timeout_msg)
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index c55f04f..1e867a3 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,67 +1,71 @@
 import unittest
 import os
-from yaksh.python_code_evaluator import PythonCodeEvaluator
+from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
+from yaksh.python_stdout_evaluator import PythonStdoutEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 from textwrap import dedent
 
 
-class PythonEvaluationTestCases(unittest.TestCase):
+class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "Python"
-        self.test = None
-        self.test_case_data = [{"func_name": "add",
-                                "expected_answer": "5",
-                                "test_id": u'null',
-                                "pos_args": ["3", "2"],
-                                "kw_args": {}
-                                }]
+        self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
+            {"test_case": 'assert(add(-1,2)==1)'},
+            {"test_case":  'assert(add(-1,-2)==-3)'},
+        ]
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop in"
+            " your code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a + b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        user_answer = "def add(a,b):\n\treturn a + b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get('error'), "Correct answer")
 
     def test_incorrect_answer(self):
-        user_answer = dedent("""
-        def add(a, b):
-            return a - b
-        """)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEqual(result.get("error"), "AssertionError  in: assert add(3, 2) == 5")
+        user_answer = "def add(a,b):\n\treturn a - b"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'),
+            "AssertionError  in: assert(add(1,2)==3)"
+        )
 
     def test_infinite_loop(self):
-        user_answer = dedent("""
-        def add(a, b):
-            while True:
-                pass
-        """)
-        timeout_msg = ("Code took more than {0} seconds to run. "
-                       "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
-        self.assertFalse(result.get("success"))
-        self.assertEquals(result.get("error"), timeout_msg)
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), self.timeout_msg)
 
     def test_syntax_error(self):
         user_answer = dedent("""
         def add(a, b);
             return a + b
         """)
-        syntax_error_msg = ["Traceback", "call", "File", "line", "<string>",
-                            "SyntaxError", "invalid syntax"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        syntax_error_msg = ["Traceback", 
+            "call", 
+            "File", 
+            "line", 
+            "<string>",
+            "SyntaxError", 
+            "invalid syntax"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -73,11 +77,18 @@ class PythonEvaluationTestCases(unittest.TestCase):
         def add(a, b):
         return a + b
         """)
-        indent_error_msg = ["Traceback", "call", "File", "line", "<string>",
-                            "IndentationError", "indented block"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        indent_error_msg = ["Traceback", "call", 
+            "File",
+            "line",
+            "<string>",
+            "IndentationError",
+            "indented block"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
@@ -86,10 +97,17 @@ class PythonEvaluationTestCases(unittest.TestCase):
 
     def test_name_error(self):
         user_answer = ""
-        name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        name_error_msg = ["Traceback", 
+            "call",
+            "NameError",
+            "name",
+            "defined"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -101,11 +119,16 @@ class PythonEvaluationTestCases(unittest.TestCase):
         def add(a, b):
             return add(3, 3)
         """)
-        recursion_error_msg = ["Traceback", "call", "RuntimeError",
-                                "maximum recursion depth exceeded"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        recursion_error_msg = ["Traceback",
+            "call", 
+            "RuntimeError",
+            "maximum recursion depth exceeded"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -117,10 +140,17 @@ class PythonEvaluationTestCases(unittest.TestCase):
         def add(a):
             return a + b
         """)
-        type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        type_error_msg = ["Traceback", 
+            "call", 
+            "TypeError",
+            "exactly",
+            "argument"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
@@ -133,15 +163,73 @@ class PythonEvaluationTestCases(unittest.TestCase):
             c = 'a'
             return int(a) + int(b) + int(c)
         """)
-        value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"]
-        get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
-                                            self.language, user_answer)
-        result = get_evaluator.evaluate()
+        value_error_msg = ["Traceback",
+            "call",
+            "ValueError",
+            "invalid literal",
+            "base"
+        ]
+        get_class = PythonAssertionEvaluator()
+        kwargs = {'user_answer': user_answer,
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
         err = result.get("error").splitlines()
         self.assertFalse(result.get("success"))
         self.assertEqual(2, len(err))
         for msg in value_error_msg:
             self.assertIn(msg, result.get("error"))
 
+class PythonStdoutEvaluationTestCases(unittest.TestCase):
+    def setUp(self):
+        self.test_case_data = [{"expected_output": "0 1 1 2 3"}]
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop"
+            " in your code.").format(SERVER_TIMEOUT)
+
+    def test_correct_answer(self):
+        user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertEqual(result.get('error'), "Correct answer")
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer, 
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), "Incorrect Answer")
+
+    def test_direct_printed_answer(self):
+        user_answer = "print '0 1 1 2 3'"
+        error_msg = ("Incorrect Answer: Please avoid printing"
+            " the expected output directly"
+        )
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer, 
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), error_msg)
+
+    def test_infinite_loop(self):
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        get_class = PythonStdoutEvaluator()
+        kwargs = {'user_answer': user_answer,
+            'test_case_data': self.test_case_data
+        }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), 'Incorrect Answer')
+
+
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 30af041..242f260 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -1,45 +1,60 @@
 import unittest
 import os
+from yaksh import code_evaluator as evaluator
 from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 
 class ScilabEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.language = "scilab"
-        self.ref_code_path = "scilab_files/test_add.sce"
+        self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
         self.in_dir = "/tmp"
-        self.test_case_data = []
         self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
-        self.test = None
+                            "You probably have an infinite loop" 
+                            " in your code.").format(SERVER_TIMEOUT)
 
     def test_correct_answer(self):
-        user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction"
-        get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
-                                             self.language, user_answer,
-                                             self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
-
-        self.assertTrue(result.get("success"))
-        self.assertEqual(result.get("error"), "Correct answer")
+        user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
+                        "\n\tc=a+b;\nendfunction")
+        get_class = ScilabCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertEquals(result.get('error'), "Correct answer")
+        self.assertTrue(result.get('success'))
 
     def test_error(self):
-        user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction"
-        get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
-                                             self.language, user_answer,
-                                             self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
- 
+        user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
+                        "\n\tc=a+b;\ndis(\tendfunction")
+        get_class = ScilabCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs) 
         self.assertFalse(result.get("success"))
         self.assertTrue("error" in result.get("error"))
 
-    def test_infinite_loop(self):
-        user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a;\nwhile(1==1)\nend\nendfunction"
-        get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
-                                         self.language, user_answer,
-                                         self.ref_code_path, self.in_dir)
-        result = get_class.evaluate()
 
+    def test_incorrect_answer(self):
+        user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
+                        "\n\tc=a-b;\nendfunction")
+        get_class = ScilabCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs)
+        self.assertFalse(result.get('success'))
+        self.assertIn("Message", result.get('error'))
+        self.assertTrue(result.get('error').splitlines > 1)
+
+    def test_infinite_loop(self):
+        user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
+                        "\n\tc=a;\nwhile(1==1)\nend\nendfunction")
+        get_class = ScilabCodeEvaluator(self.in_dir)
+        kwargs = {'user_answer': user_answer, 
+                    'test_case_data': self.test_case_data
+                }
+        result = get_class.evaluate(**kwargs) 
         self.assertFalse(result.get("success"))
         self.assertEquals(result.get("error"), self.timeout_msg)