Fix test cases to reflect code server and python evaluator changes

3 files changed, 411 insertions, 658 deletions

diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 43cd0ea..688002f 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -88,6 +88,148 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                         result.get('error')
                       )
 
+    def test_partial_incorrect_answer(self):
+        # Given
+        user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
+        test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0},
+                            {"test_case_type": "standardtestcase", "test_case":  'assert(add(-1,-2)==-3)', 'weight': 1.0},
+                            {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0}
+                               ]
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': True
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('weight'), 2.0)
+        self.assertIn('AssertionError  in: assert(add(-1,2)==1)',
+                        result.get('error')
+                      )
+        self.assertIn('AssertionError  in: assert(add(-1,-2)==-3)',
+                        result.get('error')
+                      )
+
+    def test_infinite_loop(self):
+        # Given
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        # When
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assertEqual(result.get('error'), self.timeout_msg)
+
+    def test_syntax_error(self):
+        # Given
+        user_answer = dedent("""
+        def add(a, b);
+            return a + b
+        """)
+        syntax_error_msg = ["Traceback",
+                            "call",
+                            "File",
+                            "line",
+                            "<string>",
+                            "SyntaxError",
+                            "invalid syntax"
+                            ]
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        # When
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
+        err = result.get("error").splitlines()
+
+        # Then
+        self.assertFalse(result.get("success"))
+        self.assertEqual(5, len(err))
+        for msg in syntax_error_msg:
+            self.assertIn(msg, result.get("error"))
+
+    def test_indent_error(self):
+        # Given
+        user_answer = dedent("""
+        def add(a, b):
+        return a + b
+        """)
+        indent_error_msg = ["Traceback", "call",
+                            "File",
+                            "line",
+                            "<string>",
+                            "IndentationError",
+                            "indented block"
+                            ]
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        # When
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
+        err = result.get("error").splitlines()
+
+        # Then
+        self.assertFalse(result.get("success"))
+        self.assertEqual(5, len(err))
+        for msg in indent_error_msg:
+            self.assertIn(msg, result.get("error"))
 
     def test_name_error(self):
         # Given
@@ -98,20 +240,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                           "name",
                           "defined"
                           ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(3, len(err))
         for msg in name_error_msg:
             self.assertIn(msg, result.get("error"))
 
@@ -125,15 +277,25 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                                "call",
                                "maximum recursion depth exceeded"
                                ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
@@ -152,20 +314,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                           "TypeError",
                           "argument"
                           ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(3, len(err))
         for msg in type_error_msg:
             self.assertIn(msg, result.get("error"))
 
@@ -182,41 +354,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                            "invalid literal",
                            "base"
                            ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(9, len(err))
+        self.assertEqual(4, len(err))
         for msg in value_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_file_based_assert(self):
         # Given
-        self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}]
+        self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}]
         self.file_paths = [('/tmp/test.txt', False)]
         user_answer = dedent("""
             def ans():
                 with open("test.txt") as f:
                     return f.read()[0]
             """)
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertIn("Correct answer", result.get('error'))
@@ -227,7 +419,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         """ Tests the user answer with just an incorrect test case """
 
         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
-        test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False',
+        test_case_data = [{"test_case_type": "standardtestcase",
+                             "test_case": 's="abbb"\nasert palindrome(s)==False',
                              "weight": 0.0
                             }
                           ]
@@ -239,20 +432,30 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                             "SyntaxError",
                             "invalid syntax"
                             ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(6, len(err))
+        self.assertEqual(5, len(err))
         for msg in syntax_error_msg:
             self.assertIn(msg, result.get("error"))
 
@@ -262,45 +465,61 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
          first and then with an incorrect test case """
         # Given
         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
-        test_case_data = [{"test_case": 'assert(palindrome("abba")==True)',
+        test_case_data = [{"test_case_type": "standardtestcase",
+                             "test_case": 'assert(palindrome("abba")==True)',
                              "weight": 0.0
                             },
-                          {"test_case": 's="abbb"\nassert palindrome(S)==False',
+                          {"test_case_type": "standardtestcase",
+                             "test_case": 's="abbb"\nassert palindrome(S)==False',
                              "weight": 0.0
                             }
                           ]
         name_error_msg = ["Traceback",
                           "call",
+                          "File",
+                          "line",
+                          "<string>",
                           "NameError",
                           "name 'S' is not defined"
                           ]
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                }
 
         # When
-        evaluator = PythonAssertionEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
         err = result.get("error").splitlines()
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(7, len(err))
+        self.assertEqual(3, len(err))
         for msg in name_error_msg:
             self.assertIn(msg, result.get("error"))
 
-
 class PythonStdIOEvaluationTestCases(unittest.TestCase):
     def setUp(self):
         with open('/tmp/test.txt', 'wb') as f:
             f.write('2'.encode('ascii'))
         self.file_paths = None
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
 
     def test_correct_answer_integer(self):
         # Given
-        self.test_case_data = [{"expected_input": "1\n2",
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                                "expected_input": "1\n2",
                                 "expected_output": "3",
                                 "weight": 0.0
                                 }]
@@ -310,14 +529,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 print(a+b)
                              """
                              )
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'partial_grading': False
-                  }
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'partial_grading': False
+        #           }
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertTrue(result.get('success'))
@@ -325,7 +552,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_correct_answer_list(self):
         # Given
-        self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                                "expected_input": "1,2,3\n5,6,7",
                                 "expected_output": "[1, 2, 3, 5, 6, 7]",
                                 "weight": 0.0
                                 }]
@@ -338,14 +566,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 print(a+b)
                              """
                              )
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'partial_grading': False
+        #           }
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertTrue(result.get('success'))
@@ -353,7 +590,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_correct_answer_string(self):
         # Given
-        self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                                "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
                                 "expected_output": "2",
                                 "weight": 0.0
                                 }]
@@ -364,14 +602,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 print(a.count(b))
                              """
                              )
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'partial_grading': False
-                  }
+
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'partial_grading': False
+        #           }
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertTrue(result.get('success'))
@@ -379,7 +626,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_incorrect_answer_integer(self):
         # Given
-        self.test_case_data = [{"expected_input": "1\n2",
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                                "expected_input": "1\n2",
                                 "expected_output": "3",
                                 "weight": 0.0
                                 }]
@@ -389,14 +637,22 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 print(a-b)
                              """
                              )
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'partial_grading': False
-                  }
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'partial_grading': False
+        #           }
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertFalse(result.get('success'))
@@ -404,7 +660,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_file_based_answer(self):
         # Given
-        self.test_case_data = [{"expected_input": "",
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                                "expected_input": "",
                                 "expected_output": "2",
                                 "weight": 0.0
                                 }]
@@ -416,15 +673,23 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 print(a[0])
                              """
                              )
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths,
-                  'partial_grading': False
-                  }
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': self.test_case_data,
+        #           'file_paths': self.file_paths,
+        #           'partial_grading': False
+        #           }
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertEqual(result.get('error'), "Correct answer\n")
@@ -432,7 +697,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_infinite_loop(self):
         # Given
-        test_case_data = [{"expected_input": "1\n2",
+        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+                            "expected_input": "1\n2",
                             "expected_output": "3",
                             "weight": 0.0
                             }]
@@ -440,552 +706,28 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                             "You probably have an infinite loop in"
                             " your code.").format(SERVER_TIMEOUT)
         user_answer = "while True:\n\tpass"
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': test_case_data,
-                  'partial_grading': False
-                  }
+        # kwargs = {'user_answer': user_answer,
+        #           'test_case_data': test_case_data,
+        #           'partial_grading': False
+        #           }
+
+        kwargs = {'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': self.test_case_data
+                }
 
         # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
+        evaluator = CodeEvaluator(self.in_dir)
+        result = evaluator.evaluate(kwargs)
 
         # Then
         self.assertEqual(result.get('error'), timeout_msg)
         self.assertFalse(result.get('success'))
 
-# class PythonAssertionEvaluationTestCases(unittest.TestCase):
-#     def setUp(self):
-#         with open('/tmp/test.txt', 'wb') as f:
-#             f.write('2'.encode('ascii'))
-#         tmp_in_dir_path = tempfile.mkdtemp()
-#         self.in_dir = tmp_in_dir_path
-#         self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0},
-#                                {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0},
-#                                {"test_case":  'assert(add(-1,-2)==-3)', 'weight': 0.0},
-#                                ]
-#         self.timeout_msg = ("Code took more than {0} seconds to run. "
-#                             "You probably have an infinite loop in"
-#                             " your code.").format(SERVER_TIMEOUT)
-#         self.file_paths = None
-
-#     def tearDown(self):
-#         os.remove('/tmp/test.txt')
-#         shutil.rmtree(self.in_dir)
-
-#     def test_correct_answer(self):
-#         # Given
-#         user_answer = "def add(a,b):\n\treturn a + b"
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertTrue(result.get('success'))
-#         self.assertIn("Correct answer", result.get('error'))
-
-#     def test_incorrect_answer(self):
-#         # Given
-#         user_answer = "def add(a,b):\n\treturn a - b"
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertFalse(result.get('success'))
-#         self.assertIn('AssertionError  in: assert(add(1,2)==3)',
-#                         result.get('error')
-#                       )
-#         self.assertIn('AssertionError  in: assert(add(-1,2)==1)',
-#                         result.get('error')
-#                       )
-#         self.assertIn('AssertionError  in: assert(add(-1,-2)==-3)',
-#                         result.get('error')
-#                       )
-
-#     def test_partial_incorrect_answer(self):
-#         # Given
-#         user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
-#         test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0},
-#                                {"test_case":  'assert(add(-1,-2)==-3)', 'weight': 1.0},
-#                                {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0}
-#                                ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': True
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertFalse(result.get('success'))
-#         self.assertEqual(result.get('weight'), 2.0)
-#         self.assertIn('AssertionError  in: assert(add(-1,2)==1)',
-#                         result.get('error')
-#                       )
-#         self.assertIn('AssertionError  in: assert(add(-1,-2)==-3)',
-#                         result.get('error')
-#                       )
-
-#     def test_infinite_loop(self):
-#         # Given
-#         user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertFalse(result.get('success'))
-#         self.assertEqual(result.get('error'), self.timeout_msg)
-
-#     def test_syntax_error(self):
-#         # Given
-#         user_answer = dedent("""
-#         def add(a, b);
-#             return a + b
-#         """)
-#         syntax_error_msg = ["Traceback",
-#                             "call",
-#                             "File",
-#                             "line",
-#                             "<string>",
-#                             "SyntaxError",
-#                             "invalid syntax"
-#                             ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(5, len(err))
-#         for msg in syntax_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_indent_error(self):
-#         # Given
-#         user_answer = dedent("""
-#         def add(a, b):
-#         return a + b
-#         """)
-#         indent_error_msg = ["Traceback", "call",
-#                             "File",
-#                             "line",
-#                             "<string>",
-#                             "IndentationError",
-#                             "indented block"
-#                             ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(5, len(err))
-#         for msg in indent_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_name_error(self):
-#         # Given
-#         user_answer = ""
-#         name_error_msg = ["Traceback",
-#                           "call",
-#                           "NameError",
-#                           "name",
-#                           "defined"
-#                           ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(3, len(err))
-#         for msg in name_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_recursion_error(self):
-#         # Given
-#         user_answer = dedent("""
-#         def add(a, b):
-#             return add(3, 3)
-#         """)
-#         recursion_error_msg = ["Traceback",
-#                                "call",
-#                                "maximum recursion depth exceeded"
-#                                ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         for msg in recursion_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_type_error(self):
-#         # Given
-#         user_answer = dedent("""
-#         def add(a):
-#             return a + b
-#         """)
-#         type_error_msg = ["Traceback",
-#                           "call",
-#                           "TypeError",
-#                           "argument"
-#                           ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(3, len(err))
-#         for msg in type_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_value_error(self):
-#         # Given
-#         user_answer = dedent("""
-#         def add(a, b):
-#             c = 'a'
-#             return int(a) + int(b) + int(c)
-#         """)
-#         value_error_msg = ["Traceback",
-#                            "call",
-#                            "ValueError",
-#                            "invalid literal",
-#                            "base"
-#                            ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(4, len(err))
-#         for msg in value_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-#     def test_file_based_assert(self):
-#         # Given
-#         self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}]
-#         self.file_paths = [('/tmp/test.txt', False)]
-#         user_answer = dedent("""
-#             def ans():
-#                 with open("test.txt") as f:
-#                     return f.read()[0]
-#             """)
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertIn("Correct answer", result.get('error'))
-#         self.assertTrue(result.get('success'))
-
-#     def test_single_testcase_error(self):
-#         # Given
-#         """ Tests the user answer with just an incorrect test case """
-
-#         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
-#         test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False',
-#                              "weight": 0.0
-#                             }
-#                           ]
-#         syntax_error_msg = ["Traceback",
-#                             "call",
-#                             "File",
-#                             "line",
-#                             "<string>",
-#                             "SyntaxError",
-#                             "invalid syntax"
-#                             ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(5, len(err))
-#         for msg in syntax_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-
-#     def test_multiple_testcase_error(self):
-#         """ Tests the user answer with an correct test case
-#          first and then with an incorrect test case """
-#         # Given
-#         user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
-#         test_case_data = [{"test_case": 'assert(palindrome("abba")==True)',
-#                              "weight": 0.0
-#                             },
-#                           {"test_case": 's="abbb"\nassert palindrome(S)==False',
-#                              "weight": 0.0
-#                             }
-#                           ]
-#         name_error_msg = ["Traceback",
-#                           "call",
-#                           "File",
-#                           "line",
-#                           "<string>",
-#                           "NameError",
-#                           "name 'S' is not defined"
-#                           ]
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonAssertionEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-#         err = result.get("error").splitlines()
-
-#         # Then
-#         self.assertFalse(result.get("success"))
-#         self.assertEqual(3, len(err))
-#         for msg in name_error_msg:
-#             self.assertIn(msg, result.get("error"))
-
-
-# class PythonStdIOEvaluationTestCases(unittest.TestCase):
-#     def setUp(self):
-#         with open('/tmp/test.txt', 'wb') as f:
-#             f.write('2'.encode('ascii'))
-#         self.file_paths = None
-
-#     def test_correct_answer_integer(self):
-#         # Given
-#         self.test_case_data = [{"expected_input": "1\n2",
-#                                 "expected_output": "3",
-#                                 "weight": 0.0
-#                                 }]
-#         user_answer = dedent("""
-#                                 a = int(input())
-#                                 b = int(input())
-#                                 print(a+b)
-#                              """
-#                              )
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertTrue(result.get('success'))
-#         self.assertIn("Correct answer", result.get('error'))
-
-#     def test_correct_answer_list(self):
-#         # Given
-#         self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
-#                                 "expected_output": "[1, 2, 3, 5, 6, 7]",
-#                                 "weight": 0.0
-#                                 }]
-#         user_answer = dedent("""
-#                                 from six.moves import input
-#                                 input_a = input()
-#                                 input_b = input()
-#                                 a = [int(i) for i in input_a.split(',')]
-#                                 b = [int(i) for i in input_b.split(',')]
-#                                 print(a+b)
-#                              """
-#                              )
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertTrue(result.get('success'))
-#         self.assertIn("Correct answer", result.get('error'))
-
-#     def test_correct_answer_string(self):
-#         # Given
-#         self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
-#                                 "expected_output": "2",
-#                                 "weight": 0.0
-#                                 }]
-#         user_answer = dedent("""
-#                                 from six.moves import input
-#                                 a = str(input())
-#                                 b = str(input())
-#                                 print(a.count(b))
-#                              """
-#                              )
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertTrue(result.get('success'))
-#         self.assertIn("Correct answer", result.get('error'))
-
-#     def test_incorrect_answer_integer(self):
-#         # Given
-#         self.test_case_data = [{"expected_input": "1\n2",
-#                                 "expected_output": "3",
-#                                 "weight": 0.0
-#                                 }]
-#         user_answer = dedent("""
-#                                 a = int(input())
-#                                 b = int(input())
-#                                 print(a-b)
-#                              """
-#                              )
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertFalse(result.get('success'))
-#         self.assertIn("Incorrect answer", result.get('error'))
-
-#     def test_file_based_answer(self):
-#         # Given
-#         self.test_case_data = [{"expected_input": "",
-#                                 "expected_output": "2",
-#                                 "weight": 0.0
-#                                 }]
-#         self.file_paths = [('/tmp/test.txt', False)]
-
-#         user_answer = dedent("""
-#                             with open("test.txt") as f:
-#                                 a = f.read()
-#                                 print(a[0])
-#                              """
-#                              )
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': self.test_case_data,
-#                   'file_paths': self.file_paths,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertEqual(result.get('error'), "Correct answer\n")
-#         self.assertTrue(result.get('success'))
-
-#     def test_infinite_loop(self):
-#         # Given
-#         test_case_data = [{"expected_input": "1\n2",
-#                             "expected_output": "3",
-#                             "weight": 0.0
-#                             }]
-#         timeout_msg = ("Code took more than {0} seconds to run. "
-#                             "You probably have an infinite loop in"
-#                             " your code.").format(SERVER_TIMEOUT)
-#         user_answer = "while True:\n\tpass"
-#         kwargs = {'user_answer': user_answer,
-#                   'test_case_data': test_case_data,
-#                   'partial_grading': False
-#                   }
-
-#         # When
-#         evaluator = PythonStdioEvaluator()
-#         result = evaluator.evaluate(**kwargs)
-
-#         # Then
-#         self.assertEqual(result.get('error'), timeout_msg)
-#         self.assertFalse(result.get('success'))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
index 5f1b29e..3e172ec 100644
--- a/yaksh/python_assertion_evaluator.py
+++ b/yaksh/python_assertion_evaluator.py
@@ -27,10 +27,10 @@ class PythonAssertionEvaluator(object):
         self.test_case = test_case_data.get('test_case')
         self.weight = test_case_data.get('weight')
 
-    def __del__(self):
-        # Delete the created file.
-        if self.files:
-            delete_files(self.files)
+    # def __del__(self):
+    #     # Delete the created file.
+    #     if self.files:
+    #         delete_files(self.files)
 
 
     # def setup(self):
diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py
index 1506685..b618a0b 100644
--- a/yaksh/python_stdio_evaluator.py
+++ b/yaksh/python_stdio_evaluator.py
@@ -31,24 +31,36 @@ def redirect_stdout():
 class PythonStdioEvaluator(CodeEvaluator):
     """Tests the Python code obtained from Code Server"""
 
-    def setup(self):
-        super(PythonStdioEvaluator, self).setup()
+    # def setup(self):
+    #     super(PythonStdioEvaluator, self).setup()
+    #     self.files = []
+
+    # def teardown(self):
+    #     # Delete the created file.
+    #     if self.files:
+    #         delete_files(self.files)
+    #     super(PythonStdioEvaluator, self).teardown()
+    def __init__(self, metadata, test_case_data):
         self.files = []
 
-    def teardown(self):
-        # Delete the created file.
-        if self.files:
-            delete_files(self.files)
-        super(PythonStdioEvaluator, self).teardown()
+        # Set metadata values
+        self.user_answer = metadata.get('user_answer')
+        self.file_paths = metadata.get('file_paths')
+        self.partial_grading = metadata.get('partial_grading')
+
+        # Set test case data values
+        self.expected_input = test_case_data.get('expected_input')
+        self.expected_output = test_case_data.get('expected_output')
+        self.weight = test_case_data.get('weight')        
 
 
-    def compile_code(self, user_answer, file_paths, expected_input, expected_output, weight):
-        if file_paths:
-            self.files = copy_files(file_paths)
-        submitted = compile(user_answer, '<string>', mode='exec')
-        if expected_input:
+    def compile_code(self): # user_answer, file_paths, expected_input, expected_output, weight):
+        if self.file_paths:
+            self.files = copy_files(self.file_paths)
+        submitted = compile(self.user_answer, '<string>', mode='exec')
+        if self.expected_input:
             input_buffer = StringIO()
-            input_buffer.write(expected_input)
+            input_buffer.write(self.expected_input)
             input_buffer.seek(0)
             sys.stdin = input_buffer
         with redirect_stdout() as output_buffer:
@@ -57,16 +69,15 @@ class PythonStdioEvaluator(CodeEvaluator):
         self.output_value = output_buffer.getvalue().rstrip("\n")
         return self.output_value
 
-    def check_code(self, user_answer, file_paths, partial_grading, expected_input,
-         expected_output, weight):
+    def check_code(self): # user_answer, file_paths, partial_grading, expected_input, expected_output, weight):
         success = False
         test_case_weight = 0.0
 
         tb = None
-        if self.output_value == expected_output:
+        if self.output_value == self.expected_output:
             success = True
             err = "Correct answer"
-            test_case_weight = weight
+            test_case_weight = self.weight
         else:
             success = False
             err = dedent("""
@@ -74,8 +85,8 @@ class PythonStdioEvaluator(CodeEvaluator):
                 Given input - {0}
                 Expected output - {1}
                 Your output - {2}
-                """.format(expected_input,
-                     expected_output,
+                """.format(self.expected_input,
+                     self.expected_output,
                      self.output_value
                     )
                 )