Merge pull request #138 from maheshgudi/python_assertion_bug_fix

Python assertion bug fix

3 files changed, 271 insertions, 127 deletions

diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index e139b55..8a9b7a6 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -101,6 +101,7 @@ class CodeEvaluator(object):
         # Do whatever testing needed.
         try:
             for test_case in test_case_data:
+                success = False
                 self.compile_code(user_answer, file_paths, **test_case)
                 success, err = self.check_code(user_answer, file_paths, **test_case)
                 if not success:
@@ -116,8 +117,6 @@ class CodeEvaluator(object):
             tb_list = traceback.format_exception(exc_type, exc_value, exc_tb)
             if len(tb_list) > 2:
                 del tb_list[1:3]
-            else:
-                pass
             err = "Error: {0}".format("".join(tb_list))
         finally:
             # Set back any original signal handler.
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 3c07907..2a109d5 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -9,164 +9,205 @@ from textwrap import dedent
 class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
         self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
-            {"test_case": 'assert(add(-1,2)==1)'},
-            {"test_case":  'assert(add(-1,-2)==-3)'},
-        ]
+                               {"test_case": 'assert(add(-1,2)==1)'},
+                               {"test_case":  'assert(add(-1,-2)==-3)'},
+                               ]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
-            "You probably have an infinite loop in"
-            " your code.").format(SERVER_TIMEOUT)
+                            "You probably have an infinite loop in"
+                            " your code.").format(SERVER_TIMEOUT)
         self.file_paths = None
 
     def test_correct_answer(self):
+        # Given
         user_answer = "def add(a,b):\n\treturn a + b"
-        get_class = PythonAssertionEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
                   'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertTrue(result.get('success'))
         self.assertEqual(result.get('error'), "Correct answer")
 
     def test_incorrect_answer(self):
+        # Given
         user_answer = "def add(a,b):\n\treturn a - b"
-        get_class = PythonAssertionEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
                   'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertFalse(result.get('success'))
         self.assertEqual(result.get('error'),
-            "AssertionError  in: assert(add(1,2)==3)"
-        )
+                         "AssertionError  in: assert(add(1,2)==3)"
+                         )
 
     def test_infinite_loop(self):
+        # Given
         user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
-        get_class = PythonAssertionEvaluator()
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertFalse(result.get('success'))
         self.assertEqual(result.get('error'), self.timeout_msg)
 
-
+    def test_syntax_error(self):
+        # Given
         user_answer = dedent("""
         def add(a, b);
             return a + b
         """)
-        syntax_error_msg = ["Traceback", 
-            "call", 
-            "File", 
-            "line", 
-            "<string>",
-            "SyntaxError", 
-            "invalid syntax"
-        ]
-        get_class = PythonAssertionEvaluator()
+        syntax_error_msg = ["Traceback",
+                            "call",
+                            "File",
+                            "line",
+                            "<string>",
+                            "SyntaxError",
+                            "invalid syntax"
+                            ]
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
         for msg in syntax_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_indent_error(self):
+        # Given
         user_answer = dedent("""
         def add(a, b):
         return a + b
         """)
-        indent_error_msg = ["Traceback", "call", 
-            "File",
-            "line",
-            "<string>",
-            "IndentationError",
-            "indented block"
-        ]
-        get_class = PythonAssertionEvaluator()
+        indent_error_msg = ["Traceback", "call",
+                            "File",
+                            "line",
+                            "<string>",
+                            "IndentationError",
+                            "indented block"
+                            ]
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(5, len(err))
         for msg in indent_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_name_error(self):
+        # Given
         user_answer = ""
-        name_error_msg = ["Traceback", 
-            "call",
-            "NameError",
-            "name",
-            "defined"
-        ]
-        get_class = PythonAssertionEvaluator()
+        name_error_msg = ["Traceback",
+                          "call",
+                          "NameError",
+                          "name",
+                          "defined"
+                          ]
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(3, len(err))
         for msg in name_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_recursion_error(self):
+        # Given
         user_answer = dedent("""
         def add(a, b):
             return add(3, 3)
         """)
         recursion_error_msg = ["Traceback",
-            "call", 
-            "RuntimeError",
-            "maximum recursion depth exceeded"
-        ]
-        get_class = PythonAssertionEvaluator()
+                               "call",
+                               "RuntimeError",
+                               "maximum recursion depth exceeded"
+                               ]
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(969, len(err))
         for msg in recursion_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_type_error(self):
+        # Given
         user_answer = dedent("""
         def add(a):
             return a + b
         """)
-        type_error_msg = ["Traceback", 
-            "call", 
-            "TypeError",
-            "exactly",
-            "argument"
-        ]
-        get_class = PythonAssertionEvaluator()
+        type_error_msg = ["Traceback",
+                          "call",
+                          "TypeError",
+                          "exactly",
+                          "argument"
+                          ]
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(3, len(err))
         for msg in type_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_value_error(self):
+        # Given
         user_answer = dedent("""
         def add(a, b):
             c = 'a'
@@ -178,19 +219,24 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                            "invalid literal",
                            "base"
                            ]
-        get_class = PythonAssertionEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
                   'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
         err = result.get("error").splitlines()
+
+        # Then
         self.assertFalse(result.get("success"))
         self.assertEqual(4, len(err))
         for msg in value_error_msg:
             self.assertIn(msg, result.get("error"))
 
     def test_file_based_assert(self):
+        # Given
         self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
         self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
         user_answer = dedent("""
@@ -198,15 +244,84 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                 with open("test.txt") as f:
                     return f.read()[0]
             """)
-        get_class = PythonAssertionEvaluator()
         kwargs = {'user_answer': user_answer,
-            'test_case_data': self.test_case_data,
-            'file_paths': self.file_paths
-        }
-        result = get_class.evaluate(**kwargs)
+                  'test_case_data': self.test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertEqual(result.get('error'), "Correct answer")
         self.assertTrue(result.get('success'))
 
+    def test_single_testcase_error(self):
+        # Given
+        """ Tests the user answer with just an incorrect test case """
+
+        user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
+        test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'}
+                          ]
+        syntax_error_msg = ["Traceback",
+                            "call",
+                            "File",
+                            "line",
+                            "<string>",
+                            "SyntaxError",
+                            "invalid syntax"
+                            ]
+        kwargs = {'user_answer': user_answer,
+                  'test_case_data': test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+        err = result.get("error").splitlines()
+
+        # Then
+        self.assertFalse(result.get("success"))
+        self.assertEqual(5, len(err))
+        for msg in syntax_error_msg:
+            self.assertIn(msg, result.get("error"))
+
+
+    def test_multiple_testcase_error(self):
+        """ Tests the user answer with an correct test case
+         first and then with an incorrect test case """
+        # Given
+        user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
+        test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'},
+                          {"test_case": 's="abbb"\nassert palindrome(S)==False'}
+                          ]
+        name_error_msg = ["Traceback",
+                          "call",
+                          "File",
+                          "line",
+                          "<string>",
+                          "NameError",
+                          "name 'S' is not defined"
+                          ]
+        kwargs = {'user_answer': user_answer,
+                  'test_case_data': test_case_data,
+                  'file_paths': self.file_paths
+                  }
+
+        # When
+        evaluator = PythonAssertionEvaluator()
+        result = evaluator.evaluate(**kwargs)
+        err = result.get("error").splitlines()
+
+        # Then
+        self.assertFalse(result.get("success"))
+        self.assertEqual(3, len(err))
+        for msg in name_error_msg:
+            self.assertIn(msg, result.get("error"))
+
+
 class PythonStdoutEvaluationTestCases(unittest.TestCase):
     def setUp(self):
         self.test_case_data = [{"expected_input": None,
@@ -216,86 +331,106 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop"
                             " in your code.").format(SERVER_TIMEOUT)
-        self.file_paths = None
 
     def test_correct_answer(self):
+        # Given
         user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
-        get_class = PythonStdioEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertEqual(result.get('error'), "Correct Answer")
         self.assertTrue(result.get('success'))
 
     def test_incorrect_answer(self):
+        # Given
         user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
-        get_class = PythonStdioEvaluator()
-        kwargs = {'user_answer': user_answer, 
+        kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
-                  'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect Answer", result.get('error'))
 
     def test_infinite_loop(self):
+        # Given
         user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
-        get_class = PythonStdioEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data
                   }
-    
-class PythonStdIOEvaluator(unittest.TestCase):
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
+        self.assertEqual(result.get('error'), self.timeout_msg)
+        self.assertFalse(result.get('success'))
+
+
+class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def setUp(self):
-        self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop"
-                            " in your code.").format(SERVER_TIMEOUT)
+        self.file_paths = None
 
-    def test_add_two_integers_correct(self):
+    def test_correct_answer_integer(self):
+        # Given
         self.test_case_data = [{"expected_input": "1\n2",
                                 "expected_output": "3"
                                 }]
         user_answer = dedent("""
-                                a = int(raw_input())
-                                b = int(raw_input())
+                                a = input()
+                                b = input()
                                 print a+b
                              """
                              )
-
-        get_class = PythonStdioEvaluator()
-
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertTrue(result.get('success'))
         self.assertIn("Correct Answer", result.get('error'))
 
-    def test_add_two_lists(self):
+    def test_correct_answer_list(self):
+        # Given
         self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
                                 "expected_output": "[1, 2, 3, 5, 6, 7]"
                                 }]
         user_answer = dedent("""
-                                from ast import literal_eval
-                                a = literal_eval(raw_input())
-                                b = literal_eval(raw_input())
+                                a = input()
+                                b = input()
                                 print a+b
                              """
                              )
-
-        get_class = PythonStdioEvaluator()
-
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertTrue(result.get('success'))
         self.assertIn("Correct Answer", result.get('error'))
 
-    def test_find_string_in_sentence(self):
+    def test_correct_answer_string(self):
+        # Given
         self.test_case_data = [{"expected_input": """the quick brown fox jumps\
                                                      over the lazy dog\nthe""",
                                 "expected_output": "2"
@@ -306,37 +441,43 @@ class PythonStdIOEvaluator(unittest.TestCase):
                                 print (a.count(b))
                              """
                              )
-
-        get_class = PythonStdioEvaluator()
-
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertTrue(result.get('success'))
         self.assertIn("Correct Answer", result.get('error'))
 
-    def test_add_two_integers_incorrect(self):
+    def test_incorrect_answer_integer(self):
+        # Given
         self.test_case_data = [{"expected_input": "1\n2",
                                 "expected_output": "3"
                                 }]
         user_answer = dedent("""
-                                a = int(raw_input())
-                                b = int(raw_input())
+                                a = input()
+                                b = input()
                                 print a-b
                              """
                              )
-
-        get_class = PythonStdioEvaluator()
-
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect Answer", result.get('error'))
 
     def test_file_based_answer(self):
+        # Given
         self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
         self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
 
@@ -346,12 +487,16 @@ class PythonStdIOEvaluator(unittest.TestCase):
                                 print a[0]
                              """
                              )
-        get_class = PythonStdioEvaluator()
         kwargs = {'user_answer': user_answer,
                   'test_case_data': self.test_case_data,
                   'file_paths': self.file_paths
                   }
-        result = get_class.evaluate(**kwargs)
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
         self.assertEqual(result.get('error'), "Correct Answer")
         self.assertTrue(result.get('success'))
 
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
index 04a4e69..1b66fd2 100644
--- a/yaksh/python_assertion_evaluator.py
+++ b/yaksh/python_assertion_evaluator.py
@@ -47,8 +47,8 @@ class PythonAssertionEvaluator(CodeEvaluator):
             fname, lineno, func, text = info[-1]
             text = str(test_case).splitlines()[lineno-1]
             err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
-        except TimeoutException:
-            raise
+        except Exception:
+            raise  # Exception will be caught in CodeEvaluator.
         else:
             success = True
             err = 'Correct answer'