Fix test cases and corresponding changes in evaluators for Python 2/3 compatibility

5 files changed, 113 insertions, 95 deletions

diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 01e248f..7fdff48 100755
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 import unittest
 import os
+import shutil
+import tempfile
 from yaksh.bash_code_evaluator import BashCodeEvaluator
 from yaksh.bash_stdio_evaluator import BashStdioEvaluator
 from yaksh.settings import SERVER_TIMEOUT
@@ -9,15 +11,23 @@ from textwrap import dedent
 
 class BashAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
         self.test_case_data = [
             {"test_case": "bash_files/sample.sh,bash_files/sample.args"}
         ]
-        self.in_dir = os.getcwd()
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
         self.timeout_msg = ("Code took more than {0} seconds to run. "
             "You probably have an infinite loop in your"
             " code.").format(SERVER_TIMEOUT)
         self.file_paths = None
 
+    def tearDown(self):
+        os.remove('/tmp/test.txt')
+        shutil.rmtree(self.in_dir)
+
     def test_correct_answer(self):
         user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
             " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
@@ -56,7 +66,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
         self.assertEqual(result.get("error"), self.timeout_msg)
 
     def test_file_based_assert(self):
-        self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+        self.file_paths = [('/tmp/test.txt', False)]
         self.test_case_data = [
             {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"}
         ]
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index d57affa..b3551bf 100755
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 import unittest
 import os
+import shutil
+import tempfile
 from yaksh.cpp_code_evaluator import CppCodeEvaluator
 from yaksh.cpp_stdio_evaluator import CppStdioEvaluator
 from yaksh.settings import SERVER_TIMEOUT
@@ -9,13 +11,20 @@ from textwrap import dedent
 
 class CAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
         self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
-        self.in_dir = os.getcwd()
+        self.in_dir = tmp_in_dir_path
         self.timeout_msg = ("Code took more than {0} seconds to run. "
             "You probably have an infinite loop in your"
             " code.").format(SERVER_TIMEOUT)
         self.file_paths = None
 
+    def tearDown(self):
+        os.remove('/tmp/test.txt')
+        shutil.rmtree(self.in_dir)
+
     def test_correct_answer(self):
         user_answer = "int add(int a, int b)\n{return a+b;}"
         get_class = CppCodeEvaluator(self.in_dir)
@@ -35,9 +44,10 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
                     'file_paths': self.file_paths
                 }
         result = get_class.evaluate(**kwargs)
+        lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect:", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        self.assertTrue(lines_of_error > 1)
 
     def test_compilation_error(self):
         user_answer = "int add(int a, int b)\n{return a+b}"
@@ -62,7 +72,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
         self.assertEqual(result.get("error"), self.timeout_msg)
 
     def test_file_based_assert(self):
-        self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+        self.file_paths = [('/tmp/test.txt', False)]
         self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}]
         user_answer = dedent("""
             #include<stdio.h>
@@ -160,9 +170,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
                   'test_case_data': self.test_case_data
                   }
         result = get_class.evaluate(**kwargs)
+        lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        self.assertTrue(lines_of_error > 1)
 
     def test_error(self):
         user_answer = dedent("""
@@ -281,9 +292,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
                   'test_case_data': self.test_case_data
                   }
         result = get_class.evaluate(**kwargs)
+        lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        self.assertTrue(lines_of_error > 1)
 
     def test_cpp_error(self):
         user_answer = dedent("""
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index f94c6d1..a9708e8 100755
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 import unittest
 import os
+import shutil
+import tempfile
 from yaksh import code_evaluator as evaluator
 from yaksh.java_code_evaluator import JavaCodeEvaluator
 from yaksh.java_stdio_evaluator import JavaStdioEvaluator
@@ -10,10 +12,13 @@ from textwrap import dedent
 
 class JavaAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
         self.test_case_data = [
             {"test_case": "java_files/main_square.java"}
         ]
-        self.in_dir = os.getcwd()
+        self.in_dir = tmp_in_dir_path
         evaluator.SERVER_TIMEOUT = 9
         self.timeout_msg = ("Code took more than {0} seconds to run. "
             "You probably have an infinite loop in"
@@ -21,7 +26,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
         self.file_paths = None
 
     def tearDown(self):
-        evaluator.SERVER_TIMEOUT = 2
+        os.remove('/tmp/test.txt')
+        shutil.rmtree(self.in_dir)
 
     def test_correct_answer(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
@@ -43,8 +49,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
                 }
         result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get('success'))
-        self.assertIn("Incorrect:", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        lines_of_error = len(result.get('error').splitlines())
+        self.assertFalse(result.get('success'))
+        self.assertIn("Incorrect", result.get('error'))
+        self.assertTrue(lines_of_error > 1)
 
     def test_error(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
@@ -69,7 +77,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
         self.assertEqual(result.get("error"), self.timeout_msg)
 
     def test_file_based_assert(self):
-        self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+        self.file_paths = [("/tmp/test.txt", False)]
         self.test_case_data = [
             {"test_case": "java_files/read_file.java"}
         ]
@@ -169,9 +177,10 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
                   'test_case_data': self.test_case_data
                   }
         result = get_class.evaluate(**kwargs)
+        lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        self.assertTrue(lines_of_error > 1)
 
     def test_error(self):
 
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 6852136..b93aa13 100755
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,14 +1,23 @@
 from __future__ import absolute_import
+from six.moves import input
 import unittest
 import os
+import tempfile
+import shutil
+from textwrap import dedent
+
+# Local import
 from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
 from yaksh.python_stdio_evaluator import PythonStdioEvaluator
 from yaksh.settings import SERVER_TIMEOUT
-from textwrap import dedent
 
 
 class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
         self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
                                {"test_case": 'assert(add(-1,2)==1)'},
                                {"test_case":  'assert(add(-1,-2)==-3)'},
@@ -18,6 +27,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
                             " your code.").format(SERVER_TIMEOUT)
         self.file_paths = None
 
+    def tearDown(self):
+        os.remove('/tmp/test.txt')
+        shutil.rmtree(self.in_dir)
+
     def test_correct_answer(self):
         # Given
         user_answer = "def add(a,b):\n\treturn a + b"
@@ -160,7 +173,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         """)
         recursion_error_msg = ["Traceback",
                                "call",
-                               "RuntimeError",
                                "maximum recursion depth exceeded"
                                ]
         kwargs = {'user_answer': user_answer,
@@ -175,7 +187,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
 
         # Then
         self.assertFalse(result.get("success"))
-        self.assertEqual(969, len(err))
         for msg in recursion_error_msg:
             self.assertIn(msg, result.get("error"))
 
@@ -188,7 +199,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
         type_error_msg = ["Traceback",
                           "call",
                           "TypeError",
-                          "exactly",
                           "argument"
                           ]
         kwargs = {'user_answer': user_answer,
@@ -239,7 +249,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def test_file_based_assert(self):
         # Given
         self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
-        self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+        self.file_paths = [('/tmp/test.txt', False)]
         user_answer = dedent("""
             def ans():
                 with open("test.txt") as f:
@@ -323,65 +333,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
             self.assertIn(msg, result.get("error"))
 
 
-class PythonStdoutEvaluationTestCases(unittest.TestCase):
-    def setUp(self):
-        self.test_case_data = [{"expected_input": None,
-                                "expected_output": "0 1 1 2 3"
-                                }]
-
-        self.timeout_msg = ("Code took more than {0} seconds to run. "
-                            "You probably have an infinite loop"
-                            " in your code.").format(SERVER_TIMEOUT)
-
-    def test_correct_answer(self):
-        # Given
-        user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  }
-
-        # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
-
-        # Then
-        self.assertEqual(result.get('error'), "Correct answer")
-        self.assertTrue(result.get('success'))
-
-    def test_incorrect_answer(self):
-        # Given
-        user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data,
-                  }
-
-        # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
-
-        # Then
-        self.assertFalse(result.get('success'))
-        self.assertIn("Incorrect answer", result.get('error'))
-
-    def test_infinite_loop(self):
-        # Given
-        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
-        kwargs = {'user_answer': user_answer,
-                  'test_case_data': self.test_case_data
-                  }
-
-        # When
-        evaluator = PythonStdioEvaluator()
-        result = evaluator.evaluate(**kwargs)
-
-        # Then
-        self.assertEqual(result.get('error'), self.timeout_msg)
-        self.assertFalse(result.get('success'))
-
-
 class PythonStdIOEvaluationTestCases(unittest.TestCase):
-
     def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
         self.file_paths = None
 
     def test_correct_answer_integer(self):
@@ -390,9 +345,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 "expected_output": "3"
                                 }]
         user_answer = dedent("""
-                                a = input()
-                                b = input()
-                                print a+b
+                                a = int(input())
+                                b = int(input())
+                                print(a+b)
                              """
                              )
         kwargs = {'user_answer': user_answer,
@@ -409,13 +364,16 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_correct_answer_list(self):
         # Given
-        self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
+        self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
                                 "expected_output": "[1, 2, 3, 5, 6, 7]"
                                 }]
         user_answer = dedent("""
-                                a = input()
-                                b = input()
-                                print a+b
+                                from six.moves import input
+                                input_a = input()
+                                input_b = input()
+                                a = [int(i) for i in input_a.split(',')]
+                                b = [int(i) for i in input_b.split(',')]
+                                print(a+b)
                              """
                              )
         kwargs = {'user_answer': user_answer,
@@ -432,14 +390,14 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
 
     def test_correct_answer_string(self):
         # Given
-        self.test_case_data = [{"expected_input": """the quick brown fox jumps\
-                                                     over the lazy dog\nthe""",
+        self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
                                 "expected_output": "2"
                                 }]
         user_answer = dedent("""
-                                a = raw_input()
-                                b = raw_input()
-                                print (a.count(b))
+                                from six.moves import input
+                                a = str(input())
+                                b = str(input())
+                                print(a.count(b))
                              """
                              )
         kwargs = {'user_answer': user_answer,
@@ -460,9 +418,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
                                 "expected_output": "3"
                                 }]
         user_answer = dedent("""
-                                a = input()
-                                b = input()
-                                print a-b
+                                a = int(input())
+                                b = int(input())
+                                print(a-b)
                              """
                              )
         kwargs = {'user_answer': user_answer,
@@ -480,12 +438,12 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
     def test_file_based_answer(self):
         # Given
         self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
-        self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+        self.file_paths = [('/tmp/test.txt', False)]
 
         user_answer = dedent("""
                             with open("test.txt") as f:
                                 a = f.read()
-                                print a[0]
+                                print(a[0])
                              """
                              )
         kwargs = {'user_answer': user_answer,
@@ -501,5 +459,26 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
         self.assertEqual(result.get('error'), "Correct answer")
         self.assertTrue(result.get('success'))
 
+    def test_infinite_loop(self):
+        # Given
+        test_case_data = [{"expected_input": "1\n2",
+                                "expected_output": "3"
+                                }]
+        timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in"
+                            " your code.").format(SERVER_TIMEOUT)
+        user_answer = "while True:\n\tpass"
+        kwargs = {'user_answer': user_answer,
+                  'test_case_data': test_case_data
+                  }
+
+        # When
+        evaluator = PythonStdioEvaluator()
+        result = evaluator.evaluate(**kwargs)
+
+        # Then
+        self.assertEqual(result.get('error'), timeout_msg)
+        self.assertFalse(result.get('success'))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index f1b29dd..0bb7bfd 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -1,19 +1,26 @@
 from __future__ import absolute_import
 import unittest
 import os
+import shutil
+import tempfile
+
 from yaksh import code_evaluator as evaluator
 from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 
 class ScilabEvaluationTestCases(unittest.TestCase):
     def setUp(self):
+        tmp_in_dir_path = tempfile.mkdtemp()
         self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
-        self.in_dir = os.getcwd()
+        self.in_dir = tmp_in_dir_path
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop" 
                             " in your code.").format(SERVER_TIMEOUT)
         self.file_paths = None
 
+    def tearDown(self):
+        shutil.rmtree(self.in_dir)
+
     def test_correct_answer(self):
         user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
                         "\n\tc=a+b;\nendfunction")
@@ -36,7 +43,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
                 }
         result = get_class.evaluate(**kwargs) 
         self.assertFalse(result.get("success"))
-        self.assertTrue("error" in result.get("error"))
+        self.assertTrue('error' in result.get("error"))
 
 
     def test_incorrect_answer(self):
@@ -48,9 +55,10 @@ class ScilabEvaluationTestCases(unittest.TestCase):
                     'file_paths': self.file_paths
                 }
         result = get_class.evaluate(**kwargs)
+        lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Message", result.get('error'))
-        self.assertTrue(result.get('error').splitlines > 1)
+        self.assertTrue(lines_of_error > 1)
 
     def test_infinite_loop(self):
         user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"