From 6b6e58b06bd49e36edd87a027c08d223571a0c0b Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 27 Sep 2016 15:08:18 +0530 Subject: Fix test cases and corresponding changes in evaluators for Python 2/3 compatibility --- yaksh/evaluator_tests/test_bash_evaluation.py | 14 ++- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 22 +++- yaksh/evaluator_tests/test_java_evaluation.py | 21 ++-- yaksh/evaluator_tests/test_python_evaluation.py | 137 ++++++++++-------------- yaksh/evaluator_tests/test_scilab_evaluation.py | 14 ++- 5 files changed, 113 insertions(+), 95 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 01e248f..7fdff48 100755 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import unittest import os +import shutil +import tempfile from yaksh.bash_code_evaluator import BashCodeEvaluator from yaksh.bash_stdio_evaluator import BashStdioEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -9,15 +11,23 @@ from textwrap import dedent class BashAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args"} ] - self.in_dir = os.getcwd() + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) self.file_paths = None + def tearDown(self): + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) + def test_correct_answer(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" @@ -56,7 +66,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): self.assertEqual(result.get("error"), self.timeout_msg) def test_file_based_assert(self): - self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"} ] diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d57affa..b3551bf 100755 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import unittest import os +import shutil +import tempfile from yaksh.cpp_code_evaluator import CppCodeEvaluator from yaksh.cpp_stdio_evaluator import CppStdioEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -9,13 +11,20 @@ from textwrap import dedent class CAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] - self.in_dir = os.getcwd() + self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) self.file_paths = None + def tearDown(self): + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) + def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) @@ -35,9 +44,10 @@ class CAssertionEvaluationTestCases(unittest.TestCase): 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect:", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + self.assertTrue(lines_of_error > 1) def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" @@ -62,7 +72,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): self.assertEqual(result.get("error"), self.timeout_msg) def test_file_based_assert(self): - self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}] user_answer = dedent(""" #include @@ -160,9 +170,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + self.assertTrue(lines_of_error > 1) def test_error(self): user_answer = dedent(""" @@ -281,9 +292,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + self.assertTrue(lines_of_error > 1) def test_cpp_error(self): user_answer = dedent(""" diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index f94c6d1..a9708e8 100755 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -1,6 +1,8 @@ from __future__ import absolute_import import unittest import os +import shutil +import tempfile from yaksh import code_evaluator as evaluator from yaksh.java_code_evaluator import JavaCodeEvaluator from yaksh.java_stdio_evaluator import JavaStdioEvaluator @@ -10,10 +12,13 @@ from textwrap import dedent class JavaAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ {"test_case": "java_files/main_square.java"} ] - self.in_dir = os.getcwd() + self.in_dir = tmp_in_dir_path evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -21,7 +26,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): self.file_paths = None def tearDown(self): - evaluator.SERVER_TIMEOUT = 2 + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) def test_correct_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" @@ -43,8 +49,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) - self.assertIn("Incorrect:", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + lines_of_error = len(result.get('error').splitlines()) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect", result.get('error')) + self.assertTrue(lines_of_error > 1) def test_error(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" @@ -69,7 +77,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): self.assertEqual(result.get("error"), self.timeout_msg) def test_file_based_assert(self): - self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [ {"test_case": "java_files/read_file.java"} ] @@ -169,9 +177,10 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + self.assertTrue(lines_of_error > 1) def test_error(self): diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 6852136..b93aa13 100755 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -1,14 +1,23 @@ from __future__ import absolute_import +from six.moves import input import unittest import os +import tempfile +import shutil +from textwrap import dedent + +# Local import from yaksh.python_assertion_evaluator import PythonAssertionEvaluator from yaksh.python_stdio_evaluator import PythonStdioEvaluator from yaksh.settings import SERVER_TIMEOUT -from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, {"test_case": 'assert(add(-1,2)==1)'}, {"test_case": 'assert(add(-1,-2)==-3)'}, @@ -18,6 +27,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): " your code.").format(SERVER_TIMEOUT) self.file_paths = None + def tearDown(self): + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) + def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" @@ -160,7 +173,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """) recursion_error_msg = ["Traceback", "call", - "RuntimeError", "maximum recursion depth exceeded" ] kwargs = {'user_answer': user_answer, @@ -175,7 +187,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): # Then self.assertFalse(result.get("success")) - self.assertEqual(969, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error")) @@ -188,7 +199,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): type_error_msg = ["Traceback", "call", "TypeError", - "exactly", "argument" ] kwargs = {'user_answer': user_answer, @@ -239,7 +249,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): # Given self.test_case_data = [{"test_case": "assert(ans()=='2')"}] - self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: @@ -323,65 +333,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): self.assertIn(msg, result.get("error")) -class PythonStdoutEvaluationTestCases(unittest.TestCase): - def setUp(self): - self.test_case_data = [{"expected_input": None, - "expected_output": "0 1 1 2 3" - }] - - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) - - def test_correct_answer(self): - # Given - user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertEqual(result.get('error'), "Correct answer") - self.assertTrue(result.get('success')) - - def test_incorrect_answer(self): - # Given - user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertFalse(result.get('success')) - self.assertIn("Incorrect answer", result.get('error')) - - def test_infinite_loop(self): - # Given - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } - - # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) - - # Then - self.assertEqual(result.get('error'), self.timeout_msg) - self.assertFalse(result.get('success')) - - class PythonStdIOEvaluationTestCases(unittest.TestCase): - def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) self.file_paths = None def test_correct_answer_integer(self): @@ -390,9 +345,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): "expected_output": "3" }] user_answer = dedent(""" - a = input() - b = input() - print a+b + a = int(input()) + b = int(input()) + print(a+b) """ ) kwargs = {'user_answer': user_answer, @@ -409,13 +364,16 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given - self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]", + self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]" }] user_answer = dedent(""" - a = input() - b = input() - print a+b + from six.moves import input + input_a = input() + input_b = input() + a = [int(i) for i in input_a.split(',')] + b = [int(i) for i in input_b.split(',')] + print(a+b) """ ) kwargs = {'user_answer': user_answer, @@ -432,14 +390,14 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given - self.test_case_data = [{"expected_input": """the quick brown fox jumps\ - over the lazy dog\nthe""", + self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2" }] user_answer = dedent(""" - a = raw_input() - b = raw_input() - print (a.count(b)) + from six.moves import input + a = str(input()) + b = str(input()) + print(a.count(b)) """ ) kwargs = {'user_answer': user_answer, @@ -460,9 +418,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): "expected_output": "3" }] user_answer = dedent(""" - a = input() - b = input() - print a-b + a = int(input()) + b = int(input()) + print(a-b) """ ) kwargs = {'user_answer': user_answer, @@ -480,12 +438,12 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given self.test_case_data = [{"expected_input": "", "expected_output": "2"}] - self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" with open("test.txt") as f: a = f.read() - print a[0] + print(a[0]) """ ) kwargs = {'user_answer': user_answer, @@ -501,5 +459,26 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) + def test_infinite_loop(self): + # Given + test_case_data = [{"expected_input": "1\n2", + "expected_output": "3" + }] + timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + user_answer = "while True:\n\tpass" + kwargs = {'user_answer': user_answer, + 'test_case_data': test_case_data + } + + # When + evaluator = PythonStdioEvaluator() + result = evaluator.evaluate(**kwargs) + + # Then + self.assertEqual(result.get('error'), timeout_msg) + self.assertFalse(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index f1b29dd..0bb7bfd 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -1,19 +1,26 @@ from __future__ import absolute_import import unittest import os +import shutil +import tempfile + from yaksh import code_evaluator as evaluator from yaksh.scilab_code_evaluator import ScilabCodeEvaluator from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): + tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] - self.in_dir = os.getcwd() + self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) self.file_paths = None + def tearDown(self): + shutil.rmtree(self.in_dir) + def test_correct_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") @@ -36,7 +43,7 @@ class ScilabEvaluationTestCases(unittest.TestCase): } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) - self.assertTrue("error" in result.get("error")) + self.assertTrue('error' in result.get("error")) def test_incorrect_answer(self): @@ -48,9 +55,10 @@ class ScilabEvaluationTestCases(unittest.TestCase): 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Message", result.get('error')) - self.assertTrue(result.get('error').splitlines > 1) + self.assertTrue(lines_of_error > 1) def test_infinite_loop(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" -- cgit