summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py27
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py50
-rw-r--r--yaksh/evaluator_tests/test_code_evaluation.py3
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py92
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py147
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py19
6 files changed, 199 insertions, 139 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index addc5e6..66ade19 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -1,5 +1,8 @@
+from __future__ import unicode_literals
import unittest
import os
+import shutil
+import tempfile
from yaksh.bash_code_evaluator import BashCodeEvaluator
from yaksh.bash_stdio_evaluator import BashStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
@@ -8,15 +11,23 @@ from textwrap import dedent
class BashAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [
{"test_case": "bash_files/sample.sh,bash_files/sample.args"}
]
- self.in_dir = os.getcwd()
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
self.file_paths = None
+ def tearDown(self):
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
+
def test_correct_answer(self):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
" && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
@@ -28,7 +39,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
- self.assertEquals(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct answer")
def test_error(self):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] "
@@ -52,10 +63,10 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_file_based_assert(self):
- self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.file_paths = [('/tmp/test.txt', False)]
self.test_case_data = [
{"test_case": "bash_files/sample1.sh,bash_files/sample1.args"}
]
@@ -67,7 +78,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get("success"))
- self.assertEquals(result.get("error"), "Correct answer")
+ self.assertEqual(result.get("error"), "Correct answer")
class BashStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
@@ -88,7 +99,7 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"test_case_data": test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -108,7 +119,7 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"test_case_data": test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -142,7 +153,7 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"test_case_data": test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 0042d0f..c990436 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -1,5 +1,8 @@
+from __future__ import absolute_import
import unittest
import os
+import shutil
+import tempfile
from yaksh.cpp_code_evaluator import CppCodeEvaluator
from yaksh.cpp_stdio_evaluator import CppStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
@@ -8,13 +11,20 @@ from textwrap import dedent
class CAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
- self.in_dir = os.getcwd()
+ self.in_dir = tmp_in_dir_path
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
self.file_paths = None
+ def tearDown(self):
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
+
def test_correct_answer(self):
user_answer = "int add(int a, int b)\n{return a+b;}"
get_class = CppCodeEvaluator(self.in_dir)
@@ -24,7 +34,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
- self.assertEquals(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct answer")
def test_incorrect_answer(self):
user_answer = "int add(int a, int b)\n{return a-b;}"
@@ -34,9 +44,10 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
+ lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
self.assertIn("Incorrect:", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ self.assertTrue(lines_of_error > 1)
def test_compilation_error(self):
user_answer = "int add(int a, int b)\n{return a+b}"
@@ -58,10 +69,10 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_file_based_assert(self):
- self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.file_paths = [('/tmp/test.txt', False)]
self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}]
user_answer = dedent("""
#include<stdio.h>
@@ -82,8 +93,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
- self.assertEquals(result.get('error'), "Correct answer")
-
+ self.assertEqual(result.get('error'), "Correct answer")
class CppStdioEvaluationTestCases(unittest.TestCase):
@@ -107,7 +117,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -127,7 +137,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -145,7 +155,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -160,9 +170,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
+ lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
self.assertIn("Incorrect", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ self.assertTrue(lines_of_error > 1)
def test_error(self):
user_answer = dedent("""
@@ -192,7 +203,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -208,7 +219,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_correct_answer(self):
@@ -225,7 +236,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_array_input(self):
@@ -246,7 +257,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_string_input(self):
@@ -265,7 +276,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_incorrect_answer(self):
@@ -281,9 +292,10 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
+ lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
self.assertIn("Incorrect", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ self.assertTrue(lines_of_error > 1)
def test_cpp_error(self):
user_answer = dedent("""
@@ -315,7 +327,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_cpp_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -332,7 +344,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py
index cbca32d..88e0253 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_code_evaluation.py
@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
import unittest
import os
from yaksh import python_assertion_evaluator
@@ -38,7 +39,7 @@ class RegistryTestCase(unittest.TestCase):
"stdiobasedtestcase": stdout_evaluator_path
}
)
- self.assertEquals(evaluator_class, class_name)
+ self.assertEqual(evaluator_class, class_name)
def tearDown(self):
self.registry_object = None
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index 74ac677..e375bdb 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -1,5 +1,8 @@
+from __future__ import unicode_literals
import unittest
import os
+import shutil
+import tempfile
from yaksh import code_evaluator as evaluator
from yaksh.java_code_evaluator import JavaCodeEvaluator
from yaksh.java_stdio_evaluator import JavaStdioEvaluator
@@ -9,10 +12,13 @@ from textwrap import dedent
class JavaAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [
{"test_case": "java_files/main_square.java"}
]
- self.in_dir = os.getcwd()
+ self.in_dir = tmp_in_dir_path
evaluator.SERVER_TIMEOUT = 9
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
@@ -20,7 +26,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
self.file_paths = None
def tearDown(self):
- evaluator.SERVER_TIMEOUT = 2
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
def test_correct_answer(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
@@ -30,7 +37,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -42,8 +49,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect:", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ lines_of_error = len(result.get('error').splitlines())
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect", result.get('error'))
+ self.assertTrue(lines_of_error > 1)
def test_error(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
@@ -65,10 +74,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_file_based_assert(self):
- self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.file_paths = [("/tmp/test.txt", False)]
self.test_case_data = [
{"test_case": "java_files/read_file.java"}
]
@@ -97,11 +106,15 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get("success"))
- self.assertEquals(result.get("error"), "Correct answer")
+ self.assertEqual(result.get("error"), "Correct answer")
class JavaStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
self.test_case_data = [{'expected_output': '11',
'expected_input': '5\n6'}]
evaluator.SERVER_TIMEOUT = 4
@@ -109,8 +122,10 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
"You probably have an infinite loop in"
" your code.").format(evaluator.SERVER_TIMEOUT)
- def teardown(self):
+ def tearDown(self):
evaluator.SERVER_TIMEOUT = 4
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
def test_correct_answer(self):
user_answer = dedent("""
@@ -122,12 +137,12 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
int b = s.nextInt();
System.out.print(a+b);
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_array_input(self):
@@ -144,12 +159,12 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
a[i] = s.nextInt();
System.out.print(a[i]);}
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
@@ -163,14 +178,15 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
int b = s.nextInt();
System.out.print(a);
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
+ lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
self.assertIn("Incorrect", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ self.assertTrue(lines_of_error > 1)
def test_error(self):
@@ -179,7 +195,7 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
{
System.out.print("a");
}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
@@ -196,13 +212,13 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
{
System.out.print("a");}
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
@@ -214,12 +230,12 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
int b = 6;
System.out.print(a+b);
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_string_input(self):
@@ -234,13 +250,45 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
String b = s.nextLine();
System.out.print(a+b);
}}""")
- get_class = JavaStdioEvaluator()
+ get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
+ def test_file_based_stdout(self):
+ self.file_paths = [("/tmp/test.txt", False)]
+ self.test_case_data = [{'expected_output': '2',
+ 'expected_input': ''}]
+ user_answer = dedent("""
+ import java.io.BufferedReader;
+ import java.io.FileReader;
+ import java.io.IOException;
+ class Test{
+ public static void main(String[] args) throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader("test.txt"));
+ try {
+ StringBuilder sb = new StringBuilder();
+ String line = br.readLine();
+ while (line != null) {
+ sb.append(line);
+ line = br.readLine();}
+ System.out.print(sb.toString());
+ } finally {
+ br.close();
+ }}}
+ """)
+ get_class = JavaStdioEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get("success"))
+ self.assertEqual(result.get("error"), "Correct answer")
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 2a109d5..45cc40d 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,13 +1,22 @@
+from __future__ import unicode_literals
import unittest
import os
+import tempfile
+import shutil
+from textwrap import dedent
+
+# Local import
from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
from yaksh.python_stdio_evaluator import PythonStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
-from textwrap import dedent
class PythonAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
{"test_case": 'assert(add(-1,2)==1)'},
{"test_case": 'assert(add(-1,-2)==-3)'},
@@ -17,6 +26,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
" your code.").format(SERVER_TIMEOUT)
self.file_paths = None
+ def tearDown(self):
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
+
def test_correct_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
@@ -159,7 +172,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
""")
recursion_error_msg = ["Traceback",
"call",
- "RuntimeError",
"maximum recursion depth exceeded"
]
kwargs = {'user_answer': user_answer,
@@ -174,7 +186,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get("success"))
- self.assertEqual(969, len(err))
for msg in recursion_error_msg:
self.assertIn(msg, result.get("error"))
@@ -187,7 +198,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
type_error_msg = ["Traceback",
"call",
"TypeError",
- "exactly",
"argument"
]
kwargs = {'user_answer': user_answer,
@@ -238,7 +248,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
def test_file_based_assert(self):
# Given
self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
- self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.file_paths = [('/tmp/test.txt', False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
@@ -322,65 +332,10 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
self.assertIn(msg, result.get("error"))
-class PythonStdoutEvaluationTestCases(unittest.TestCase):
- def setUp(self):
- self.test_case_data = [{"expected_input": None,
- "expected_output": "0 1 1 2 3"
- }]
-
- self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop"
- " in your code.").format(SERVER_TIMEOUT)
-
- def test_correct_answer(self):
- # Given
- user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertEqual(result.get('error'), "Correct Answer")
- self.assertTrue(result.get('success'))
-
- def test_incorrect_answer(self):
- # Given
- user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertFalse(result.get('success'))
- self.assertIn("Incorrect Answer", result.get('error'))
-
- def test_infinite_loop(self):
- # Given
- user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertEqual(result.get('error'), self.timeout_msg)
- self.assertFalse(result.get('success'))
-
-
class PythonStdIOEvaluationTestCases(unittest.TestCase):
-
def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
self.file_paths = None
def test_correct_answer_integer(self):
@@ -389,9 +344,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
"expected_output": "3"
}]
user_answer = dedent("""
- a = input()
- b = input()
- print a+b
+ a = int(input())
+ b = int(input())
+ print(a+b)
"""
)
kwargs = {'user_answer': user_answer,
@@ -404,17 +359,20 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct Answer", result.get('error'))
+ self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_list(self):
# Given
- self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
+ self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
"expected_output": "[1, 2, 3, 5, 6, 7]"
}]
user_answer = dedent("""
- a = input()
- b = input()
- print a+b
+ from six.moves import input
+ input_a = input()
+ input_b = input()
+ a = [int(i) for i in input_a.split(',')]
+ b = [int(i) for i in input_b.split(',')]
+ print(a+b)
"""
)
kwargs = {'user_answer': user_answer,
@@ -427,18 +385,18 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct Answer", result.get('error'))
+ self.assertIn("Correct answer", result.get('error'))
def test_correct_answer_string(self):
# Given
- self.test_case_data = [{"expected_input": """the quick brown fox jumps\
- over the lazy dog\nthe""",
+ self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
"expected_output": "2"
}]
user_answer = dedent("""
- a = raw_input()
- b = raw_input()
- print (a.count(b))
+ from six.moves import input
+ a = str(input())
+ b = str(input())
+ print(a.count(b))
"""
)
kwargs = {'user_answer': user_answer,
@@ -451,7 +409,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertTrue(result.get('success'))
- self.assertIn("Correct Answer", result.get('error'))
+ self.assertIn("Correct answer", result.get('error'))
def test_incorrect_answer_integer(self):
# Given
@@ -459,9 +417,9 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
"expected_output": "3"
}]
user_answer = dedent("""
- a = input()
- b = input()
- print a-b
+ a = int(input())
+ b = int(input())
+ print(a-b)
"""
)
kwargs = {'user_answer': user_answer,
@@ -474,17 +432,17 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
# Then
self.assertFalse(result.get('success'))
- self.assertIn("Incorrect Answer", result.get('error'))
+ self.assertIn("Incorrect answer", result.get('error'))
def test_file_based_answer(self):
# Given
self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
- self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.file_paths = [('/tmp/test.txt', False)]
user_answer = dedent("""
with open("test.txt") as f:
a = f.read()
- print a[0]
+ print(a[0])
"""
)
kwargs = {'user_answer': user_answer,
@@ -497,8 +455,29 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
result = evaluator.evaluate(**kwargs)
# Then
- self.assertEqual(result.get('error'), "Correct Answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
+ def test_infinite_loop(self):
+ # Given
+ test_case_data = [{"expected_input": "1\n2",
+ "expected_output": "3"
+ }]
+ timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in"
+ " your code.").format(SERVER_TIMEOUT)
+ user_answer = "while True:\n\tpass"
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': test_case_data
+ }
+
+ # When
+ evaluator = PythonStdioEvaluator()
+ result = evaluator.evaluate(**kwargs)
+
+ # Then
+ self.assertEqual(result.get('error'), timeout_msg)
+ self.assertFalse(result.get('success'))
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index f5e3767..b366480 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -1,18 +1,26 @@
+from __future__ import unicode_literals
import unittest
import os
+import shutil
+import tempfile
+
from yaksh import code_evaluator as evaluator
from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
from yaksh.settings import SERVER_TIMEOUT
class ScilabEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
- self.in_dir = os.getcwd()
+ self.in_dir = tmp_in_dir_path
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop"
" in your code.").format(SERVER_TIMEOUT)
self.file_paths = None
+ def tearDown(self):
+ shutil.rmtree(self.in_dir)
+
def test_correct_answer(self):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a+b;\nendfunction")
@@ -22,7 +30,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
- self.assertEquals(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_error(self):
@@ -35,7 +43,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertTrue("error" in result.get("error"))
+ self.assertTrue('error' in result.get("error"))
def test_incorrect_answer(self):
@@ -47,9 +55,10 @@ class ScilabEvaluationTestCases(unittest.TestCase):
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
+ lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
self.assertIn("Message", result.get('error'))
- self.assertTrue(result.get('error').splitlines > 1)
+ self.assertTrue(lines_of_error > 1)
def test_infinite_loop(self):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
@@ -61,7 +70,7 @@ class ScilabEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
- self.assertEquals(result.get("error"), self.timeout_msg)
+ self.assertEqual(result.get("error"), self.timeout_msg)
if __name__ == '__main__':
unittest.main()
generated by cgit (git 2.34.1) at 2024-12-21 11:19:37 +0000