summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py159
1 files changed, 127 insertions, 32 deletions
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 1e867a3..dfdc6ec 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,7 +1,7 @@
import unittest
import os
from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
-from yaksh.python_stdout_evaluator import PythonStdoutEvaluator
+from yaksh.python_stdio_evaluator import PythonStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
@@ -164,15 +164,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
return int(a) + int(b) + int(c)
""")
value_error_msg = ["Traceback",
- "call",
- "ValueError",
- "invalid literal",
- "base"
- ]
+ "call",
+ "ValueError",
+ "invalid literal",
+ "base"
+ ]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
@@ -180,55 +180,150 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
-class PythonStdoutEvaluationTestCases(unittest.TestCase):
+
+class PythonStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.test_case_data = [{"expected_output": "0 1 1 2 3"}]
+ self.test_case_data = [{"expected_input": None,
+ "expected_output": "0 1 1 2 3"
+ }]
+
self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop"
- " in your code.").format(SERVER_TIMEOUT)
+ "You probably have an infinite loop"
+ " in your code.").format(SERVER_TIMEOUT)
def test_correct_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
- get_class = PythonStdoutEvaluator()
+ get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
- self.assertEqual(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct Answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
- get_class = PythonStdoutEvaluator()
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ get_class = PythonStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), "Incorrect Answer")
+ self.assertIn("Incorrect Answer", result.get('error'))
def test_direct_printed_answer(self):
user_answer = "print '0 1 1 2 3'"
error_msg = ("Incorrect Answer: Please avoid printing"
- " the expected output directly"
- )
- get_class = PythonStdoutEvaluator()
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ " the expected output directly"
+ )
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), error_msg)
def test_infinite_loop(self):
- user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
- get_class = PythonStdoutEvaluator()
+ user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
+ get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertEqual(result.get('error'), self.timeout_msg)
+
+
+class PythonStdIOEvaluator(unittest.TestCase):
+
+ def setUp(self):
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop"
+ " in your code.").format(SERVER_TIMEOUT)
+
+ def test_add_two_integers_correct(self):
+ self.test_case_data = [{"expected_input": "1\n2",
+ "expected_output": "3"
+ }]
+ user_answer = dedent("""
+ a = int(raw_input())
+ b = int(raw_input())
+ print a+b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_add_two_lists(self):
+ self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
+ "expected_output": "[1, 2, 3, 5, 6, 7]"
+ }]
+ user_answer = dedent("""
+ from ast import literal_eval
+ a = literal_eval(raw_input())
+ b = literal_eval(raw_input())
+ print a+b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_find_string_in_sentence(self):
+ self.test_case_data = [{"expected_input": """the quick brown fox jumps\
+ over the lazy dog\nthe""",
+ "expected_output": "2"
+ }]
+ user_answer = dedent("""
+ a = raw_input()
+ b = raw_input()
+ print (a.count(b))
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_add_two_integers_incorrect(self):
+ self.test_case_data = [{"expected_input": "1\n2",
+ "expected_output": "4"
+ }]
+ user_answer = dedent("""
+ a = int(raw_input())
+ b = int(raw_input())
+ print a+b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), 'Incorrect Answer')
+ self.assertIn("Incorrect Answer", result.get('error'))
if __name__ == '__main__':
generated by cgit (git 2.34.1) at 2024-12-23 04:40:56 +0000