From f1da39aded67efa3da145851f0e9f687a3e434e5 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 19 Dec 2016 11:44:55 +0530 Subject: Change all evaluator structure and make sure eval test cases pass --- yaksh/evaluator_tests/test_bash_evaluation.py | 211 ++++++++++++++++++++------ 1 file changed, 161 insertions(+), 50 deletions(-) (limited to 'yaksh/evaluator_tests/test_bash_evaluation.py') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 99e5122..8888ee6 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -3,6 +3,7 @@ import unittest import os import shutil import tempfile +from yaksh.code_evaluator import CodeEvaluator from yaksh.bash_code_evaluator import BashCodeEvaluator from yaksh.bash_stdio_evaluator import BashStdioEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -15,6 +16,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] @@ -32,39 +34,78 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") def test_error(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator(self.in_dir) + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -72,17 +113,31 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] user_answer = ("#!/bin/bash\ncat $1") - get_class = BashCodeEvaluator() - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + # get_class = BashCodeEvaluator() + # kwargs = {'user_answer': user_answer, + # 'partial_grading': True, + # 'test_case_data': self.test_case_data, + # 'file_paths': self.file_paths + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") @@ -92,6 +147,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) + self.file_paths = None + def test_correct_answer(self): user_answer = dedent(""" #!/bin/bash @@ -102,14 +159,28 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -123,15 +194,29 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', - 'expected_input': '1,2,3\n4,5,6\n7,8,9', - 'weight': 0.0 + 'expected_input': '1,2,3\n4,5,6\n7,8,9', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -144,14 +229,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -164,14 +262,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '10', 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + # get_class = BashStdioEvaluator() + # kwargs = {"user_answer": user_answer, + # "partial_grading": True, + # "test_case_data": test_case_data + # } + # result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + evaluator = CodeEvaluator(self.in_dir) + result = evaluator.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) -- cgit