From ec5cb67b3314d05675c501cd78ce92d02ae8fde4 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 6 Jan 2017 12:33:26 +0530 Subject: added test cases for hook evaluator --- yaksh/evaluator_tests/test_bash_evaluation.py | 259 ++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) (limited to 'yaksh/evaluator_tests/test_bash_evaluation.py') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 4b551d7..0662831 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -269,5 +269,264 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertTrue(result.get('success')) + +class BashHookEvaluationTestCases(EvaluatorBaseTest): + + def setUp(self): + self.f_path = os.path.join(tempfile.gettempdir(), "test.txt") + with open(self.f_path, 'wb') as f: + f.write('2'.encode('ascii')) + self.in_dir = tempfile.mkdtemp() + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) + self.file_paths = None + + def tearDown(self): + os.remove(self.f_path) + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Hello, world! + """ + ) + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Goodbye, world! + """ + ) + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output('Incorrect Answer', result.get('error')) + + def test_assert_with_hook(self): + # Given + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" + " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + ) + assert_test_case = dedent(""" + #!/bin/bash + [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 )) + """) + + assert_test_case_args = "1 2\n2 1" + + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "echo $(( $1 + $2 ))" in user_answer: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": assert_test_case, + "test_case_args":assert_test_case_args, + 'weight': 1.0 + }, + {"test_case_type": "hooktestcase", + "hook_code": hook_code, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 2.0) + + def test_multiple_hooks(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Hello, world! + """ + ) + + hook_code_1 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "echo -n Hello, world!" in user_answer: + success, err, mark_fraction = True, "", 0.5 + return success, err, mark_fraction + """ + ) + hook_code_2 = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code_1, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code_2, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 1.5) + + def test_infinite_loop(self): + # Given + user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" + " do echo "" > /dev/null ; done") + + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output(self.timeout_msg, result.get('error')) + + if __name__ == '__main__': unittest.main() -- cgit