added test cases for hook evaluator

1 files changed, 259 insertions, 0 deletions

diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 4b551d7..0662831 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -269,5 +269,264 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         self.assertTrue(result.get('success'))
 
+
+class BashHookEvaluationTestCases(EvaluatorBaseTest):
+
+    def setUp(self):
+        self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+        with open(self.f_path, 'wb') as f:
+            f.write('2'.encode('ascii'))
+        self.in_dir = tempfile.mkdtemp()
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop in your"
+            " code.").format(SERVER_TIMEOUT)
+        self.file_paths = None
+
+    def tearDown(self):
+        os.remove(self.f_path)
+        shutil.rmtree(self.in_dir)
+
+    def test_correct_answer(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Hello, world!
+                             """
+                             )
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Goodbye, world!
+                             """
+                             )
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output('Incorrect Answer', result.get('error'))
+    
+    def test_assert_with_hook(self):
+        # Given
+        user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
+                       " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
+                       )
+        assert_test_case = dedent("""
+            #!/bin/bash
+            [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))
+            """)
+
+        assert_test_case_args = "1 2\n2 1"
+        
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "echo $(( $1 + $2 ))" in user_answer:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": assert_test_case,
+                           "test_case_args":assert_test_case_args,
+                           'weight': 1.0
+                           },
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 2.0)
+
+    def test_multiple_hooks(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Hello, world!
+                             """
+                             )
+        
+        hook_code_1 = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "echo -n Hello, world!" in user_answer:
+                                    success, err, mark_fraction = True, "", 0.5
+                                return success, err, mark_fraction
+                            """
+                            )
+        hook_code_2 = dedent("""\
+                    def check_answer(user_answer):
+                        import subprocess
+                        success = False
+                        err = "Incorrect Answer"
+                        mark_fraction = 0.0
+                        proc = subprocess.Popen(user_answer, shell=True,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.PIPE
+                                                )
+                        stdout,stderr = proc.communicate()
+                        if stdout == "Hello, world!":
+                            success, err, mark_fraction = True, "", 1.0
+                        return success, err, mark_fraction
+                    """
+                    )
+
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_1, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_2, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 1.5)
+        
+    def test_infinite_loop(self):
+        # Given
+        user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
+                       " do echo "" > /dev/null ; done")
+
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+            )
+
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
 if __name__ == '__main__':
     unittest.main()