Merge pull request #176 from maheshgudi/hook_evaluator

Hook evaluator

12 files changed, 1270 insertions, 25 deletions

diff --git a/yaksh/documentation/images/hook_testcase.jpg b/yaksh/documentation/images/hook_testcase.jpg
new file mode 100644
index 0000000..3018050
--- /dev/null
+++ b/yaksh/documentation/images/hook_testcase.jpg
diff --git a/yaksh/documentation/moderator_docs/creating_question.rst b/yaksh/documentation/moderator_docs/creating_question.rst
index ec273f5..f99bf7f 100644
--- a/yaksh/documentation/moderator_docs/creating_question.rst
+++ b/yaksh/documentation/moderator_docs/creating_question.rst
@@ -234,6 +234,24 @@ How to write Test cases
 
 		For MCC based question, check the correct checkbox for multiple correct options.
 
+	* **Create Hook based Test Case**
+
+		Select Hook from Add Test Case field.
+
+		In Hook based test case type, moderator is provided with a evaluator function
+		called **check_answer** which is provided with a parameter called **user_answer**.
+
+		**user_answer** is the code of the student in string format.
+
+		A moderator can check the string for specific words in the user answer
+		and/or compile and execute the user answer (using standard python libraries) to 
+		evaluate and hence return the mark fraction.
+
+
+		.. image:: ../images/hook_testcase.jpg
+				:width: 80%
+
+
 
 Features in Question
 --------------------
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 4b551d7..482d45e 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -269,5 +269,266 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         self.assertTrue(result.get('success'))
 
+
+class BashHookEvaluationTestCases(EvaluatorBaseTest):
+
+    def setUp(self):
+        self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+        with open(self.f_path, 'wb') as f:
+            f.write('2'.encode('ascii'))
+        self.in_dir = tempfile.mkdtemp()
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop in your"
+            " code.").format(SERVER_TIMEOUT)
+        self.file_paths = None
+
+    def tearDown(self):
+        os.remove(self.f_path)
+        shutil.rmtree(self.in_dir)
+
+    def test_correct_answer(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Hello, world!
+                             """
+                             )
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Goodbye, world!
+                             """
+                             )
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output('Incorrect Answer', result.get('error'))
+    
+    def test_assert_with_hook(self):
+        # Given
+        user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
+                       " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
+                       )
+        assert_test_case = dedent("""
+            #!/bin/bash
+            [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))
+            """)
+
+        assert_test_case_args = "1 2\n2 1"
+        
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "echo $(( $1 + $2 ))" in user_answer:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": assert_test_case,
+                           "test_case_args":assert_test_case_args,
+                           'weight': 1.0
+                           },
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 2.0)
+
+    def test_multiple_hooks(self):
+        # Given
+        user_answer = dedent(""" #!/bin/bash
+                             echo -n Hello, world!
+                             """
+                             )
+        
+        hook_code_1 = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "echo -n Hello, world!" in user_answer:
+                                    success, err, mark_fraction = True, "", 0.5
+                                return success, err, mark_fraction
+                            """
+                            )
+        hook_code_2 = dedent("""\
+                    def check_answer(user_answer):
+                        import subprocess
+                        import sys
+                        success = False
+                        err = "Incorrect Answer"
+                        mark_fraction = 0.0
+                        proc = subprocess.Popen(user_answer, shell=True,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.PIPE
+                                                )
+                        stdout,stderr = proc.communicate()
+
+                        if stdout.decode('utf-8') == "Hello, world!":
+                            success, err, mark_fraction = True, "", 1.0
+                        return success, err, mark_fraction
+                    """
+                    )
+
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_1, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_2, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 1.5)
+        
+    def test_infinite_loop(self):
+        # Given
+        user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
+                       " do echo "" > /dev/null ; done")
+
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                proc = subprocess.Popen(user_answer, shell=True,
+                                                        stdout=subprocess.PIPE,
+                                                        stderr=subprocess.PIPE
+                                                        )
+                                stdout,stderr = proc.communicate()
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+            )
+
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'bash'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index d734cf2..304f1cb 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -639,5 +639,335 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         self.assertTrue(result.get('success'))
 
+class CppHookEvaluationTestCases(EvaluatorBaseTest):
+
+    def setUp(self):
+        self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+        with open(self.f_path, 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop in your"
+            " code.").format(SERVER_TIMEOUT)
+        self.file_paths = None
+
+    def tearDown(self):
+        os.remove(self.f_path)
+        shutil.rmtree(self.in_dir)
+
+    def test_correct_answer(self):
+        # Given
+        user_answer = dedent("""\
+                             #include<stdio.h>
+                             main()
+                             {
+                                printf("Hello, world!");
+                              }
+                              """)
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.c", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["gcc Test.c", "./a.out"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'cpp'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        # Given
+        user_answer = dedent("""\
+                             #include<stdio.h>
+                             main()
+                             {
+                                printf("Goodbye, world!");
+                              }
+                              """)
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.c", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["gcc Test.c", "./a.out"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'cpp'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output('Incorrect Answer', result.get('error'))
+    
+    def test_assert_with_hook(self):
+        # Given
+        user_answer = "int add(int a, int b)\n{return a+b;}"
+
+
+        assert_test_case = dedent("""\
+                                  #include <stdio.h>
+                                  #include <stdlib.h>
+
+                                  extern int add(int, int);
+
+                                  template <class T>
+
+                                  void check(T expect, T result)
+                                  {
+                                      if (expect == result)
+                                      {
+                                      printf("Correct: Expected %d got %d ",expect,result);
+                                      }
+                                      else
+                                      {
+                                      printf("Incorrect: Expected %d got %d ",expect,result);
+                                      exit (1);
+                                     }
+                                  }
+
+                                  int main(void)
+                                  {
+                                      int result;
+                                      result = add(0,0);
+                                          printf("Input submitted to the function: 0, 0");
+                                      check(0, result);
+                                      result = add(2,3);
+                                          printf("Input submitted to the function: 2 3");
+                                      check(5,result);
+                                      printf("All Correct");
+                                      return 0;
+                                  }
+                                  """)
+
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "return a+b;" in user_answer:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": assert_test_case,
+                           'weight': 1.0
+                           },
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'cpp'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 2.0)
+
+    def test_multiple_hooks(self):
+        # Given
+        user_answer = dedent("""\
+                             #include<stdio.h>
+                             main()
+                             {
+                                printf("Hello, world!");
+                              }
+                              """)
+        
+        hook_code_1 = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.c", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["gcc Test.c", "./a.out"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+        hook_code_2 = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if 'printf("Hello, world!");' in user_answer:
+                                    success, err, mark_fraction = True, "", 0.5
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_1, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_2, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'cpp'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 1.5)
+        
+    def test_infinite_loop(self):
+        # Given
+        user_answer = dedent("""\
+                             #include<stdio.h>
+                             int main(void){
+                             while(0==0){
+                             printf("abc");}
+                             }""")
+
+        hook_code= dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.c", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["gcc Test.c", "./a.out"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'cpp'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_grader_evaluation.py
index cb783b0..d11f4a0 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_grader_evaluation.py
@@ -16,34 +16,27 @@ class RegistryTestCase(unittest.TestCase):
         stdio_evaluator_path = ("yaksh.python_stdio_evaluator."
             "PythonStdIOEvaluator"
         )
+
+        hook_evaluator_path = ("yaksh.hook_evaluator."
+            "HookEvaluator"
+        )
         code_evaluators['python'] = \
         {"standardtestcase": assertion_evaluator_path,
-        "stdiobasedtestcase": stdio_evaluator_path
+        "stdiobasedtestcase": stdio_evaluator_path,
+        "hooktestcase": hook_evaluator_path
         }
 
     def test_set_register(self):
         evaluator_class = self.registry_object.get_class("python", 
             "standardtestcase"
         )
-        assertion_evaluator_path = ("yaksh.python_assertion_evaluator"
-            ".PythonAssertionEvaluator"
-        )
-        stdio_evaluator_path = ("yaksh.python_stdio_evaluator."
-            "PythonStdIOEvaluator"
-        )
         class_name = getattr(python_assertion_evaluator, 
             'PythonAssertionEvaluator'
         )
-        self.registry_object.register("python", 
-            {"standardtestcase": assertion_evaluator_path,
-                "stdiobasedtestcase": stdio_evaluator_path
-            }
-        )
         self.assertEqual(evaluator_class, class_name)
 
     def tearDown(self):
         self.registry_object = None
 
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index b53d8aa..3d127af 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -507,5 +507,334 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         self.assertTrue(result.get("success"))
 
 
+class JavaHookEvaluationTestCases(EvaluatorBaseTest):
+
+    def setUp(self):
+        self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+        with open(self.f_path, 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
+        self.file_paths = None
+        gd.SERVER_TIMEOUT = 9
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+            "You probably have an infinite loop in"
+            " your code.").format(gd.SERVER_TIMEOUT)
+
+    def tearDown(self):
+        gd.SERVER_TIMEOUT = 4
+        os.remove(self.f_path)
+        shutil.rmtree(self.in_dir)
+
+    def test_correct_answer(self):
+        # Given
+        user_answer = dedent("""\
+                             class Test
+                             {public static void main(String[] args){
+                             System.out.print("Hello, world!");
+                             }}
+                             """)
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.java", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["javac Test.java", "java Test"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        # Given
+        user_answer = dedent("""\
+                             class Test
+                             {public static void main(String[] args){
+                             System.out.print("Goodbye, world!");
+                             }}
+                             """)
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.java", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["javac Test.java", "java Test"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output('Incorrect Answer', result.get('error'))
+    
+    def test_assert_with_hook(self):
+        # Given
+        user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
+        assert_test_case = dedent("""
+            class main
+            {
+                public static <E> void check(E expect, E result)
+                {
+                    if(result.equals(expect))
+                    {
+                        System.out.println("Correct:Output expected "+expect+" and got "+result);
+                    }
+                else
+                {
+                    System.out.println("Incorrect:Output expected "+expect+" but got "+result);
+                    System.exit(1);
+                }
+                }
+                public static void main(String arg[])
+                {
+                   Test t = new Test();
+                   int result, input, output;
+                   input = 0; output = 0;
+                   result = t.square_num(input);
+                   System.out.println("Input submitted to the function: "+input);
+                   check(output, result);
+                   input = 5; output = 25;
+                   result = t.square_num(input);
+                   System.out.println("Input submitted to the function: "+input);
+                   check(output, result);
+                   input = 6; output = 36;
+                   result = t.square_num(input);
+                   System.out.println("Input submitted to the function: "+input);
+                   check(output, result);
+                }
+            }
+            """)
+        
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "return a*a" in user_answer:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": assert_test_case,
+                           'weight': 1.0
+                           },
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'java'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 2.0)
+
+    def test_multiple_hooks(self):
+        # Given
+        user_answer = dedent("""\
+                             class Test
+                             {public static void main(String[] args){
+                             System.out.print("Hello, world!");
+                             }}
+                             """)
+        
+        hook_code_1 = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.java", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["javac Test.java", "java Test"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        hook_code_2 = dedent("""\
+                             def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if 'System.out.print("Hello, world!");' in user_answer:
+                                    success, err, mark_fraction = True, "", 0.5
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_1, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_2, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'java'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 1.5)
+        
+    def test_infinite_loop(self):
+        # Given
+        user_answer = dedent("""\
+                             class Test
+                             {public static void main(String[] args){
+                             while(0==0)
+                            {
+                            System.out.print("a");}
+                            }}""")
+
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                with open("Test.java", "w+") as f:
+                                    f.write(user_answer)
+                                import subprocess
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                def _run_command(cmd):
+                                    proc = subprocess.Popen("{}".format(cmd),
+                                                            shell=True,
+                                                            stdout=subprocess.PIPE,
+                                                            stderr=subprocess.PIPE
+                                                            )
+                                    stdout,stderr = proc.communicate()
+                                    return stdout,stderr
+                                cmds = ["javac Test.java", "java Test"]
+                                for cmd in cmds:
+                                    stdout, stderr = _run_command(cmd)
+                                if stdout.decode("utf-8") == "Hello, world!":
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 43dfe6b..1fba73e 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -7,8 +7,6 @@ from textwrap import dedent
 
 # Local import
 from yaksh.grader import Grader
-from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
-from yaksh.python_stdio_evaluator import PythonStdIOEvaluator
 from yaksh.settings import SERVER_TIMEOUT
 
 
@@ -643,5 +641,217 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
         self.assertFalse(result.get('success'))
 
 
+class PythonHookEvaluationTestCases(EvaluatorBaseTest):
+
+    def setUp(self):
+        with open('/tmp/test.txt', 'wb') as f:
+            f.write('2'.encode('ascii'))
+        tmp_in_dir_path = tempfile.mkdtemp()
+        self.in_dir = tmp_in_dir_path
+        self.timeout_msg = ("Code took more than {0} seconds to run. "
+                            "You probably have an infinite loop in"
+                            " your code.").format(SERVER_TIMEOUT)
+        self.file_paths = None
+
+    def tearDown(self):
+        os.remove('/tmp/test.txt')
+        shutil.rmtree(self.in_dir)
+
+    def test_correct_answer(self):
+        # Given
+        user_answer = "def add(a,b):\n\treturn a + b"
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                exec(user_answer, globals())
+                                if add(1,2) == 3:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+
+    def test_incorrect_answer(self):
+        # Given
+        user_answer = "def add(a,b):\n\treturn a - b"
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                exec(user_answer, globals())
+                                if add(1,2) == 3:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output('Incorrect Answer', result.get('error'))
+    
+    def test_assert_with_hook(self):
+        # Given
+        user_answer = "def add(a,b):\n\treturn a + b"
+        assert_test_case = "assert add(1,2) == 3"
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "return a + b" in user_answer:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+        test_case_data = [{"test_case_type": "standardtestcase",
+                           "test_case": assert_test_case, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 2.0)
+
+    def test_multiple_hooks(self):
+        # Given
+        user_answer = "def add(a,b):\n\treturn a + b"
+        hook_code_1 = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                if "return a + b" in user_answer:
+                                    success, err, mark_fraction = True, "", 0.5
+                                return success, err, mark_fraction
+                            """
+                            )
+        hook_code_2 = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                exec(user_answer, globals())
+                                if add(1,2) == 3:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+
+
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_1, 'weight': 1.0},
+                          {"test_case_type": "hooktestcase",
+                           "hook_code": hook_code_2, 'weight': 1.0},
+                          ]
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': True,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertTrue(result.get('success'))
+        self.assertEqual(result.get("weight"), 1.5)
+        
+    def test_infinite_loop(self):
+        # Given
+        user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+        hook_code = dedent("""\
+                            def check_answer(user_answer):
+                                success = False
+                                err = "Incorrect Answer"
+                                mark_fraction = 0.0
+                                exec(user_answer, globals())
+                                if add(1,2) == 3:
+                                    success, err, mark_fraction = True, "", 1.0
+                                return success, err, mark_fraction
+                            """
+                            )
+        test_case_data = [{"test_case_type": "hooktestcase",
+                           "hook_code": hook_code,"weight": 1.0
+                            }]
+
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'python'
+                    },
+                    'test_case_data': test_case_data,
+                  }
+
+        # When
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
+        # Then
+        self.assertFalse(result.get('success'))
+        self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/yaksh/forms.py b/yaksh/forms.py
index 1d18d29..8a90dee 100644
--- a/yaksh/forms.py
+++ b/yaksh/forms.py
@@ -1,6 +1,7 @@
 from django import forms
 from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course,\
-                         QuestionPaper, StandardTestCase, StdIOBasedTestCase
+                         QuestionPaper, StandardTestCase, StdIOBasedTestCase, \
+                         HookTestCase
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
@@ -40,7 +41,8 @@ test_case_types = (
         ("standardtestcase", "Standard Testcase"),
         ("stdiobasedtestcase", "StdIO Based Testcase"),
         ("mcqtestcase", "MCQ Testcase"),
-    )
+        ("hooktestcase", "Hook Testcase"),
+        )
 
 UNAME_CHARS = letters + "._" + digits
 PWD_CHARS = letters + punctuation + digits
@@ -296,3 +298,4 @@ class QuestionPaperForm(forms.ModelForm):
     class Meta:
         model = QuestionPaper
         fields = ['shuffle_questions']
+
diff --git a/yaksh/grader.py b/yaksh/grader.py
index 086abb7..a9a3738 100644
--- a/yaksh/grader.py
+++ b/yaksh/grader.py
@@ -120,7 +120,6 @@ class Grader(object):
         for test_case in test_case_data:
             test_case_instance = create_evaluator_instance(metadata, test_case)
             test_case_instances.append(test_case_instance)
-
         return test_case_instances
 
 
diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py
new file mode 100644
index 0000000..3956da1
--- /dev/null
+++ b/yaksh/hook_evaluator.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+import sys
+import traceback
+import os
+
+# Local imports
+from .file_utils import copy_files, delete_files
+from .base_evaluator import BaseEvaluator
+from .grader import TimeoutException
+
+
+class HookEvaluator(BaseEvaluator):
+    def __init__(self, metadata, test_case_data):
+        self.files = []
+
+        # Set metadata values
+        self.user_answer = metadata.get('user_answer')
+        self.file_paths = metadata.get('file_paths')
+        self.partial_grading = metadata.get('partial_grading')
+
+        # Set test case data values
+        self.hook_code = test_case_data.get('hook_code')
+        self.weight = test_case_data.get('weight')
+
+    def teardown(self):
+        # Delete the created file.
+        if self.files:
+            delete_files(self.files)
+
+    def check_code(self):
+        """ Function evaluates user answer by running a python based hook code
+        against it.
+        Returns
+        --------
+        Returns a tuple (success, error, test_case_weight)
+
+        success - Boolean, indicating if code was executed successfully, correctly
+        mark_fraction - Float, indicating fraction of the weight to a test case
+        error - String, error message if success is false
+
+        returns (True, "Correct answer", 1.0) : If the student script passes all
+        test cases/have same output, when compared to the instructor script
+
+        returns (False, error_msg, 0.0): If the student script fails a single
+        test/have dissimilar output, when compared to the instructor script.
+
+        Returns (False, error_msg, 0.0): If mandatory arguments are not files or if
+        the required permissions are not given to the file(s).
+        """
+        success = False
+        mark_fraction = 0.0
+        try:
+            tb = None
+            _tests = compile(self.hook_code, '<string>', mode='exec')
+            hook_scope = {}
+            exec(_tests, hook_scope)
+            check = hook_scope["check_answer"]
+            success, err, mark_fraction = check(self.user_answer)
+        except TimeoutException:
+            raise
+        except Exception:
+            msg = traceback.format_exc(limit=0)
+            err = "Error in Hook code: {0}".format(msg)
+        del tb
+        return success, err, mark_fraction
+ 
\ No newline at end of file
diff --git a/yaksh/models.py b/yaksh/models.py
index ca41885..ad61872 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -4,6 +4,7 @@ import json
 from random import sample, shuffle
 from itertools import islice, cycle
 from collections import Counter
+from textwrap import dedent
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth.models import User
@@ -1219,5 +1220,31 @@ class McqTestCase(TestCase):
 
 
 class HookTestCase(TestCase):
-    code = models.TextField()
+    hook_code = models.TextField(default=dedent\
+                ("""\
+                 def check_answer(user_answer):
+                    ''' Evaluates user answer to return -
+                    success - Boolean, indicating if code was executed correctly
+                    mark_fraction - Float, indicating fraction of the
+                                   weight to a test case
+                    error - String, error message if success is false'''
+                    success = False
+                    err = "Incorrect Answer" # Please make the error message more specific
+                    mark_fraction = 0.0
+
+                    # write your code here
+
+                    return success, err, mark_fraction
+
+                """)
+
+                                )
     weight = models.FloatField(default=1.0)
+
+    def get_field_value(self):
+        return {"test_case_type": "hooktestcase", "hook_code": self.hook_code,
+                "weight": self.weight}
+
+    def __str__(self):
+        return u'Hook Testcase | Correct: {0}'.format(self.hook_code)
+
diff --git a/yaksh/settings.py b/yaksh/settings.py
index 0e432cf..72f9fda 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -21,18 +21,26 @@ URL_ROOT = ''
 
 code_evaluators = {
     "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator",
-               "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator"
+               "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator",
+               "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
                },
     "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator",
-          "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator"
+          "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator",
+          "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
           },
     "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator",
-            "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator"
+            "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator",
+            "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
             },
     "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator",
-             "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator"},
+             "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator",
+             "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
+             },
     "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator",
-            "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator"
+             "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator",
+             "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
              },
-    "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"},
+    "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator",
+               "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
+               },
 }