Modify tests to handle hidden tests

6 files changed, 86 insertions, 54 deletions

diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 689f65b..031a9de 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -25,7 +25,7 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
             {"test_case": self.tc_data,
              "test_case_args": self.tc_data_args,
              "test_case_type": "standardtestcase",
-             "weight": 0.0
+             "weight": 0.0, "hidden": False
              }
         ]
         self.in_dir = tempfile.mkdtemp()
@@ -66,7 +66,8 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
             {"test_case": tc_data,
              "test_case_args": "",
              "test_case_type": "standardtestcase",
-             "weight": 0.0
+             "weight": 0.0,
+             "hidden": True
              }
         ]
         kwargs = {'metadata': {
@@ -129,6 +130,7 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get("success"))
+        self.assertFalse(result.get("error")[0]["hidden"])
         self.assert_correct_output("Error",
                                    result.get("error")[0]["message"])
 
@@ -171,7 +173,8 @@ class BashAssertionEvaluationTestCases(EvaluatorBaseTest):
             "test_case": self.tc_data,
             "test_case_args": self.tc_data_args,
             "test_case_type": "standardtestcase",
-            "weight": 0.0
+            "weight": 0.0,
+            "hidden": True
         }]
         user_answer = ("#!/bin/bash\ncat $1")
         kwargs = {'metadata': {
@@ -241,7 +244,7 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
         test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
                            'expected_input': '1,2,3\n4,5,6\n7,8,9',
                            'test_case_type': 'stdiobasedtestcase',
-                           'weight': 0.0
+                           'weight': 0.0,
                            }]
         kwargs = {
                   'metadata': {
@@ -271,7 +274,8 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
             'expected_output': '11',
             'expected_input': '5\n6',
             'test_case_type': 'stdiobasedtestcase',
-            'weight': 0.0
+            'weight': 0.0,
+            'hidden': True
             }]
         kwargs = {
                   'metadata': {
@@ -289,6 +293,7 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         result_error = result.get('error')[0].get('error_msg')
         self.assert_correct_output("Incorrect", result_error)
+        self.assertTrue(result.get('error')[0]['hidden'])
         self.assertFalse(result.get('success'))
 
     def test_stdout_only(self):
@@ -400,7 +405,8 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest):
                                 return success, err, mark_fraction
                             """)
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0}]
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": True}]
 
         kwargs = {
                   'metadata': {
@@ -417,6 +423,7 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
+        self.assertTrue(result.get('error')[0]['hidden'])
         self.assert_correct_output('Incorrect Answer',
                                    result.get('error')[0]['message'])
 
@@ -448,7 +455,8 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest):
                            'weight': 1.0
                            },
                           {"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, 'weight': 1.0},
+                           "hook_code": hook_code, 'weight': 1.0,
+                           'hidden': True},
                           ]
         kwargs = {
                   'metadata': {
@@ -546,7 +554,8 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest):
                             """)
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0}]
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": False}]
 
         kwargs = {
                   'metadata': {
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 242d0b0..b7c6018 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -54,7 +54,7 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
             """)
         self.test_case_data = [{"test_case": self.tc_data,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0, "hidden": False
                                 }]
         self.in_dir = tmp_in_dir_path
         self.timeout_msg = ("Code took more than {0} seconds to run. "
@@ -184,9 +184,9 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
                 check(50, result);
             }
             """)
-        self.test_case_data = [{"test_case": self.tc_data,
+        test_case_data = [{"test_case": self.tc_data,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0,
                                 }]
         user_answer = dedent("""
             #include<stdio.h>
@@ -206,7 +206,7 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -257,9 +257,9 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
                         {
                         return a+b;
                         }""")
-        self.test_case_data = [{"test_case": self.tc_data,
+        test_case_data = [{"test_case": self.tc_data,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0,
                                 }]
         kwargs = {
                   'metadata': {
@@ -267,7 +267,7 @@ class CAssertionEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -287,6 +287,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                                 'expected_input': '5\n6',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
+                                'hidden': True
                                 }]
         self.in_dir = tempfile.mkdtemp()
         self.timeout_msg = ("Code took more than {0} seconds to run. "
@@ -324,7 +325,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_array_input(self):
         # Given
-        self.test_case_data = [{'expected_output': '561',
+        test_case_data = [{'expected_output': '561',
                                 'expected_input': '5\n6\n1',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
@@ -344,7 +345,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -356,7 +357,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_string_input(self):
         # Given
-        self.test_case_data = [{'expected_output': 'abc',
+        test_case_data = [{'expected_output': 'abc',
                                 'expected_input': 'abc',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
@@ -374,7 +375,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -408,6 +409,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
         result_error = result.get('error')[0].get('error_msg')
+        self.assertTrue(result.get('error')[0].get('hidden'))
         self.assertFalse(result.get('success'))
         self.assert_correct_output("Incorrect", result_error)
         self.assertTrue(lines_of_error > 0)
@@ -472,7 +474,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_only_stdout(self):
         # Given
-        self.test_case_data = [{'expected_output': '11',
+        test_case_data = [{'expected_output': '11',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
                                 }]
@@ -488,7 +490,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -522,11 +524,12 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
 
         # Then
+        self.assertFalse(result.get('hidden'))
         self.assertTrue(result.get('success'))
 
     def test_cpp_array_input(self):
         # Given
-        self.test_case_data = [{'expected_output': '561',
+        test_case_data = [{'expected_output': '561',
                                 'expected_input': '5\n6\n1',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
@@ -547,7 +550,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -559,7 +562,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_cpp_string_input(self):
         # Given
-        self.test_case_data = [{'expected_output': 'abc',
+        test_case_data = [{'expected_output': 'abc',
                                 'expected_input': 'abc',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
@@ -578,7 +581,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -613,6 +616,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
         result_error = result.get('error')[0].get('error_msg')
+        self.assertTrue(result.get('error')[0].get('hidden'))
         self.assertFalse(result.get('success'))
         self.assert_correct_output("Incorrect", result_error)
         self.assertTrue(lines_of_error > 0)
@@ -675,7 +679,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_cpp_only_stdout(self):
         # Given
-        self.test_case_data = [{'expected_output': '11',
+        test_case_data = [{'expected_output': '11',
                                'expected_input': '',
                                 'weight': 0.0,
                                 'test_case_type': 'stdiobasedtestcase',
@@ -693,7 +697,7 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
                     'file_paths': self.file_paths,
                     'partial_grading': False,
                     'language': 'cpp'
-                    }, 'test_case_data': self.test_case_data,
+                    }, 'test_case_data': test_case_data,
                   }
 
         # When
@@ -806,7 +810,8 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest):
                             """)
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0}]
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": True}]
         kwargs = {
                   'metadata': {
                     'user_answer': user_answer,
@@ -821,6 +826,7 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
 
         # Then
+        self.assertTrue(result.get('error')[0]['hidden'])
         self.assertFalse(result.get('success'))
         self.assert_correct_output('Incorrect Answer',
                                    result.get('error')[0]['message'])
@@ -1000,7 +1006,8 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest):
                             """)
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0}]
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": False}]
 
         kwargs = {
                   'metadata': {
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index 3e30ba3..e762852 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -62,7 +62,7 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         self.test_case_data = [
             {"test_case": self.tc_data,
              "test_case_type": "standardtestcase",
-             "weight": 0.0
+             "weight": 0.0, "hidden": False
              }
         ]
         self.in_dir = tmp_in_dir_path
@@ -119,6 +119,7 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         self.assertFalse(result.get('success'))
         for error in errors:
             self.assertEqual(error.get('exception'), 'AssertionError')
+            self.assertFalse(error.get('hidden'))
 
     def test_error(self):
         # Given
@@ -140,6 +141,7 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
         self.assertFalse(result.get("success"))
         for error in errors:
             self.assertEqual(error.get('exception'), 'CompilationError')
+            self.assertFalse(result.get('hidden'))
 
     def test_infinite_loop(self):
         # Given
@@ -276,7 +278,7 @@ class JavaAssertionEvaluationTestCases(EvaluatorBaseTest):
                        "{\n\treturn a;\n\t}\n}")
         self.test_case_data = [{"test_case": self.tc_data,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0, "hidden": True
                                 }]
         kwargs = {
                   'metadata': {
@@ -408,6 +410,7 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Then
         lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
         result_error = result.get('error')[0].get('error_msg')
+        self.assertFalse(result.get('error')[0].get('hidden'))
         self.assertFalse(result.get('success'))
         self.assert_correct_output("Incorrect", result_error)
         self.assertTrue(lines_of_error > 0)
@@ -437,6 +440,7 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         self.assertFalse(result.get("success"))
         for error in errors:
             self.assertEqual(error.get('exception'), 'CompilationError')
+            self.assertFalse(error.get('hidden'))
 
     def test_infinite_loop(self):
         # Given
@@ -474,7 +478,7 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
         # Given
         self.test_case_data = [{'expected_output': '11',
                                 'test_case_type': 'stdiobasedtestcase',
-                                'weight': 0.0
+                                'weight': 0.0, 'hidden': False
                                 }]
         user_answer = dedent("""
         class Test
@@ -677,7 +681,8 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest):
                             """)
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": True
                            }]
         kwargs = {
                   'metadata': {
@@ -693,6 +698,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
 
         # Then
+        self.assertTrue(result.get('error')[0]['hidden'])
         self.assertFalse(result.get('success'))
         self.assert_correct_output('Incorrect Answer',
                                    result.get('error')[0]['message'])
@@ -876,7 +882,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest):
                             """)
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0
+                           "hook_code": hook_code, "weight": 1.0,
                            }]
 
         kwargs = {
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 35c2322..343c8fb 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -25,13 +25,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         self.in_dir = tmp_in_dir_path
         self.test_case_data = [{"test_case_type": "standardtestcase",
                                 "test_case": 'assert(add(1,2)==3)',
-                                'weight': 0.0},
+                                'weight': 0.0, 'hidden': True},
                                {"test_case_type": "standardtestcase",
                                 "test_case": 'assert(add(-1,2)==1)',
-                                'weight': 0.0},
+                                'weight': 0.0, 'hidden': True},
                                {"test_case_type": "standardtestcase",
                                 "test_case":  'assert(add(-1,-2)==-3)',
-                                'weight': 0.0},
+                                'weight': 0.0, 'hidden': True},
                                ]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in"
@@ -80,6 +80,7 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
         for error in result.get("error"):
             self.assertEqual(error['exception'], 'AssertionError')
+            self.assertTrue(error['hidden'])
             self.assertEqual(
                 error['message'],
                 "Expected answer from the test case did not match the output"
@@ -92,13 +93,13 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
         test_case_data = [{"test_case_type": "standardtestcase",
                            "test_case": 'assert(add(-1,2)==1)',
-                           'weight': 1.0},
+                           'weight': 1.0, 'hidden': False},
                           {"test_case_type": "standardtestcase",
                            "test_case":  'assert(add(-1,-2)==-3)',
-                           'weight': 1.0},
+                           'weight': 1.0, 'hidden': False},
                           {"test_case_type": "standardtestcase",
                            "test_case": 'assert(add(1,2)==3)',
-                           'weight': 2.0}
+                           'weight': 2.0, 'hidden': False}
                           ]
         kwargs = {'metadata': {
                   'user_answer': user_answer,
@@ -119,6 +120,7 @@ class PythonAssertionEvaluationTestCases(EvaluatorBaseTest):
         given_test_case_list.remove('assert(add(1,2)==3)')
         for error in result.get("error"):
             self.assertEqual(error['exception'], 'AssertionError')
+            self.assertFalse(error['hidden'])
             self.assertEqual(
                 error['message'],
                 "Expected answer from the test case did not match the output"
@@ -489,7 +491,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_correct_answer_integer(self):
         # Given
-        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+        test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                 "expected_input": "1\n2",
                                 "expected_output": "3",
                                 "weight": 0.0
@@ -505,7 +507,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
                   'file_paths': self.file_paths,
                   'partial_grading': False,
                   'language': 'python'},
-                  'test_case_data': self.test_case_data
+                  'test_case_data': test_case_data
                   }
 
         # When
@@ -517,7 +519,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_correct_answer_list(self):
         # Given
-        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+        test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                 "expected_input": "1,2,3\n5,6,7",
                                 "expected_output": "[1, 2, 3, 5, 6, 7]",
                                 "weight": 0.0
@@ -536,7 +538,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
                   'file_paths': self.file_paths,
                   'partial_grading': False,
                   'language': 'python'},
-                  'test_case_data': self.test_case_data
+                  'test_case_data': test_case_data
                   }
 
         # When
@@ -548,7 +550,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_correct_answer_string(self):
         # Given
-        self.test_case_data = [{
+        test_case_data = [{
             "test_case_type": "stdiobasedtestcase",
             "expected_input": ("the quick brown fox jumps over "
                                "the lazy dog\nthe"),
@@ -567,7 +569,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
                   'file_paths': self.file_paths,
                   'partial_grading': False,
                   'language': 'python'},
-                  'test_case_data': self.test_case_data
+                  'test_case_data': test_case_data
                   }
 
         # When
@@ -579,10 +581,10 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_incorrect_answer_integer(self):
         # Given
-        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+        test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                 "expected_input": "1\n2",
                                 "expected_output": "3",
-                                "weight": 0.0
+                                "weight": 0.0, 'hidden': True
                                 }]
         user_answer = dedent("""
                                 a = int(input())
@@ -595,7 +597,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
                   'file_paths': self.file_paths,
                   'partial_grading': False,
                   'language': 'python'},
-                  'test_case_data': self.test_case_data
+                  'test_case_data': test_case_data
                   }
 
         # When
@@ -604,6 +606,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
+        self.assertTrue(result.get('error')[0].get('hidden'))
         self.assert_correct_output(
             "Incorrect Answer: Line number(s) 1 did not match.",
             result.get('error')[0].get('error_msg')
@@ -611,7 +614,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
 
     def test_file_based_answer(self):
         # Given
-        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
+        test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                 "expected_input": "",
                                 "expected_output": "2",
                                 "weight": 0.0
@@ -629,7 +632,7 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
                   'file_paths': self.file_paths,
                   'partial_grading': False,
                   'language': 'python'},
-                  'test_case_data': self.test_case_data
+                  'test_case_data': test_case_data
                   }
 
         # When
@@ -786,7 +789,8 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest):
                            )
 
         test_case_data = [{"test_case_type": "hooktestcase",
-                           "hook_code": hook_code, "weight": 1.0
+                           "hook_code": hook_code, "weight": 1.0,
+                           "hidden": True
                            }]
 
         kwargs = {'metadata': {
@@ -803,6 +807,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest):
 
         # Then
         self.assertFalse(result.get('success'))
+        self.assertTrue(result.get('error')[0]['hidden'])
         self.assert_correct_output('Incorrect Answer',
                                    result.get('error')[0]['message'])
 
diff --git a/yaksh/evaluator_tests/test_r_evaluation.py b/yaksh/evaluator_tests/test_r_evaluation.py
index b4b81ae..a196d91 100644
--- a/yaksh/evaluator_tests/test_r_evaluation.py
+++ b/yaksh/evaluator_tests/test_r_evaluation.py
@@ -44,7 +44,7 @@ class RAssertionEvaluationTestCase(EvaluatorBaseTest):
         )
         self.test_case_data = [{"test_case": self.test_case,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0, "hidden": True
                                 }]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in"
@@ -108,6 +108,7 @@ class RAssertionEvaluationTestCase(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
         errors = result.get('error')
         # Then
+        self.assertTrue(result.get("error")[0]['hidden'])
         self.assertFalse(result.get('success'))
         self.assertEqual(errors[0]['message'], err)
 
@@ -134,6 +135,7 @@ class RAssertionEvaluationTestCase(EvaluatorBaseTest):
         errors = result.get('error')
 
         # Then
+        self.assertTrue(result.get("error")[0]['hidden'])
         self.assertFalse(result.get("success"))
         self.assertIn("object 'a' not found", errors[0]['message'])
 
@@ -159,6 +161,7 @@ class RAssertionEvaluationTestCase(EvaluatorBaseTest):
         errors = result.get('error')
 
         # Then
+        self.assertTrue(result.get("error")[0]['hidden'])
         self.assertFalse(result.get("success"))
         err = errors[0]['message']
         self.assertIn("is.null(obj) == FALSE is not TRUE", err)
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 41abf94..b08d348 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -48,7 +48,7 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest):
             """)
         self.test_case_data = [{"test_case": self.tc_data,
                                 "test_case_type": "standardtestcase",
-                                "weight": 0.0
+                                "weight": 0.0, 'hidden': True
                                 }]
         self.in_dir = tmp_in_dir_path
         self.file_paths = None
@@ -93,6 +93,7 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest):
         grader = Grader(self.in_dir)
         result = grader.evaluate(kwargs)
         self.assertFalse(result.get("success"))
+        self.assertTrue(result.get("error")[0]['hidden'])
         self.assert_correct_output('error', result.get("error")[0]['message'])
 
     def test_incorrect_answer(self):
@@ -111,6 +112,7 @@ class ScilabEvaluationTestCases(EvaluatorBaseTest):
         result = grader.evaluate(kwargs)
         lines_of_error = len(result.get('error')[0]['message'].splitlines())
         self.assertFalse(result.get('success'))
+        self.assertTrue(result.get("error")[0]['hidden'])
         self.assert_correct_output("Message",
                                    result.get('error')[0]["message"])
         self.assertTrue(lines_of_error > 1)