Multiple test cases passed as dicts, check_code() is iterated based on no. of test cases

13 files changed, 45 insertions, 35 deletions

diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py
index fbfa232..b649af7 100644
--- a/yaksh/bash_code_evaluator.py
+++ b/yaksh/bash_code_evaluator.py
@@ -37,7 +37,7 @@ class BashCodeEvaluator(CodeEvaluator):
         super(BashCodeEvaluator, self).teardown()
         os.remove(self.submit_code_path)
 
-    def check_code(self, user_answer, test_case_data):
+    def check_code(self, user_answer, test_case):
         """ Function validates student script using instructor script as
         reference. Test cases can optionally be provided.  The first argument
         ref_path, is the path to instructor script, it is assumed to
@@ -60,7 +60,8 @@ class BashCodeEvaluator(CodeEvaluator):
         the required permissions are not given to the file(s).
 
         """
-        ref_code_path = test_case_data[0]
+        # ref_code_path = test_case_data[0]
+        ref_code_path = test_case
         get_ref_path, get_test_case_path = ref_code_path.strip().split(',')
         get_ref_path = get_ref_path.strip()
         get_test_case_path = get_test_case_path.strip()
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py
index b777b67..39f0d48 100644
--- a/yaksh/code_evaluator.py
+++ b/yaksh/code_evaluator.py
@@ -115,7 +115,8 @@ class CodeEvaluator(object):
         self._change_dir(self.in_dir)
 
     # def safe_evaluate(self, args):
-    def safe_evaluate(self, **kwargs):
+    # def safe_evaluate(self, **kwargs): #@@@v2
+    def safe_evaluate(self, user_answer, test_case_data):
         # Add a new signal handler for the execution of this code.
         prev_handler = create_signal_handler()
         success = False
@@ -124,7 +125,11 @@ class CodeEvaluator(object):
         # Do whatever testing needed.
         try:
             # success, err = self.check_code(*args)
-            success, err = self.check_code(**kwargs)
+            # success, err = self.check_code(**kwargs) #@@@v2
+            for test_case in test_case_data:
+                success, err = self.check_code(user_answer, **test_case)
+                if not success:
+                    break
 
         except TimeoutException:
             err = self.timeout_msg
diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py
index 6d18982..8cdc27c 100644
--- a/yaksh/cpp_code_evaluator.py
+++ b/yaksh/cpp_code_evaluator.py
@@ -36,7 +36,7 @@ class CppCodeEvaluator(CodeEvaluator):
                                                 ref_output_path)
         return compile_command, compile_main
 
-    def check_code(self, user_answer, test_case_data):
+    def check_code(self, user_answer, test_case):
             """ Function validates student code using instructor code as
             reference.The first argument ref_code_path, is the path to
             instructor code, it is assumed to have executable permission.
@@ -56,7 +56,7 @@ class CppCodeEvaluator(CodeEvaluator):
             if the required permissions are not given to the file(s).
 
             """
-            ref_code_path = test_case_data[0]
+            ref_code_path = test_case
             clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path)
 
             if not isfile(clean_ref_code_path):
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 5fa3d2d..39247f7 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -5,7 +5,7 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class BashEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.test_case_data = ["bash_files/sample.sh,bash_files/sample.args"]
+        self.test_case_data = [{"test_case": "bash_files/sample.sh,bash_files/sample.args"}]
         self.in_dir = "/tmp"
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 9d37fb4..a07150d 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -5,7 +5,7 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class CEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.test_case_data = ["c_cpp_files/main.cpp"]
+        self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
         self.in_dir = "/tmp"
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
@@ -23,7 +23,7 @@ class CEvaluationTestCases(unittest.TestCase):
     def test_incorrect_answer(self):
         user_answer = "int add(int a, int b)\n{return a-b;}"
         get_class = CppCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer, 
+        kwargs = {'user_answer': user_answer,
                     'test_case_data': self.test_case_data
                 }
         result = get_class.evaluate(**kwargs)
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index a5852a3..eacd62e 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -7,7 +7,7 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class JavaEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.test_case_data = ["java_files/main_square.java"]
+        self.test_case_data = [{"test_case": "java_files/main_square.java"}]
         self.in_dir = "/tmp"
         evaluator.SERVER_TIMEOUT = 9
         self.timeout_msg = ("Code took more than {0} seconds to run. "
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index b23942f..39d4723 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -8,7 +8,10 @@ from textwrap import dedent
 
 class PythonAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.test_case_data = ['assert(add(1,2)==3)']
+        self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
+                                {"test_case": 'assert(add(-1,2)==1)'},
+                                {"test_case":  'assert(add(-1,-2)==-3)'},
+                            ]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
 
@@ -148,7 +151,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
 
 class PythonStdoutEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.output = ['0 1 1 2 3']
+        self.test_case_data = [{"expected_output": "0 1 1 2 3"}]
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
 
@@ -157,7 +160,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
         user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
         get_class = PythonStdoutEvaluator()
         kwargs = {'user_answer': user_answer,
-                    'test_case_data': self.output
+                    'test_case_data': self.test_case_data
                 }
         result = get_class.evaluate(**kwargs)
         self.assertEqual(result.get('error'), "Correct answer")
@@ -167,7 +170,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
         user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
         get_class = PythonStdoutEvaluator()
         kwargs = {'user_answer': user_answer, 
-                    'test_case_data': self.output
+                    'test_case_data': self.test_case_data
                 }
         result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get('success'))
@@ -178,7 +181,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
         error_msg = "Incorrect Answer: Please avoid printing the expected output directly"
         get_class = PythonStdoutEvaluator()
         kwargs = {'user_answer': user_answer, 
-                    'test_case_data': self.output
+                    'test_case_data': self.test_case_data
                 }
         result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get('success'))
@@ -188,7 +191,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
         user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
         get_class = PythonStdoutEvaluator()
         kwargs = {'user_answer': user_answer,
-                    'test_case_data': self.output
+                    'test_case_data': self.test_case_data
                 }
         result = get_class.evaluate(**kwargs)
         self.assertFalse(result.get('success'))
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 2b459fc..928cd7e 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -6,7 +6,7 @@ from yaksh.settings import SERVER_TIMEOUT
 
 class ScilabEvaluationTestCases(unittest.TestCase):
     def setUp(self):
-        self.test_case_data = ["scilab_files/test_add.sce"]
+        self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
         self.in_dir = "/tmp"
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py
index e99448c..b325208 100644
--- a/yaksh/java_code_evaluator.py
+++ b/yaksh/java_code_evaluator.py
@@ -29,7 +29,7 @@ class JavaCodeEvaluator(CodeEvaluator):
                                              user_code_directory)
         return compile_command, compile_main
 
-    def check_code(self, user_answer, test_case_data):
+    def check_code(self, user_answer, test_case):
         """ Function validates student code using instructor code as
         reference.The first argument ref_code_path, is the path to
         instructor code, it is assumed to have executable permission.
@@ -49,7 +49,7 @@ class JavaCodeEvaluator(CodeEvaluator):
         if the required permissions are not given to the file(s).
 
         """
-        ref_code_path = test_case_data[0]
+        ref_code_path = test_case
         clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path)
 
         if not isfile(clean_ref_code_path):
diff --git a/yaksh/models.py b/yaksh/models.py
index 331446f..fd60036 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -228,8 +228,8 @@ class Question(models.Model):
 
         test_cases = self.testcase_set.all()
         for test in test_cases:
-            test_instance = test.get_child_instance(self.test_case_type)
-            test_case_field_value = test_instance.get_field_value()
+            test_case_child_instance = test.get_child_instance(self.test_case_type)
+            test_case_instance_dict = test_case_child_instance.get_instance_as_dict()
             test_case_data.append(test_case_field_value)
 
         question_data['test_case_data'] = test_case_data
@@ -779,17 +779,18 @@ class StandardTestCase(TestCase):
     test_case = models.TextField(blank=True)
 
     def get_field_value(self):
-        return self.test_case
+        return {"test_case": self.test_case}
+
 
 class StdoutBasedTestCase(TestCase):
-    output = models.TextField(blank=True)
+    expected_output = models.TextField(blank=True)
 
     def get_field_value(self):
-        return self.output
+        return {"expected_output": self.expected_output}
 
 class McqTestCase(TestCase):
     options = models.TextField()
     correct = models.BooleanField(default=False)
 
-    def validate(self, user_answer):
-        pass
+    def get_field_value(self):
+        return {"options": self.options, "correct": self.correct}
diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py
index 0615d84..15ff8fd 100644
--- a/yaksh/python_assertion_evaluator.py
+++ b/yaksh/python_assertion_evaluator.py
@@ -13,21 +13,21 @@ class PythonAssertionEvaluator(CodeEvaluator):
     """Tests the Python code obtained from Code Server"""
 
     # def check_code(self, test, user_answer, ref_code_path):
-    def check_code(self, user_answer, test_case_data):
+    # def check_code(self, user_answer, test_case_data): #@@@v2
+    def check_code(self, user_answer, test_case):
         success = False
         try:
             tb = None
             submitted = compile(user_answer, '<string>', mode='exec')
             g = {}
             exec submitted in g
-            for test_code in test_case_data:
-                _tests = compile(test_code, '<string>', mode='exec')
-                exec _tests in g
+            _tests = compile(test_case, '<string>', mode='exec')
+            exec _tests in g
         except AssertionError:
             type, value, tb = sys.exc_info()
             info = traceback.extract_tb(tb)
             fname, lineno, func, text = info[-1]
-            text = str(test_code).splitlines()[lineno-1]
+            text = str(test_case).splitlines()[lineno-1]
             err = "{0} {1} in: {2}".format(type.__name__, str(value), text)
         except TimeoutException:
             raise
diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py
index 815752f..9443e37 100644
--- a/yaksh/python_stdout_evaluator.py
+++ b/yaksh/python_stdout_evaluator.py
@@ -25,11 +25,11 @@ def redirect_stdout():
 class PythonStdoutEvaluator(CodeEvaluator):
     """Tests the Python code obtained from Code Server"""
 
-    def check_code(self, user_answer, test_case_data):
+    def check_code(self, user_answer, expected_output):
         success = False
 
         tb = None
-        expected_output = test_case_data[0]
+        # expected_output = test_case_data[0]
         submitted = compile(user_answer, '<string>', mode='exec')
         with redirect_stdout() as output_buffer:
             g = {}
diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py
index 6ddfa5a..61642fd 100644
--- a/yaksh/scilab_code_evaluator.py
+++ b/yaksh/scilab_code_evaluator.py
@@ -33,8 +33,8 @@ class ScilabCodeEvaluator(CodeEvaluator):
         # Delete the created file.
         os.remove(self.submit_code_path)
 
-    def check_code(self, user_answer, test_case_data):
-        ref_code_path = test_case_data[0]
+    def check_code(self, user_answer, test_case):
+        ref_code_path = test_case
         clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path)
         user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip())