Merge pull request #163 from ankitjavalkar/code-server-refactor2016-form

Code Evaluator refactoring

1 files changed, 195 insertions, 103 deletions

diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index f7ecd97..36eb6a5 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -3,12 +3,14 @@ import unittest
 import os
 import shutil
 import tempfile
-from yaksh import code_evaluator as evaluator
-from yaksh.java_code_evaluator import JavaCodeEvaluator
-from yaksh.java_stdio_evaluator import JavaStdioEvaluator
-from yaksh.settings import SERVER_TIMEOUT
 from textwrap import dedent
 
+# Local Import
+from yaksh import grader as gd
+from yaksh.grader import Grader
+from yaksh.java_code_evaluator import JavaCodeEvaluator
+from yaksh.java_stdio_evaluator import JavaStdIOEvaluator
+
 
 class JavaAssertionEvaluationTestCases(unittest.TestCase):
     def setUp(self):
@@ -17,41 +19,56 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
         tmp_in_dir_path = tempfile.mkdtemp()
         self.test_case_data = [
             {"test_case": "java_files/main_square.java",
+            "test_case_type": "standardtestcase",
             "weight": 0.0
             }
         ]
         self.in_dir = tmp_in_dir_path
-        evaluator.SERVER_TIMEOUT = 9
+        self.file_paths = None
+        gd.SERVER_TIMEOUT = 9
         self.timeout_msg = ("Code took more than {0} seconds to run. "
             "You probably have an infinite loop in"
-            " your code.").format(evaluator.SERVER_TIMEOUT)
-        self.file_paths = None
+            " your code.").format(gd.SERVER_TIMEOUT)
+
 
     def tearDown(self):
+        gd.SERVER_TIMEOUT = 4
         os.remove('/tmp/test.txt')
         shutil.rmtree(self.in_dir)
 
     def test_correct_answer(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
-        get_class = JavaCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs)
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_incorrect_answer(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
-        get_class = JavaCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs)
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertFalse(result.get('success'))
         lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
@@ -60,25 +77,37 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
 
     def test_error(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
-        get_class = JavaCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs) 
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertFalse(result.get("success"))
         self.assertTrue("Error" in result.get("error"))
 
     def test_infinite_loop(self):
         user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
-        get_class = JavaCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs) 
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertFalse(result.get("success"))
         self.assertEqual(result.get("error"), self.timeout_msg)
 
@@ -86,6 +115,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
         self.file_paths = [("/tmp/test.txt", False)]
         self.test_case_data = [
             {"test_case": "java_files/read_file.java",
+            "test_case_type": "standardtestcase",
             "weight": 0.0
             }
         ]
@@ -107,34 +137,41 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
                 br.close();
             }}}
             """)
-        get_class = JavaCodeEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs)
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertTrue(result.get("success"))
         self.assertEqual(result.get("error"), "Correct answer\n")
 
-class JavaStdioEvaluationTestCases(unittest.TestCase):
-
+class JavaStdIOEvaluationTestCases(unittest.TestCase):
     def setUp(self):
         with open('/tmp/test.txt', 'wb') as f:
             f.write('2'.encode('ascii'))
         tmp_in_dir_path = tempfile.mkdtemp()
         self.in_dir = tmp_in_dir_path
         self.test_case_data = [{'expected_output': '11',
-                               'expected_input': '5\n6',
-                               'weight': 0.0
-                               }]
-        evaluator.SERVER_TIMEOUT = 4
+                                'expected_input': '5\n6',
+                                'test_case_type': 'stdiobasedtestcase',
+                                'weight': 0.0
+                            }]
+        self.file_paths = None
+        gd.SERVER_TIMEOUT = 9
         self.timeout_msg = ("Code took more than {0} seconds to run. "
                             "You probably have an infinite loop in"
-                            " your code.").format(evaluator.SERVER_TIMEOUT)
+                            " your code.").format(gd.SERVER_TIMEOUT)
 
     def tearDown(self):
-        evaluator.SERVER_TIMEOUT = 4
+        gd.SERVER_TIMEOUT = 4
         os.remove('/tmp/test.txt')
         shutil.rmtree(self.in_dir)
 
@@ -148,19 +185,26 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          int b = s.nextInt();
          System.out.print(a+b);
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_array_input(self):
-
         self.test_case_data = [{'expected_output': '561',
                                 'expected_input': '5\n6\n1',
+                                'test_case_type': 'stdiobasedtestcase',
                                 'weight': 0.0
                                 }]
         user_answer = dedent("""
@@ -173,17 +217,23 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          a[i] = s.nextInt();
          System.out.print(a[i]);}
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_incorrect_answer(self):
-
         user_answer = dedent("""
         import java.util.Scanner;
         class Test
@@ -193,35 +243,47 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          int b = s.nextInt();
          System.out.print(a);
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         lines_of_error = len(result.get('error').splitlines())
         self.assertFalse(result.get('success'))
         self.assertIn("Incorrect", result.get('error'))
         self.assertTrue(lines_of_error > 1)
 
     def test_error(self):
-
         user_answer = dedent("""
         class Test
         {
          System.out.print("a");
         }""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertFalse(result.get("success"))
         self.assertTrue("Compilation Error" in result.get("error"))
 
     def test_infinite_loop(self):
-
         user_answer = dedent("""
         class Test
         {public static void main(String[] args){
@@ -229,19 +291,27 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          {
          System.out.print("a");}
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertFalse(result.get("success"))
         self.assertEqual(result.get("error"), self.timeout_msg)
 
     def test_only_stdout(self):
         self.test_case_data = [{'expected_output': '11',
-                               'expected_input': '',
-                               'weight': 0.0
+                                'expected_input': '',
+                                'test_case_type': 'stdiobasedtestcase',
+                                'weight': 0.0
                                }]
         user_answer = dedent("""
         class Test
@@ -250,19 +320,27 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          int b = 6;
          System.out.print(a+b);
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_string_input(self):
         self.test_case_data = [{'expected_output': 'HelloWorld',
-                               'expected_input': 'Hello\nWorld',
-                               'weight': 0.0
+                                'expected_input': 'Hello\nWorld',
+                                'test_case_type': 'stdiobasedtestcase',
+                                'weight': 0.0
                                }]
         user_answer = dedent("""
         import java.util.Scanner;
@@ -273,20 +351,28 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
          String b = s.nextLine();
          System.out.print(a+b);
         }}""")
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
-                    'test_case_data': self.test_case_data
-                }
-        result = get_class.evaluate(**kwargs)
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
+                    'test_case_data': self.test_case_data,
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertEqual(result.get('error'), "Correct answer\n")
         self.assertTrue(result.get('success'))
 
     def test_file_based_stdout(self):
         self.file_paths = [("/tmp/test.txt", False)]
         self.test_case_data = [{'expected_output': '2',
-                               'expected_input': '',
-                               'weight': 0.0
+                                'expected_input': '',
+                                'test_case_type': 'stdiobasedtestcase',
+                                'weight': 0.0
                                }]
         user_answer = dedent("""
             import java.io.BufferedReader;
@@ -306,13 +392,19 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
                 br.close();
             }}}
             """)
-        get_class = JavaStdioEvaluator(self.in_dir)
-        kwargs = {'user_answer': user_answer,
-                    'partial_grading': True,
+        kwargs = {
+                  'metadata': {
+                    'user_answer': user_answer,
+                    'file_paths': self.file_paths,
+                    'partial_grading': False,
+                    'language': 'java'
+                    },
                     'test_case_data': self.test_case_data,
-                    'file_paths': self.file_paths
-                }
-        result = get_class.evaluate(**kwargs)
+                  }
+
+        grader = Grader(self.in_dir)
+        result = grader.evaluate(kwargs)
+
         self.assertTrue(result.get("success"))
         self.assertEqual(result.get("error"), "Correct answer\n")