diff options
| -rw-r--r-- | yaksh/bash_code_evaluator.py | 2 | ||||
| -rw-r--r-- | yaksh/bash_stdio_evaluator.py | 2 | ||||
| -rw-r--r-- | yaksh/code_evaluator.py | 1 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_bash_evaluation.py | 59 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 125 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_java_evaluation.py | 89 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 103 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_scilab_evaluation.py | 16 | ||||
| -rw-r--r-- | yaksh/python_stdio_evaluator.py | 13 | ||||
| -rw-r--r-- | yaksh/tests/test_code_server.py | 24 |
10 files changed, 291 insertions, 143 deletions
diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 7dc0f0f..4ca5445 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -28,7 +28,7 @@ class BashCodeEvaluator(CodeEvaluator): delete_files(self.files) super(BashCodeEvaluator, self).teardown() - def check_code(self, file_paths, partial_grading, test_case, marks): + def check_code(self, user_answer, file_paths, partial_grading, test_case, marks): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 0bab0f0..2826a6b 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -50,4 +50,4 @@ class BashStdioEvaluator(StdIOEvaluator): expected_output ) test_case_marks = float(marks) if partial_grading and success else 0.0 - return success, err + return success, err, test_case_marks diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index b404d57..1672efa 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -123,6 +123,7 @@ class CodeEvaluator(object): ) if success: marks += test_case_marks + error = err else: error += err + "\n" diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 66ade19..5f4be4b 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -13,12 +13,12 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) - tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ - {"test_case": "bash_files/sample.sh,bash_files/sample.args"} + {"test_case": "bash_files/sample.sh,bash_files/sample.args", + "marks": 0.0 + } ] - tmp_in_dir_path = tempfile.mkdtemp() - self.in_dir = tmp_in_dir_path + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) @@ -33,7 +33,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -45,7 +46,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -57,7 +59,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -68,11 +71,14 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ - {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"} + {"test_case": "bash_files/sample1.sh,bash_files/sample1.args", + "marks": 0.0 + } ] user_answer = ("#!/bin/bash\ncat $1") get_class = BashCodeEvaluator() kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -82,6 +88,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): class BashStdioEvaluationTestCases(unittest.TestCase): def setUp(self): + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) @@ -93,11 +100,15 @@ class BashStdioEvaluationTestCases(unittest.TestCase): echo -n `expr $A + $B` """ ) - test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'marks': 0.0 + }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -112,12 +123,14 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', - 'expected_input': '1,2,3\n4,5,6\n7,8,9' + 'expected_input': '1,2,3\n4,5,6\n7,8,9', + 'marks': 0.0 }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -129,11 +142,15 @@ class BashStdioEvaluationTestCases(unittest.TestCase): echo -n `expr $A - $B` """ ) - test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'marks': 0.0 + }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -146,12 +163,14 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '10', - 'expected_input': '' + 'expected_input': '', + 'marks': 0.0 }] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, - "test_case_data": test_case_data - } + "partial_grading": True, + "test_case_data": test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index c990436..79326d4 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -14,7 +14,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() - self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] + self.test_case_data = [{"test_case": "c_cpp_files/main.cpp", + "marks": 0.0 + }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" @@ -29,6 +31,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -40,6 +43,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a-b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -52,7 +56,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -63,7 +68,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': False, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -73,7 +79,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] - self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}] + self.test_case_data = [{"test_case": "c_cpp_files/file_data.c", + "marks": 0.0 + }] user_answer = dedent(""" #include<stdio.h> char ans() @@ -88,18 +96,21 @@ class CAssertionEvaluationTestCases(unittest.TestCase): """) get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': False, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer") class CppStdioEvaluationTestCases(unittest.TestCase): - def setUp(self): - self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] - self.in_dir = os.getcwd() + self.test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6', + 'marks': 0.0 + }] + self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) @@ -114,15 +125,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'marks': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -134,15 +148,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'abc', - 'expected_input': 'abc'}] + 'expected_input': 'abc', + 'marks': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -152,8 +169,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -167,8 +185,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -184,8 +203,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -199,15 +219,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'marks': 0.0 + }] user_answer = dedent(""" #include<stdio.h> int main(void){ @@ -216,8 +239,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -233,15 +257,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_cpp_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'marks': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -254,15 +281,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_cpp_string_input(self): self.test_case_data = [{'expected_output': 'abc', - 'expected_input': 'abc'}] + 'expected_input': 'abc', + 'marks': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -273,8 +303,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -289,8 +320,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -307,8 +339,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -323,15 +356,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_cpp_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'marks': 0.0 + }] user_answer = dedent(""" #include<iostream> using namespace std; @@ -341,8 +377,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): }""") get_class = CppStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': False, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index e375bdb..33e0e35 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -16,7 +16,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ - {"test_case": "java_files/main_square.java"} + {"test_case": "java_files/main_square.java", + "marks": 0.0 + } ] self.in_dir = tmp_in_dir_path evaluator.SERVER_TIMEOUT = 9 @@ -32,7 +34,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -43,7 +46,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -57,7 +61,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_error(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -68,7 +73,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -79,7 +85,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [ - {"test_case": "java_files/read_file.java"} + {"test_case": "java_files/read_file.java", + "marks": 0.0 + } ] user_answer = dedent(""" import java.io.BufferedReader; @@ -101,9 +109,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): """) get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': True, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") @@ -116,7 +125,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path self.test_case_data = [{'expected_output': '11', - 'expected_input': '5\n6'}] + 'expected_input': '5\n6', + 'marks': 0.0 + }] evaluator.SERVER_TIMEOUT = 4 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -139,8 +150,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -148,7 +160,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5\n6\n1'}] + 'expected_input': '5\n6\n1', + 'marks': 0.0 + }] user_answer = dedent(""" import java.util.Scanner; class Test @@ -161,8 +175,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -180,8 +195,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -197,8 +213,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -214,15 +231,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': ''}] + 'expected_input': '', + 'marks': 0.0 + }] user_answer = dedent(""" class Test {public static void main(String[] args){ @@ -232,15 +252,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'HelloWorld', - 'expected_input': 'Hello\nWorld'}] + 'expected_input': 'Hello\nWorld', + 'marks': 0.0 + }] user_answer = dedent(""" import java.util.Scanner; class Test @@ -252,8 +275,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): }}""") get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'partial_grading': True, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -261,7 +285,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): def test_file_based_stdout(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [{'expected_output': '2', - 'expected_input': ''}] + 'expected_input': '', + 'marks': 0.0 + }] user_answer = dedent(""" import java.io.BufferedReader; import java.io.FileReader; @@ -282,9 +308,10 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): """) get_class = JavaStdioEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } + 'partial_grading': True, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer") diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 45cc40d..da64dc4 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -17,9 +17,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, + self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'marks': 0.0}, + {"test_case": 'assert(add(-1,2)==1)', 'marks': 0.0}, + {"test_case": 'assert(add(-1,-2)==-3)', 'marks': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -35,7 +35,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): user_answer = "def add(a,b):\n\treturn a + b" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -51,7 +52,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): user_answer = "def add(a,b):\n\treturn a - b" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -59,17 +61,22 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): result = evaluator.evaluate(**kwargs) # Then + print repr(result.get('error')) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), - "AssertionError in: assert(add(1,2)==3)" - ) + ('AssertionError in: assert(add(1,2)==3)\n' + 'AssertionError in: assert(add(-1,2)==1)\n' + 'AssertionError in: assert(add(-1,-2)==-3)\n' + ) + ) def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -96,7 +103,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -125,7 +133,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False + } # When @@ -150,7 +160,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -176,7 +187,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -202,7 +214,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -231,7 +244,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -247,7 +261,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')"}] + self.test_case_data = [{"test_case": "assert(ans()=='2')", "marks": 0.0}] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): @@ -256,7 +270,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -272,7 +287,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'} + test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + "marks": 0.0 + } ] syntax_error_msg = ["Traceback", "call", @@ -284,7 +301,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -304,8 +322,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'}, - {"test_case": 's="abbb"\nassert palindrome(S)==False'} + test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + "marks": 0.0 + }, + {"test_case": 's="abbb"\nassert palindrome(S)==False', + "marks": 0.0 + } ] name_error_msg = ["Traceback", "call", @@ -317,7 +339,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] kwargs = {'user_answer': user_answer, 'test_case_data': test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -341,7 +364,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_integer(self): # Given self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" + "expected_output": "3", + "marks": 0.0 }] user_answer = dedent(""" a = int(input()) @@ -350,7 +374,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -364,7 +389,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", - "expected_output": "[1, 2, 3, 5, 6, 7]" + "expected_output": "[1, 2, 3, 5, 6, 7]", + "marks": 0.0 }] user_answer = dedent(""" from six.moves import input @@ -376,7 +402,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -390,7 +417,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), - "expected_output": "2" + "expected_output": "2", + "marks": 0.0 }] user_answer = dedent(""" from six.moves import input @@ -400,7 +428,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): """ ) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -414,7 +443,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_incorrect_answer_integer(self): # Given self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" + "expected_output": "3", + "marks": 0.0 }] user_answer = dedent(""" a = int(input()) @@ -424,6 +454,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): ) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, + 'partial_grading': False } # When @@ -436,7 +467,10 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given - self.test_case_data = [{"expected_input": "", "expected_output": "2"}] + self.test_case_data = [{"expected_input": "", + "expected_output": "2", + "marks": 0.0 + }] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" @@ -447,7 +481,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): ) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths + 'file_paths': self.file_paths, + 'partial_grading': False } # When @@ -461,14 +496,16 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): # Given test_case_data = [{"expected_input": "1\n2", - "expected_output": "3" - }] + "expected_output": "3", + "marks": 0.0 + }] timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data + 'test_case_data': test_case_data, + 'partial_grading': False } # When diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index b366480..b46a65b 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -11,7 +11,9 @@ from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): tmp_in_dir_path = tempfile.mkdtemp() - self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] + self.test_case_data = [{"test_case": "scilab_files/test_add.sce", + "marks": 0.0 + }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" @@ -25,7 +27,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -37,7 +40,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\ndis(\tendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -50,7 +54,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a-b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } @@ -64,7 +69,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, + 'partial_grading': True, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index cbbbfd6..7df0ba1 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -31,6 +31,10 @@ def redirect_stdout(): class PythonStdioEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" + def setup(self): + super(PythonStdioEvaluator, self).setup() + self.files = [] + def teardown(self): # Delete the created file. if self.files: @@ -38,8 +42,7 @@ class PythonStdioEvaluator(CodeEvaluator): super(PythonStdioEvaluator, self).teardown() - def compile_code(self, user_answer, file_paths, expected_input, expected_output): - self.files = [] + def compile_code(self, user_answer, file_paths, expected_input, expected_output, marks): if file_paths: self.files = copy_files(file_paths) submitted = compile(user_answer, '<string>', mode='exec') @@ -54,13 +57,15 @@ class PythonStdioEvaluator(CodeEvaluator): self.output_value = output_buffer.getvalue().rstrip("\n") return self.output_value - def check_code(self, user_answer, file_paths, expected_input, expected_output): + def check_code(self, user_answer, file_paths, partial_grading, expected_input, expected_output, marks): success = False + test_case_marks = 0.0 tb = None if self.output_value == expected_output: success = True err = "Correct answer" + test_case_marks = marks else: success = False err = dedent(""" @@ -74,4 +79,4 @@ class PythonStdioEvaluator(CodeEvaluator): ) ) del tb - return success, err + return success, err, test_case_marks diff --git a/yaksh/tests/test_code_server.py b/yaksh/tests/test_code_server.py index 8835110..d446444 100644 --- a/yaksh/tests/test_code_server.py +++ b/yaksh/tests/test_code_server.py @@ -38,7 +38,11 @@ class TestCodeServer(unittest.TestCase): def test_inifinite_loop(self): # Given testdata = {'user_answer': 'while True: pass', - 'test_case_data': [{'test_case':'assert 1==2'}]} + 'partial_grading': True, + 'test_case_data': [{'test_case':'assert 1==2', + 'marks': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -53,7 +57,11 @@ class TestCodeServer(unittest.TestCase): def test_correct_answer(self): # Given testdata = {'user_answer': 'def f(): return 1', - 'test_case_data': [{'test_case':'assert f() == 1'}]} + 'partial_grading': True, + 'test_case_data': [{'test_case':'assert f() == 1', + 'marks': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -68,7 +76,11 @@ class TestCodeServer(unittest.TestCase): def test_wrong_answer(self): # Given testdata = {'user_answer': 'def f(): return 1', - 'test_case_data': [{'test_case':'assert f() == 2'}]} + 'partial_grading': True, + 'test_case_data': [{'test_case':'assert f() == 2', + 'marks': 0.0 + } + ]} # When result = self.code_server.run_code( @@ -87,7 +99,11 @@ class TestCodeServer(unittest.TestCase): def run_code(): """Run an infinite loop.""" testdata = {'user_answer': 'while True: pass', - 'test_case_data': [{'test_case':'assert 1==2'}]} + 'partial_grading': True, + 'test_case_data': [{'test_case':'assert 1==2', + 'marks': 0.0 + } + ]} result = self.code_server.run_code( 'python', 'standardtestcase', json.dumps(testdata), '' ) |