diff options
Diffstat (limited to 'yaksh/evaluator_tests')
| -rw-r--r-- | yaksh/evaluator_tests/test_bash_evaluation.py | 163 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 356 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_code_evaluation.py | 12 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_java_evaluation.py | 298 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 378 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_scilab_evaluation.py | 83 |
6 files changed, 852 insertions, 438 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 99e5122..06a56e4 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -3,8 +3,9 @@ import unittest import os import shutil import tempfile +from yaksh.grader import Grader from yaksh.bash_code_evaluator import BashCodeEvaluator -from yaksh.bash_stdio_evaluator import BashStdioEvaluator +from yaksh.bash_stdio_evaluator import BashStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent @@ -15,6 +16,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] @@ -32,39 +34,57 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" ) - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") def test_error(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") - get_class = BashCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -72,26 +92,35 @@ class BashAssertionEvaluationTestCases(unittest.TestCase): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args", + "test_case_type": "standardtestcase", "weight": 0.0 } ] user_answer = ("#!/bin/bash\ncat $1") - get_class = BashCodeEvaluator() - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class BashStdioEvaluationTestCases(unittest.TestCase): +class BashStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) + self.file_paths = None + def test_correct_answer(self): user_answer = dedent(""" #!/bin/bash @@ -102,14 +131,22 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -123,15 +160,23 @@ class BashStdioEvaluationTestCases(unittest.TestCase): """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', - 'expected_input': '1,2,3\n4,5,6\n7,8,9', - 'weight': 0.0 + 'expected_input': '1,2,3\n4,5,6\n7,8,9', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -144,14 +189,21 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertIn("Incorrect", result.get('error')) self.assertFalse(result.get('success')) @@ -164,14 +216,21 @@ class BashStdioEvaluationTestCases(unittest.TestCase): ) test_case_data = [{'expected_output': '10', 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] - get_class = BashStdioEvaluator() - kwargs = {"user_answer": user_answer, - "partial_grading": True, - "test_case_data": test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d5193d3..dc6fdc9 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -3,10 +3,14 @@ import unittest import os import shutil import tempfile +from textwrap import dedent + +# Local import +from yaksh.grader import Grader from yaksh.cpp_code_evaluator import CppCodeEvaluator -from yaksh.cpp_stdio_evaluator import CppStdioEvaluator +from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT -from textwrap import dedent + class CAssertionEvaluationTestCases(unittest.TestCase): @@ -15,6 +19,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "c_cpp_files/main.cpp", + "test_case_type": "standardtestcase", "weight": 0.0 }] self.in_dir = tmp_in_dir_path @@ -29,25 +34,37 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': False, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a-b;}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect:", result.get('error')) @@ -55,31 +72,44 @@ class CAssertionEvaluationTestCases(unittest.TestCase): def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_file_based_assert(self): self.file_paths = [('/tmp/test.txt', False)] self.test_case_data = [{"test_case": "c_cpp_files/file_data.c", + "test_case_type": "standardtestcase", "weight": 0.0 }] user_answer = dedent(""" @@ -94,26 +124,34 @@ class CAssertionEvaluationTestCases(unittest.TestCase): return buff[0]; } """) - get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, + 'language': 'cpp' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer\n") -class CppStdioEvaluationTestCases(unittest.TestCase): +class CppStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = dedent(""" @@ -123,19 +161,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%d%d",&a,&b); printf("%d",a+b); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> @@ -146,19 +192,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): for(i=0;i<3;i++){ printf("%d",a[i]);} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> @@ -167,12 +221,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): scanf("%s",a); printf("%s",a); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -183,12 +244,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) @@ -201,12 +269,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; printf("%d",a) }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -217,19 +292,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): while(0==0){ printf("abc");} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> @@ -237,12 +320,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=5,b=6; printf("%d",a+b); }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -255,19 +345,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): cin>>a>>b; cout<<a+b; }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_array_input(self): self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> @@ -279,19 +377,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): for(i=0;i<3;i++){ cout<<a[i];} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_cpp_string_input(self): self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> @@ -301,12 +407,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): cin>>a; cout<<a; }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) @@ -318,12 +431,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; cout<<a; }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) @@ -337,12 +457,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=10; cout<<a }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) @@ -354,19 +481,27 @@ class CppStdioEvaluationTestCases(unittest.TestCase): while(0==0){ cout<<"abc";} }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_cpp_only_stdout(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '', - 'weight': 0.0 + 'weight': 0.0, + 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> @@ -375,12 +510,19 @@ class CppStdioEvaluationTestCases(unittest.TestCase): int a=5,b=6; cout<<a+b; }""") - get_class = CppStdioEvaluator() - kwargs = {'user_answer': user_answer, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, 'partial_grading': False, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + 'language': 'cpp' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index 88e0253..cb783b0 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -13,12 +13,12 @@ class RegistryTestCase(unittest.TestCase): assertion_evaluator_path = ("yaksh.python_assertion_evaluator" ".PythonAssertionEvaluator" ) - stdout_evaluator_path = ("yaksh.python_stdout_evaluator." - "PythonStdoutEvaluator" + stdio_evaluator_path = ("yaksh.python_stdio_evaluator." + "PythonStdIOEvaluator" ) code_evaluators['python'] = \ {"standardtestcase": assertion_evaluator_path, - "stdiobasedtestcase": stdout_evaluator_path + "stdiobasedtestcase": stdio_evaluator_path } def test_set_register(self): @@ -28,15 +28,15 @@ class RegistryTestCase(unittest.TestCase): assertion_evaluator_path = ("yaksh.python_assertion_evaluator" ".PythonAssertionEvaluator" ) - stdout_evaluator_path = ("yaksh.python_stdout_evaluator." - "PythonStdoutEvaluator" + stdio_evaluator_path = ("yaksh.python_stdio_evaluator." + "PythonStdIOEvaluator" ) class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator' ) self.registry_object.register("python", {"standardtestcase": assertion_evaluator_path, - "stdiobasedtestcase": stdout_evaluator_path + "stdiobasedtestcase": stdio_evaluator_path } ) self.assertEqual(evaluator_class, class_name) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index f7ecd97..36eb6a5 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -3,12 +3,14 @@ import unittest import os import shutil import tempfile -from yaksh import code_evaluator as evaluator -from yaksh.java_code_evaluator import JavaCodeEvaluator -from yaksh.java_stdio_evaluator import JavaStdioEvaluator -from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent +# Local Import +from yaksh import grader as gd +from yaksh.grader import Grader +from yaksh.java_code_evaluator import JavaCodeEvaluator +from yaksh.java_stdio_evaluator import JavaStdIOEvaluator + class JavaAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -17,41 +19,56 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [ {"test_case": "java_files/main_square.java", + "test_case_type": "standardtestcase", "weight": 0.0 } ] self.in_dir = tmp_in_dir_path - evaluator.SERVER_TIMEOUT = 9 + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(evaluator.SERVER_TIMEOUT) - self.file_paths = None + " your code.").format(gd.SERVER_TIMEOUT) + def tearDown(self): + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) def test_correct_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" - get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}" - get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get('success')) lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) @@ -60,25 +77,37 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): def test_error(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" - get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" - get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) @@ -86,6 +115,7 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [ {"test_case": "java_files/read_file.java", + "test_case_type": "standardtestcase", "weight": 0.0 } ] @@ -107,34 +137,41 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase): br.close(); }}} """) - get_class = JavaCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") -class JavaStdioEvaluationTestCases(unittest.TestCase): - +class JavaStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path self.test_case_data = [{'expected_output': '11', - 'expected_input': '5\n6', - 'weight': 0.0 - }] - evaluator.SERVER_TIMEOUT = 4 + 'expected_input': '5\n6', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 + }] + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" - " your code.").format(evaluator.SERVER_TIMEOUT) + " your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): - evaluator.SERVER_TIMEOUT = 4 + gd.SERVER_TIMEOUT = 4 os.remove('/tmp/test.txt') shutil.rmtree(self.in_dir) @@ -148,19 +185,26 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): int b = s.nextInt(); System.out.print(a+b); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_array_input(self): - self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', + 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" @@ -173,17 +217,23 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): a[i] = s.nextInt(); System.out.print(a[i]);} }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_incorrect_answer(self): - user_answer = dedent(""" import java.util.Scanner; class Test @@ -193,35 +243,47 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): int b = s.nextInt(); System.out.print(a); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Incorrect", result.get('error')) self.assertTrue(lines_of_error > 1) def test_error(self): - user_answer = dedent(""" class Test { System.out.print("a"); }""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): - user_answer = dedent(""" class Test {public static void main(String[] args){ @@ -229,19 +291,27 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): { System.out.print("a");} }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', - 'expected_input': '', - 'weight': 0.0 + 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" class Test @@ -250,19 +320,27 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): int b = 6; System.out.print(a+b); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_string_input(self): self.test_case_data = [{'expected_output': 'HelloWorld', - 'expected_input': 'Hello\nWorld', - 'weight': 0.0 + 'expected_input': 'Hello\nWorld', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; @@ -273,20 +351,28 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): String b = s.nextLine(); System.out.print(a+b); }}""") - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': self.test_case_data, + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_file_based_stdout(self): self.file_paths = [("/tmp/test.txt", False)] self.test_case_data = [{'expected_output': '2', - 'expected_input': '', - 'weight': 0.0 + 'expected_input': '', + 'test_case_type': 'stdiobasedtestcase', + 'weight': 0.0 }] user_answer = dedent(""" import java.io.BufferedReader; @@ -306,13 +392,19 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): br.close(); }}} """) - get_class = JavaStdioEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer\n") diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index acf5d0a..e638049 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -6,20 +6,20 @@ import shutil from textwrap import dedent # Local import +from yaksh.grader import Grader from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdio_evaluator import PythonStdioEvaluator +from yaksh.python_stdio_evaluator import PythonStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT - class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, + self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 0.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 0.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0}, ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" @@ -33,15 +33,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -50,15 +54,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -75,19 +83,23 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_partial_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" - test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, - {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, - {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0} + test_case_data = [{"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0}, + {"test_case_type": "standardtestcase", "test_case": 'assert(add(1,2)==3)', 'weight': 2.0} ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': True + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -102,15 +114,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -130,15 +146,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "SyntaxError", "invalid syntax" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -160,16 +180,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "IndentationError", "indented block" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -187,15 +210,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "name", "defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -214,15 +242,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "call", "maximum recursion depth exceeded" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -241,15 +274,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "TypeError", "argument" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -271,15 +309,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "invalid literal", "base" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -290,22 +333,27 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_file_based_assert(self): # Given - self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}] + self.test_case_data = [{"test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0}] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertIn("Correct answer", result.get('error')) @@ -316,7 +364,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nasert palindrome(s)==False', "weight": 0.0 } ] @@ -328,15 +377,20 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "SyntaxError", "invalid syntax" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -351,10 +405,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" - test_case_data = [{"test_case": 'assert(palindrome("abba")==True)', + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": 'assert(palindrome("abba")==True)', "weight": 0.0 }, - {"test_case": 's="abbb"\nassert palindrome(S)==False', + {"test_case_type": "standardtestcase", + "test_case": 's="abbb"\nassert palindrome(S)==False', "weight": 0.0 } ] @@ -363,15 +419,19 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "NameError", "name 'S' is not defined" ] - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } # When - evaluator = PythonAssertionEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) err = result.get("error").splitlines() # Then @@ -380,16 +440,18 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): for msg in name_error_msg: self.assertIn(msg, result.get("error")) - class PythonStdIOEvaluationTestCases(unittest.TestCase): def setUp(self): with open('/tmp/test.txt', 'wb') as f: f.write('2'.encode('ascii')) self.file_paths = None + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path def test_correct_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -399,14 +461,18 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -414,7 +480,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_list(self): # Given - self.test_case_data = [{"expected_input": "1,2,3\n5,6,7", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]", "weight": 0.0 }] @@ -427,14 +494,19 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a+b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -442,7 +514,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_correct_answer_string(self): # Given - self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2", "weight": 0.0 }] @@ -453,14 +526,19 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a.count(b)) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) @@ -468,7 +546,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_incorrect_answer_integer(self): # Given - self.test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -478,14 +557,18 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a-b) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'partial_grading': False - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) @@ -493,7 +576,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_file_based_answer(self): # Given - self.test_case_data = [{"expected_input": "", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "", "expected_output": "2", "weight": 0.0 }] @@ -505,15 +589,18 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): print(a[0]) """ ) - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths, - 'partial_grading': False - } + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), "Correct answer\n") @@ -521,7 +608,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): # Given - test_case_data = [{"expected_input": "1\n2", + self.test_case_data = [{"test_case_type": "stdiobasedtestcase", + "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] @@ -529,18 +617,24 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" - kwargs = {'user_answer': user_answer, - 'test_case_data': test_case_data, - 'partial_grading': False - } + + kwargs = {'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': self.test_case_data + } # When - evaluator = PythonStdioEvaluator() - result = evaluator.evaluate(**kwargs) + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) # Then self.assertEqual(result.get('error'), timeout_msg) self.assertFalse(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index c30f652..0275ee8 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -4,48 +4,63 @@ import os import shutil import tempfile -from yaksh import code_evaluator as evaluator +from yaksh import grader as gd +from yaksh.grader import Grader from yaksh.scilab_code_evaluator import ScilabCodeEvaluator -from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): tmp_in_dir_path = tempfile.mkdtemp() self.test_case_data = [{"test_case": "scilab_files/test_add.sce", + "test_case_type": "standardtestcase", "weight": 0.0 }] self.in_dir = tmp_in_dir_path + self.file_paths = None + gd.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) - self.file_paths = None + " in your code.").format(gd.SERVER_TIMEOUT) def tearDown(self): + gd.SERVER_TIMEOUT = 4 shutil.rmtree(self.in_dir) def test_correct_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertEqual(result.get('error'), "Correct answer\n") self.assertTrue(result.get('success')) def test_error(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\ndis(\tendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertTrue('error' in result.get("error")) @@ -53,13 +68,19 @@ class ScilabEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a-b;\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + lines_of_error = len(result.get('error').splitlines()) self.assertFalse(result.get('success')) self.assertIn("Message", result.get('error')) @@ -68,13 +89,19 @@ class ScilabEvaluationTestCases(unittest.TestCase): def test_infinite_loop(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a;\nwhile(1==1)\nend\nendfunction") - get_class = ScilabCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, - 'partial_grading': True, + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'scilab' + }, 'test_case_data': self.test_case_data, - 'file_paths': self.file_paths - } - result = get_class.evaluate(**kwargs) + } + + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), self.timeout_msg) |