summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py59
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py125
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py89
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py103
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py16
5 files changed, 259 insertions, 133 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 66ade19..5f4be4b 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -13,12 +13,12 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
- tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [
- {"test_case": "bash_files/sample.sh,bash_files/sample.args"}
+ {"test_case": "bash_files/sample.sh,bash_files/sample.args",
+ "marks": 0.0
+ }
]
- tmp_in_dir_path = tempfile.mkdtemp()
- self.in_dir = tmp_in_dir_path
+ self.in_dir = tempfile.mkdtemp()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
@@ -33,7 +33,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
" && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
)
get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -45,7 +46,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] "
"&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))")
get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -57,7 +59,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
" do echo "" > /dev/null ; done")
get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -68,11 +71,14 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
def test_file_based_assert(self):
self.file_paths = [('/tmp/test.txt', False)]
self.test_case_data = [
- {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"}
+ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args",
+ "marks": 0.0
+ }
]
user_answer = ("#!/bin/bash\ncat $1")
get_class = BashCodeEvaluator()
kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -82,6 +88,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
class BashStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
+ self.in_dir = tempfile.mkdtemp()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
@@ -93,11 +100,15 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
echo -n `expr $A + $B`
"""
)
- test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ test_case_data = [{'expected_output': '11',
+ 'expected_input': '5\n6',
+ 'marks': 0.0
+ }]
get_class = BashStdioEvaluator()
kwargs = {"user_answer": user_answer,
- "test_case_data": test_case_data
- }
+ "partial_grading": True,
+ "test_case_data": test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -112,12 +123,14 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"""
)
test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
- 'expected_input': '1,2,3\n4,5,6\n7,8,9'
+ 'expected_input': '1,2,3\n4,5,6\n7,8,9',
+ 'marks': 0.0
}]
get_class = BashStdioEvaluator()
kwargs = {"user_answer": user_answer,
- "test_case_data": test_case_data
- }
+ "partial_grading": True,
+ "test_case_data": test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -129,11 +142,15 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
echo -n `expr $A - $B`
"""
)
- test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ test_case_data = [{'expected_output': '11',
+ 'expected_input': '5\n6',
+ 'marks': 0.0
+ }]
get_class = BashStdioEvaluator()
kwargs = {"user_answer": user_answer,
- "test_case_data": test_case_data
- }
+ "partial_grading": True,
+ "test_case_data": test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertIn("Incorrect", result.get('error'))
self.assertFalse(result.get('success'))
@@ -146,12 +163,14 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"""
)
test_case_data = [{'expected_output': '10',
- 'expected_input': ''
+ 'expected_input': '',
+ 'marks': 0.0
}]
get_class = BashStdioEvaluator()
kwargs = {"user_answer": user_answer,
- "test_case_data": test_case_data
- }
+ "partial_grading": True,
+ "test_case_data": test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index c990436..79326d4 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -14,7 +14,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
with open('/tmp/test.txt', 'wb') as f:
f.write('2'.encode('ascii'))
tmp_in_dir_path = tempfile.mkdtemp()
- self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
+ self.test_case_data = [{"test_case": "c_cpp_files/main.cpp",
+ "marks": 0.0
+ }]
self.in_dir = tmp_in_dir_path
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
@@ -29,6 +31,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "int add(int a, int b)\n{return a+b;}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
+ 'partial_grading': False,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -40,6 +43,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "int add(int a, int b)\n{return a-b;}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
+ 'partial_grading': False,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -52,7 +56,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
def test_compilation_error(self):
user_answer = "int add(int a, int b)\n{return a+b}"
get_class = CppCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': False,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -63,7 +68,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
def test_infinite_loop(self):
user_answer = "int add(int a, int b)\n{while(1>0){}}"
get_class = CppCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': False,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -73,7 +79,9 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
def test_file_based_assert(self):
self.file_paths = [('/tmp/test.txt', False)]
- self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}]
+ self.test_case_data = [{"test_case": "c_cpp_files/file_data.c",
+ "marks": 0.0
+ }]
user_answer = dedent("""
#include<stdio.h>
char ans()
@@ -88,18 +96,21 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
""")
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer")
class CppStdioEvaluationTestCases(unittest.TestCase):
-
def setUp(self):
- self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
- self.in_dir = os.getcwd()
+ self.test_case_data = [{'expected_output': '11',
+ 'expected_input': '5\n6',
+ 'marks': 0.0
+ }]
+ self.in_dir = tempfile.mkdtemp()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
@@ -114,15 +125,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_array_input(self):
self.test_case_data = [{'expected_output': '561',
- 'expected_input': '5\n6\n1'}]
+ 'expected_input': '5\n6\n1',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<stdio.h>
int main(void){
@@ -134,15 +148,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_string_input(self):
self.test_case_data = [{'expected_output': 'abc',
- 'expected_input': 'abc'}]
+ 'expected_input': 'abc',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<stdio.h>
int main(void){
@@ -152,8 +169,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -167,8 +185,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
@@ -184,8 +203,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("Compilation Error" in result.get("error"))
@@ -199,15 +219,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEqual(result.get("error"), self.timeout_msg)
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
- 'expected_input': ''}]
+ 'expected_input': '',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<stdio.h>
int main(void){
@@ -216,8 +239,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -233,15 +257,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_array_input(self):
self.test_case_data = [{'expected_output': '561',
- 'expected_input': '5\n6\n1'}]
+ 'expected_input': '5\n6\n1',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<iostream>
using namespace std;
@@ -254,15 +281,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_cpp_string_input(self):
self.test_case_data = [{'expected_output': 'abc',
- 'expected_input': 'abc'}]
+ 'expected_input': 'abc',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<iostream>
using namespace std;
@@ -273,8 +303,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -289,8 +320,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
@@ -307,8 +339,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("Compilation Error" in result.get("error"))
@@ -323,15 +356,18 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEqual(result.get("error"), self.timeout_msg)
def test_cpp_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
- 'expected_input': ''}]
+ 'expected_input': '',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
#include<iostream>
using namespace std;
@@ -341,8 +377,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = CppStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': False,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index e375bdb..33e0e35 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -16,7 +16,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
f.write('2'.encode('ascii'))
tmp_in_dir_path = tempfile.mkdtemp()
self.test_case_data = [
- {"test_case": "java_files/main_square.java"}
+ {"test_case": "java_files/main_square.java",
+ "marks": 0.0
+ }
]
self.in_dir = tmp_in_dir_path
evaluator.SERVER_TIMEOUT = 9
@@ -32,7 +34,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
def test_correct_answer(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -43,7 +46,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
def test_incorrect_answer(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -57,7 +61,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
def test_error(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
get_class = JavaCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -68,7 +73,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
def test_infinite_loop(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -79,7 +85,9 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
def test_file_based_assert(self):
self.file_paths = [("/tmp/test.txt", False)]
self.test_case_data = [
- {"test_case": "java_files/read_file.java"}
+ {"test_case": "java_files/read_file.java",
+ "marks": 0.0
+ }
]
user_answer = dedent("""
import java.io.BufferedReader;
@@ -101,9 +109,10 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
""")
get_class = JavaCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get("success"))
self.assertEqual(result.get("error"), "Correct answer")
@@ -116,7 +125,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
tmp_in_dir_path = tempfile.mkdtemp()
self.in_dir = tmp_in_dir_path
self.test_case_data = [{'expected_output': '11',
- 'expected_input': '5\n6'}]
+ 'expected_input': '5\n6',
+ 'marks': 0.0
+ }]
evaluator.SERVER_TIMEOUT = 4
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
@@ -139,8 +150,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -148,7 +160,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
def test_array_input(self):
self.test_case_data = [{'expected_output': '561',
- 'expected_input': '5\n6\n1'}]
+ 'expected_input': '5\n6\n1',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
import java.util.Scanner;
class Test
@@ -161,8 +175,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -180,8 +195,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
lines_of_error = len(result.get('error').splitlines())
self.assertFalse(result.get('success'))
@@ -197,8 +213,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("Compilation Error" in result.get("error"))
@@ -214,15 +231,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEqual(result.get("error"), self.timeout_msg)
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
- 'expected_input': ''}]
+ 'expected_input': '',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
class Test
{public static void main(String[] args){
@@ -232,15 +252,18 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_string_input(self):
self.test_case_data = [{'expected_output': 'HelloWorld',
- 'expected_input': 'Hello\nWorld'}]
+ 'expected_input': 'Hello\nWorld',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
import java.util.Scanner;
class Test
@@ -252,8 +275,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
}}""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
@@ -261,7 +285,9 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
def test_file_based_stdout(self):
self.file_paths = [("/tmp/test.txt", False)]
self.test_case_data = [{'expected_output': '2',
- 'expected_input': ''}]
+ 'expected_input': '',
+ 'marks': 0.0
+ }]
user_answer = dedent("""
import java.io.BufferedReader;
import java.io.FileReader;
@@ -282,9 +308,10 @@ class JavaStdioEvaluationTestCases(unittest.TestCase):
""")
get_class = JavaStdioEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'partial_grading': True,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get("success"))
self.assertEqual(result.get("error"), "Correct answer")
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 45cc40d..da64dc4 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -17,9 +17,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
f.write('2'.encode('ascii'))
tmp_in_dir_path = tempfile.mkdtemp()
self.in_dir = tmp_in_dir_path
- self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'},
- {"test_case": 'assert(add(-1,2)==1)'},
- {"test_case": 'assert(add(-1,-2)==-3)'},
+ self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'marks': 0.0},
+ {"test_case": 'assert(add(-1,2)==1)', 'marks': 0.0},
+ {"test_case": 'assert(add(-1,-2)==-3)', 'marks': 0.0},
]
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
@@ -35,7 +35,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "def add(a,b):\n\treturn a + b"
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -51,7 +52,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "def add(a,b):\n\treturn a - b"
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -59,17 +61,22 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
result = evaluator.evaluate(**kwargs)
# Then
+ print repr(result.get('error'))
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'),
- "AssertionError in: assert(add(1,2)==3)"
- )
+ ('AssertionError in: assert(add(1,2)==3)\n'
+ 'AssertionError in: assert(add(-1,2)==1)\n'
+ 'AssertionError in: assert(add(-1,-2)==-3)\n'
+ )
+ )
def test_infinite_loop(self):
# Given
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -96,7 +103,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -125,7 +133,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
+
}
# When
@@ -150,7 +160,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -176,7 +187,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -202,7 +214,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -231,7 +244,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -247,7 +261,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
def test_file_based_assert(self):
# Given
- self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
+ self.test_case_data = [{"test_case": "assert(ans()=='2')", "marks": 0.0}]
self.file_paths = [('/tmp/test.txt', False)]
user_answer = dedent("""
def ans():
@@ -256,7 +270,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
""")
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -272,7 +287,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
""" Tests the user answer with just an incorrect test case """
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
- test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False'}
+ test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False',
+ "marks": 0.0
+ }
]
syntax_error_msg = ["Traceback",
"call",
@@ -284,7 +301,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -304,8 +322,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
first and then with an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
- test_case_data = [{"test_case": 'assert(palindrome("abba")==True)'},
- {"test_case": 's="abbb"\nassert palindrome(S)==False'}
+ test_case_data = [{"test_case": 'assert(palindrome("abba")==True)',
+ "marks": 0.0
+ },
+ {"test_case": 's="abbb"\nassert palindrome(S)==False',
+ "marks": 0.0
+ }
]
name_error_msg = ["Traceback",
"call",
@@ -317,7 +339,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
kwargs = {'user_answer': user_answer,
'test_case_data': test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -341,7 +364,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_correct_answer_integer(self):
# Given
self.test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3"
+ "expected_output": "3",
+ "marks": 0.0
}]
user_answer = dedent("""
a = int(input())
@@ -350,7 +374,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
"""
)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'partial_grading': False
}
# When
@@ -364,7 +389,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_correct_answer_list(self):
# Given
self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
- "expected_output": "[1, 2, 3, 5, 6, 7]"
+ "expected_output": "[1, 2, 3, 5, 6, 7]",
+ "marks": 0.0
}]
user_answer = dedent("""
from six.moves import input
@@ -376,7 +402,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
"""
)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'partial_grading': False
}
# When
@@ -390,7 +417,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_correct_answer_string(self):
# Given
self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
- "expected_output": "2"
+ "expected_output": "2",
+ "marks": 0.0
}]
user_answer = dedent("""
from six.moves import input
@@ -400,7 +428,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
"""
)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'partial_grading': False
}
# When
@@ -414,7 +443,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_incorrect_answer_integer(self):
# Given
self.test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3"
+ "expected_output": "3",
+ "marks": 0.0
}]
user_answer = dedent("""
a = int(input())
@@ -424,6 +454,7 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
+ 'partial_grading': False
}
# When
@@ -436,7 +467,10 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_file_based_answer(self):
# Given
- self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
+ self.test_case_data = [{"expected_input": "",
+ "expected_output": "2",
+ "marks": 0.0
+ }]
self.file_paths = [('/tmp/test.txt', False)]
user_answer = dedent("""
@@ -447,7 +481,8 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
)
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
+ 'file_paths': self.file_paths,
+ 'partial_grading': False
}
# When
@@ -461,14 +496,16 @@ class PythonStdIOEvaluationTestCases(unittest.TestCase):
def test_infinite_loop(self):
# Given
test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3"
- }]
+ "expected_output": "3",
+ "marks": 0.0
+ }]
timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
user_answer = "while True:\n\tpass"
kwargs = {'user_answer': user_answer,
- 'test_case_data': test_case_data
+ 'test_case_data': test_case_data,
+ 'partial_grading': False
}
# When
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index b366480..b46a65b 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -11,7 +11,9 @@ from yaksh.settings import SERVER_TIMEOUT
class ScilabEvaluationTestCases(unittest.TestCase):
def setUp(self):
tmp_in_dir_path = tempfile.mkdtemp()
- self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
+ self.test_case_data = [{"test_case": "scilab_files/test_add.sce",
+ "marks": 0.0
+ }]
self.in_dir = tmp_in_dir_path
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop"
@@ -25,7 +27,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a+b;\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -37,7 +40,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a+b;\ndis(\tendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -50,7 +54,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a-b;\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
@@ -64,7 +69,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a;\nwhile(1==1)\nend\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
+ kwargs = {'user_answer': user_answer,
+ 'partial_grading': True,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
generated by cgit (git 2.34.1) at 2024-12-20 19:10:11 +0000