summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests/test_bash_evaluation.py
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests/test_bash_evaluation.py')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py211
1 files changed, 161 insertions, 50 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 99e5122..8888ee6 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -3,6 +3,7 @@ import unittest
import os
import shutil
import tempfile
+from yaksh.code_evaluator import CodeEvaluator
from yaksh.bash_code_evaluator import BashCodeEvaluator
from yaksh.bash_stdio_evaluator import BashStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
@@ -15,6 +16,7 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
f.write('2'.encode('ascii'))
self.test_case_data = [
{"test_case": "bash_files/sample.sh,bash_files/sample.args",
+ "test_case_type": "standardtestcase",
"weight": 0.0
}
]
@@ -32,39 +34,78 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
" && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
)
- get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
- 'partial_grading': True,
+ # get_class = BashCodeEvaluator(self.in_dir)
+ # kwargs = {'user_answer': user_answer,
+ # 'partial_grading': True,
+ # 'test_case_data': self.test_case_data,
+ # 'file_paths': self.file_paths
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer\n")
def test_error(self):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] "
"&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))")
- get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
- 'partial_grading': True,
+ # get_class = BashCodeEvaluator(self.in_dir)
+ # kwargs = {'user_answer': user_answer,
+ # 'partial_grading': True,
+ # 'test_case_data': self.test_case_data,
+ # 'file_paths': self.file_paths
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertFalse(result.get("success"))
self.assertTrue("Error" in result.get("error"))
def test_infinite_loop(self):
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
" do echo "" > /dev/null ; done")
- get_class = BashCodeEvaluator(self.in_dir)
- kwargs = {'user_answer': user_answer,
- 'partial_grading': True,
+ # get_class = BashCodeEvaluator(self.in_dir)
+ # kwargs = {'user_answer': user_answer,
+ # 'partial_grading': True,
+ # 'test_case_data': self.test_case_data,
+ # 'file_paths': self.file_paths
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertFalse(result.get("success"))
self.assertEqual(result.get("error"), self.timeout_msg)
@@ -72,17 +113,31 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
self.file_paths = [('/tmp/test.txt', False)]
self.test_case_data = [
{"test_case": "bash_files/sample1.sh,bash_files/sample1.args",
+ "test_case_type": "standardtestcase",
"weight": 0.0
}
]
user_answer = ("#!/bin/bash\ncat $1")
- get_class = BashCodeEvaluator()
- kwargs = {'user_answer': user_answer,
- 'partial_grading': True,
+ # get_class = BashCodeEvaluator()
+ # kwargs = {'user_answer': user_answer,
+ # 'partial_grading': True,
+ # 'test_case_data': self.test_case_data,
+ # 'file_paths': self.file_paths
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
- result = get_class.evaluate(**kwargs)
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertTrue(result.get("success"))
self.assertEqual(result.get("error"), "Correct answer\n")
@@ -92,6 +147,8 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
+
def test_correct_answer(self):
user_answer = dedent(""" #!/bin/bash
@@ -102,14 +159,28 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
)
test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
+ 'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
- get_class = BashStdioEvaluator()
- kwargs = {"user_answer": user_answer,
- "partial_grading": True,
- "test_case_data": test_case_data
- }
- result = get_class.evaluate(**kwargs)
+ # get_class = BashStdioEvaluator()
+ # kwargs = {"user_answer": user_answer,
+ # "partial_grading": True,
+ # "test_case_data": test_case_data
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
@@ -123,15 +194,29 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
"""
)
test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
- 'expected_input': '1,2,3\n4,5,6\n7,8,9',
- 'weight': 0.0
+ 'expected_input': '1,2,3\n4,5,6\n7,8,9',
+ 'test_case_type': 'stdiobasedtestcase',
+ 'weight': 0.0
}]
- get_class = BashStdioEvaluator()
- kwargs = {"user_answer": user_answer,
- "partial_grading": True,
- "test_case_data": test_case_data
- }
- result = get_class.evaluate(**kwargs)
+ # get_class = BashStdioEvaluator()
+ # kwargs = {"user_answer": user_answer,
+ # "partial_grading": True,
+ # "test_case_data": test_case_data
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
+
self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
@@ -144,14 +229,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
)
test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
+ 'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
- get_class = BashStdioEvaluator()
- kwargs = {"user_answer": user_answer,
- "partial_grading": True,
- "test_case_data": test_case_data
- }
- result = get_class.evaluate(**kwargs)
+ # get_class = BashStdioEvaluator()
+ # kwargs = {"user_answer": user_answer,
+ # "partial_grading": True,
+ # "test_case_data": test_case_data
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
self.assertIn("Incorrect", result.get('error'))
self.assertFalse(result.get('success'))
@@ -164,14 +262,27 @@ class BashStdioEvaluationTestCases(unittest.TestCase):
)
test_case_data = [{'expected_output': '10',
'expected_input': '',
+ 'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
- get_class = BashStdioEvaluator()
- kwargs = {"user_answer": user_answer,
- "partial_grading": True,
- "test_case_data": test_case_data
- }
- result = get_class.evaluate(**kwargs)
+ # get_class = BashStdioEvaluator()
+ # kwargs = {"user_answer": user_answer,
+ # "partial_grading": True,
+ # "test_case_data": test_case_data
+ # }
+ # result = get_class.evaluate(**kwargs)
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ evaluator = CodeEvaluator(self.in_dir)
+ result = evaluator.evaluate(kwargs)
self.assertEqual(result.get('error'), "Correct answer\n")
self.assertTrue(result.get('success'))
generated by cgit (git 2.34.1) at 2024-12-20 11:52:12 +0000