summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/documentation/images/hook_testcase.jpgbin0 -> 63419 bytes
-rw-r--r--yaksh/documentation/moderator_docs/creating_question.rst18
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py261
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py330
-rw-r--r--yaksh/evaluator_tests/test_grader_evaluation.py (renamed from yaksh/evaluator_tests/test_code_evaluation.py)19
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py329
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py214
-rw-r--r--yaksh/forms.py7
-rw-r--r--yaksh/grader.py1
-rw-r--r--yaksh/hook_evaluator.py67
-rw-r--r--yaksh/models.py29
-rw-r--r--yaksh/settings.py20
12 files changed, 1270 insertions, 25 deletions
diff --git a/yaksh/documentation/images/hook_testcase.jpg b/yaksh/documentation/images/hook_testcase.jpg
new file mode 100644
index 0000000..3018050
--- /dev/null
+++ b/yaksh/documentation/images/hook_testcase.jpg
Binary files differ
diff --git a/yaksh/documentation/moderator_docs/creating_question.rst b/yaksh/documentation/moderator_docs/creating_question.rst
index ec273f5..f99bf7f 100644
--- a/yaksh/documentation/moderator_docs/creating_question.rst
+++ b/yaksh/documentation/moderator_docs/creating_question.rst
@@ -234,6 +234,24 @@ How to write Test cases
For MCC based question, check the correct checkbox for multiple correct options.
+ * **Create Hook based Test Case**
+
+ Select Hook from Add Test Case field.
+
+ In Hook based test case type, moderator is provided with a evaluator function
+ called **check_answer** which is provided with a parameter called **user_answer**.
+
+ **user_answer** is the code of the student in string format.
+
+ A moderator can check the string for specific words in the user answer
+ and/or compile and execute the user answer (using standard python libraries) to
+ evaluate and hence return the mark fraction.
+
+
+ .. image:: ../images/hook_testcase.jpg
+ :width: 80%
+
+
Features in Question
--------------------
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 4b551d7..482d45e 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -269,5 +269,266 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest):
# Then
self.assertTrue(result.get('success'))
+
+class BashHookEvaluationTestCases(EvaluatorBaseTest):
+
+ def setUp(self):
+ self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+ with open(self.f_path, 'wb') as f:
+ f.write('2'.encode('ascii'))
+ self.in_dir = tempfile.mkdtemp()
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in your"
+ " code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
+
+ def tearDown(self):
+ os.remove(self.f_path)
+ shutil.rmtree(self.in_dir)
+
+ def test_correct_answer(self):
+ # Given
+ user_answer = dedent(""" #!/bin/bash
+ echo -n Hello, world!
+ """
+ )
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ proc = subprocess.Popen(user_answer, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ # Given
+ user_answer = dedent(""" #!/bin/bash
+ echo -n Goodbye, world!
+ """
+ )
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ proc = subprocess.Popen(user_answer, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output('Incorrect Answer', result.get('error'))
+
+ def test_assert_with_hook(self):
+ # Given
+ user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
+ " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
+ )
+ assert_test_case = dedent("""
+ #!/bin/bash
+ [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))
+ """)
+
+ assert_test_case_args = "1 2\n2 1"
+
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "echo $(( $1 + $2 ))" in user_answer:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "standardtestcase",
+ "test_case": assert_test_case,
+ "test_case_args":assert_test_case_args,
+ 'weight': 1.0
+ },
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 2.0)
+
+ def test_multiple_hooks(self):
+ # Given
+ user_answer = dedent(""" #!/bin/bash
+ echo -n Hello, world!
+ """
+ )
+
+ hook_code_1 = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "echo -n Hello, world!" in user_answer:
+ success, err, mark_fraction = True, "", 0.5
+ return success, err, mark_fraction
+ """
+ )
+ hook_code_2 = dedent("""\
+ def check_answer(user_answer):
+ import subprocess
+ import sys
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ proc = subprocess.Popen(user_answer, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+
+ if stdout.decode('utf-8') == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code_1, 'weight': 1.0},
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code_2, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 1.5)
+
+ def test_infinite_loop(self):
+ # Given
+ user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
+ " do echo "" > /dev/null ; done")
+
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ proc = subprocess.Popen(user_answer, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'bash'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index d734cf2..304f1cb 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -639,5 +639,335 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest):
# Then
self.assertTrue(result.get('success'))
+class CppHookEvaluationTestCases(EvaluatorBaseTest):
+
+ def setUp(self):
+ self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+ with open(self.f_path, 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in your"
+ " code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
+
+ def tearDown(self):
+ os.remove(self.f_path)
+ shutil.rmtree(self.in_dir)
+
+ def test_correct_answer(self):
+ # Given
+ user_answer = dedent("""\
+ #include<stdio.h>
+ main()
+ {
+ printf("Hello, world!");
+ }
+ """)
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.c", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["gcc Test.c", "./a.out"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'cpp'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ # Given
+ user_answer = dedent("""\
+ #include<stdio.h>
+ main()
+ {
+ printf("Goodbye, world!");
+ }
+ """)
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.c", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["gcc Test.c", "./a.out"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'cpp'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output('Incorrect Answer', result.get('error'))
+
+ def test_assert_with_hook(self):
+ # Given
+ user_answer = "int add(int a, int b)\n{return a+b;}"
+
+
+ assert_test_case = dedent("""\
+ #include <stdio.h>
+ #include <stdlib.h>
+
+ extern int add(int, int);
+
+ template <class T>
+
+ void check(T expect, T result)
+ {
+ if (expect == result)
+ {
+ printf("Correct: Expected %d got %d ",expect,result);
+ }
+ else
+ {
+ printf("Incorrect: Expected %d got %d ",expect,result);
+ exit (1);
+ }
+ }
+
+ int main(void)
+ {
+ int result;
+ result = add(0,0);
+ printf("Input submitted to the function: 0, 0");
+ check(0, result);
+ result = add(2,3);
+ printf("Input submitted to the function: 2 3");
+ check(5,result);
+ printf("All Correct");
+ return 0;
+ }
+ """)
+
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "return a+b;" in user_answer:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "standardtestcase",
+ "test_case": assert_test_case,
+ 'weight': 1.0
+ },
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'cpp'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 2.0)
+
+ def test_multiple_hooks(self):
+ # Given
+ user_answer = dedent("""\
+ #include<stdio.h>
+ main()
+ {
+ printf("Hello, world!");
+ }
+ """)
+
+ hook_code_1 = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.c", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["gcc Test.c", "./a.out"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+ hook_code_2 = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if 'printf("Hello, world!");' in user_answer:
+ success, err, mark_fraction = True, "", 0.5
+ return success, err, mark_fraction
+ """
+ )
+
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code_1, 'weight': 1.0},
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code_2, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'cpp'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 1.5)
+
+ def test_infinite_loop(self):
+ # Given
+ user_answer = dedent("""\
+ #include<stdio.h>
+ int main(void){
+ while(0==0){
+ printf("abc");}
+ }""")
+
+ hook_code= dedent("""\
+ def check_answer(user_answer):
+ with open("Test.c", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["gcc Test.c", "./a.out"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'cpp'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_grader_evaluation.py
index cb783b0..d11f4a0 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_grader_evaluation.py
@@ -16,34 +16,27 @@ class RegistryTestCase(unittest.TestCase):
stdio_evaluator_path = ("yaksh.python_stdio_evaluator."
"PythonStdIOEvaluator"
)
+
+ hook_evaluator_path = ("yaksh.hook_evaluator."
+ "HookEvaluator"
+ )
code_evaluators['python'] = \
{"standardtestcase": assertion_evaluator_path,
- "stdiobasedtestcase": stdio_evaluator_path
+ "stdiobasedtestcase": stdio_evaluator_path,
+ "hooktestcase": hook_evaluator_path
}
def test_set_register(self):
evaluator_class = self.registry_object.get_class("python",
"standardtestcase"
)
- assertion_evaluator_path = ("yaksh.python_assertion_evaluator"
- ".PythonAssertionEvaluator"
- )
- stdio_evaluator_path = ("yaksh.python_stdio_evaluator."
- "PythonStdIOEvaluator"
- )
class_name = getattr(python_assertion_evaluator,
'PythonAssertionEvaluator'
)
- self.registry_object.register("python",
- {"standardtestcase": assertion_evaluator_path,
- "stdiobasedtestcase": stdio_evaluator_path
- }
- )
self.assertEqual(evaluator_class, class_name)
def tearDown(self):
self.registry_object = None
-
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index b53d8aa..3d127af 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -507,5 +507,334 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest):
self.assertTrue(result.get("success"))
+class JavaHookEvaluationTestCases(EvaluatorBaseTest):
+
+ def setUp(self):
+ self.f_path = os.path.join(tempfile.gettempdir(), "test.txt")
+ with open(self.f_path, 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
+ self.file_paths = None
+ gd.SERVER_TIMEOUT = 9
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in"
+ " your code.").format(gd.SERVER_TIMEOUT)
+
+ def tearDown(self):
+ gd.SERVER_TIMEOUT = 4
+ os.remove(self.f_path)
+ shutil.rmtree(self.in_dir)
+
+ def test_correct_answer(self):
+ # Given
+ user_answer = dedent("""\
+ class Test
+ {public static void main(String[] args){
+ System.out.print("Hello, world!");
+ }}
+ """)
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.java", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["javac Test.java", "java Test"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'java'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ # Given
+ user_answer = dedent("""\
+ class Test
+ {public static void main(String[] args){
+ System.out.print("Goodbye, world!");
+ }}
+ """)
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.java", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["javac Test.java", "java Test"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'java'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output('Incorrect Answer', result.get('error'))
+
+ def test_assert_with_hook(self):
+ # Given
+ user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
+ assert_test_case = dedent("""
+ class main
+ {
+ public static <E> void check(E expect, E result)
+ {
+ if(result.equals(expect))
+ {
+ System.out.println("Correct:Output expected "+expect+" and got "+result);
+ }
+ else
+ {
+ System.out.println("Incorrect:Output expected "+expect+" but got "+result);
+ System.exit(1);
+ }
+ }
+ public static void main(String arg[])
+ {
+ Test t = new Test();
+ int result, input, output;
+ input = 0; output = 0;
+ result = t.square_num(input);
+ System.out.println("Input submitted to the function: "+input);
+ check(output, result);
+ input = 5; output = 25;
+ result = t.square_num(input);
+ System.out.println("Input submitted to the function: "+input);
+ check(output, result);
+ input = 6; output = 36;
+ result = t.square_num(input);
+ System.out.println("Input submitted to the function: "+input);
+ check(output, result);
+ }
+ }
+ """)
+
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "return a*a" in user_answer:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "standardtestcase",
+ "test_case": assert_test_case,
+ 'weight': 1.0
+ },
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'java'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 2.0)
+
+ def test_multiple_hooks(self):
+ # Given
+ user_answer = dedent("""\
+ class Test
+ {public static void main(String[] args){
+ System.out.print("Hello, world!");
+ }}
+ """)
+
+ hook_code_1 = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.java", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["javac Test.java", "java Test"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ hook_code_2 = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if 'System.out.print("Hello, world!");' in user_answer:
+ success, err, mark_fraction = True, "", 0.5
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code_1, 'weight': 1.0},
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code_2, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'java'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 1.5)
+
+ def test_infinite_loop(self):
+ # Given
+ user_answer = dedent("""\
+ class Test
+ {public static void main(String[] args){
+ while(0==0)
+ {
+ System.out.print("a");}
+ }}""")
+
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ with open("Test.java", "w+") as f:
+ f.write(user_answer)
+ import subprocess
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ def _run_command(cmd):
+ proc = subprocess.Popen("{}".format(cmd),
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ stdout,stderr = proc.communicate()
+ return stdout,stderr
+ cmds = ["javac Test.java", "java Test"]
+ for cmd in cmds:
+ stdout, stderr = _run_command(cmd)
+ if stdout.decode("utf-8") == "Hello, world!":
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'java'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 43dfe6b..1fba73e 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -7,8 +7,6 @@ from textwrap import dedent
# Local import
from yaksh.grader import Grader
-from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
-from yaksh.python_stdio_evaluator import PythonStdIOEvaluator
from yaksh.settings import SERVER_TIMEOUT
@@ -643,5 +641,217 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest):
self.assertFalse(result.get('success'))
+class PythonHookEvaluationTestCases(EvaluatorBaseTest):
+
+ def setUp(self):
+ with open('/tmp/test.txt', 'wb') as f:
+ f.write('2'.encode('ascii'))
+ tmp_in_dir_path = tempfile.mkdtemp()
+ self.in_dir = tmp_in_dir_path
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in"
+ " your code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
+
+ def tearDown(self):
+ os.remove('/tmp/test.txt')
+ shutil.rmtree(self.in_dir)
+
+ def test_correct_answer(self):
+ # Given
+ user_answer = "def add(a,b):\n\treturn a + b"
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ exec(user_answer, globals())
+ if add(1,2) == 3:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'python'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ # Given
+ user_answer = "def add(a,b):\n\treturn a - b"
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ exec(user_answer, globals())
+ if add(1,2) == 3:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'python'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output('Incorrect Answer', result.get('error'))
+
+ def test_assert_with_hook(self):
+ # Given
+ user_answer = "def add(a,b):\n\treturn a + b"
+ assert_test_case = "assert add(1,2) == 3"
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "return a + b" in user_answer:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+ test_case_data = [{"test_case_type": "standardtestcase",
+ "test_case": assert_test_case, 'weight': 1.0},
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'python'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 2.0)
+
+ def test_multiple_hooks(self):
+ # Given
+ user_answer = "def add(a,b):\n\treturn a + b"
+ hook_code_1 = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ if "return a + b" in user_answer:
+ success, err, mark_fraction = True, "", 0.5
+ return success, err, mark_fraction
+ """
+ )
+ hook_code_2 = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ exec(user_answer, globals())
+ if add(1,2) == 3:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+
+
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code_1, 'weight': 1.0},
+ {"test_case_type": "hooktestcase",
+ "hook_code": hook_code_2, 'weight': 1.0},
+ ]
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': True,
+ 'language': 'python'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertTrue(result.get('success'))
+ self.assertEqual(result.get("weight"), 1.5)
+
+ def test_infinite_loop(self):
+ # Given
+ user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
+ hook_code = dedent("""\
+ def check_answer(user_answer):
+ success = False
+ err = "Incorrect Answer"
+ mark_fraction = 0.0
+ exec(user_answer, globals())
+ if add(1,2) == 3:
+ success, err, mark_fraction = True, "", 1.0
+ return success, err, mark_fraction
+ """
+ )
+ test_case_data = [{"test_case_type": "hooktestcase",
+ "hook_code": hook_code,"weight": 1.0
+ }]
+
+ kwargs = {
+ 'metadata': {
+ 'user_answer': user_answer,
+ 'file_paths': self.file_paths,
+ 'partial_grading': False,
+ 'language': 'python'
+ },
+ 'test_case_data': test_case_data,
+ }
+
+ # When
+ grader = Grader(self.in_dir)
+ result = grader.evaluate(kwargs)
+
+ # Then
+ self.assertFalse(result.get('success'))
+ self.assert_correct_output(self.timeout_msg, result.get('error'))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/forms.py b/yaksh/forms.py
index 1d18d29..8a90dee 100644
--- a/yaksh/forms.py
+++ b/yaksh/forms.py
@@ -1,6 +1,7 @@
from django import forms
from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course,\
- QuestionPaper, StandardTestCase, StdIOBasedTestCase
+ QuestionPaper, StandardTestCase, StdIOBasedTestCase, \
+ HookTestCase
from django.contrib.auth import authenticate
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
@@ -40,7 +41,8 @@ test_case_types = (
("standardtestcase", "Standard Testcase"),
("stdiobasedtestcase", "StdIO Based Testcase"),
("mcqtestcase", "MCQ Testcase"),
- )
+ ("hooktestcase", "Hook Testcase"),
+ )
UNAME_CHARS = letters + "._" + digits
PWD_CHARS = letters + punctuation + digits
@@ -296,3 +298,4 @@ class QuestionPaperForm(forms.ModelForm):
class Meta:
model = QuestionPaper
fields = ['shuffle_questions']
+
diff --git a/yaksh/grader.py b/yaksh/grader.py
index 086abb7..a9a3738 100644
--- a/yaksh/grader.py
+++ b/yaksh/grader.py
@@ -120,7 +120,6 @@ class Grader(object):
for test_case in test_case_data:
test_case_instance = create_evaluator_instance(metadata, test_case)
test_case_instances.append(test_case_instance)
-
return test_case_instances
diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py
new file mode 100644
index 0000000..3956da1
--- /dev/null
+++ b/yaksh/hook_evaluator.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+import sys
+import traceback
+import os
+
+# Local imports
+from .file_utils import copy_files, delete_files
+from .base_evaluator import BaseEvaluator
+from .grader import TimeoutException
+
+
+class HookEvaluator(BaseEvaluator):
+ def __init__(self, metadata, test_case_data):
+ self.files = []
+
+ # Set metadata values
+ self.user_answer = metadata.get('user_answer')
+ self.file_paths = metadata.get('file_paths')
+ self.partial_grading = metadata.get('partial_grading')
+
+ # Set test case data values
+ self.hook_code = test_case_data.get('hook_code')
+ self.weight = test_case_data.get('weight')
+
+ def teardown(self):
+ # Delete the created file.
+ if self.files:
+ delete_files(self.files)
+
+ def check_code(self):
+ """ Function evaluates user answer by running a python based hook code
+ against it.
+ Returns
+ --------
+ Returns a tuple (success, error, test_case_weight)
+
+ success - Boolean, indicating if code was executed successfully, correctly
+ mark_fraction - Float, indicating fraction of the weight to a test case
+ error - String, error message if success is false
+
+ returns (True, "Correct answer", 1.0) : If the student script passes all
+ test cases/have same output, when compared to the instructor script
+
+ returns (False, error_msg, 0.0): If the student script fails a single
+ test/have dissimilar output, when compared to the instructor script.
+
+ Returns (False, error_msg, 0.0): If mandatory arguments are not files or if
+ the required permissions are not given to the file(s).
+ """
+ success = False
+ mark_fraction = 0.0
+ try:
+ tb = None
+ _tests = compile(self.hook_code, '<string>', mode='exec')
+ hook_scope = {}
+ exec(_tests, hook_scope)
+ check = hook_scope["check_answer"]
+ success, err, mark_fraction = check(self.user_answer)
+ except TimeoutException:
+ raise
+ except Exception:
+ msg = traceback.format_exc(limit=0)
+ err = "Error in Hook code: {0}".format(msg)
+ del tb
+ return success, err, mark_fraction
+ \ No newline at end of file
diff --git a/yaksh/models.py b/yaksh/models.py
index ca41885..ad61872 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -4,6 +4,7 @@ import json
from random import sample, shuffle
from itertools import islice, cycle
from collections import Counter
+from textwrap import dedent
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User
@@ -1219,5 +1220,31 @@ class McqTestCase(TestCase):
class HookTestCase(TestCase):
- code = models.TextField()
+ hook_code = models.TextField(default=dedent\
+ ("""\
+ def check_answer(user_answer):
+ ''' Evaluates user answer to return -
+ success - Boolean, indicating if code was executed correctly
+ mark_fraction - Float, indicating fraction of the
+ weight to a test case
+ error - String, error message if success is false'''
+ success = False
+ err = "Incorrect Answer" # Please make the error message more specific
+ mark_fraction = 0.0
+
+ # write your code here
+
+ return success, err, mark_fraction
+
+ """)
+
+ )
weight = models.FloatField(default=1.0)
+
+ def get_field_value(self):
+ return {"test_case_type": "hooktestcase", "hook_code": self.hook_code,
+ "weight": self.weight}
+
+ def __str__(self):
+ return u'Hook Testcase | Correct: {0}'.format(self.hook_code)
+
diff --git a/yaksh/settings.py b/yaksh/settings.py
index 0e432cf..72f9fda 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -21,18 +21,26 @@ URL_ROOT = ''
code_evaluators = {
"python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator",
- "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator"
+ "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
},
"c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator",
- "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator"
+ "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
},
"cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator",
- "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator"
+ "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
},
"java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator",
- "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator"},
+ "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
+ },
"bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator",
- "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator"
+ "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
},
- "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"},
+ "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator",
+ "hooktestcase": "yaksh.hook_evaluator.HookEvaluator"
+ },
}
generated by cgit (git 2.34.1) at 2024-12-20 20:28:29 +0000