From 65f8f0f4dd867c1d1647712560e3aef73b8c921f Mon Sep 17 00:00:00 2001 From: Mahesh Gudi Date: Thu, 29 Dec 2016 18:00:28 +0530 Subject: Python hook evaluator --- yaksh/grader.py | 3 +- yaksh/hook_evaluator.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++++ yaksh/models.py | 7 ++++ yaksh/settings.py | 20 +++++++---- 4 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 yaksh/hook_evaluator.py (limited to 'yaksh') diff --git a/yaksh/grader.py b/yaksh/grader.py index 086abb7..1d4e61e 100644 --- a/yaksh/grader.py +++ b/yaksh/grader.py @@ -120,7 +120,6 @@ class Grader(object): for test_case in test_case_data: test_case_instance = create_evaluator_instance(metadata, test_case) test_case_instances.append(test_case_instance) - return test_case_instances @@ -157,7 +156,7 @@ class Grader(object): except TimeoutException: error.append(self.timeout_msg) except OSError: - msg = traceback.format_exc(limit=0) + msg = traceback.format_exc() error.append("Error: {0}".format(msg)) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py new file mode 100644 index 0000000..5480849 --- /dev/null +++ b/yaksh/hook_evaluator.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +from __future__ import unicode_literals +import sys +import traceback +import os +from os.path import join +import importlib + +# Local imports +from .file_utils import copy_files, delete_files +from .base_evaluator import BaseEvaluator +from .grader import TimeoutException + + +class HookEvaluator(BaseEvaluator): + def __init__(self, metadata, test_case_data): + self.exec_scope = None + self.files = [] + + # Set metadata values + self.user_answer = metadata.get('user_answer') + self.file_paths = metadata.get('file_paths') + self.partial_grading = metadata.get('partial_grading') + + # Set test case data values + self.test_case = test_case_data.get('code') + self.weight = test_case_data.get('weight') + + def teardown(self): + # Delete the created file. + if self.files: + delete_files(self.files) + + def compile_code(self): + if self.file_paths: + self.files = copy_files(self.file_paths) + if self.exec_scope: + return None + else: + submitted = compile(self.user_answer, '', mode='exec') + self.exec_scope = {} + exec(submitted, self.exec_scope) + return self.exec_scope + + def check_code(self): + """ Function validates user answer by running an assertion based test case + against it + + Returns + -------- + Returns a tuple (success, error, test_case_weight) + + success - Boolean, indicating if code was executed successfully, correctly + weight - Float, indicating total weight of all successful test cases + error - String, error message if success is false + + returns (True, "Correct answer", 1.0) : If the student script passes all + test cases/have same output, when compared to the instructor script + + returns (False, error_msg, 0.0): If the student script fails a single + test/have dissimilar output, when compared to the instructor script. + + Returns (False, error_msg, 0.0): If mandatory arguments are not files or if + the required permissions are not given to the file(s). + """ + success = False + mark_fraction = 0.0 + try: + tb = None + _tests = compile(self.test_case, '', mode='exec') + exec(_tests, globals()) + success, err, mark_fraction = check_answer(self.user_answer) + except AssertionError: + type, value, tb = sys.exc_info() + info = traceback.extract_tb(tb) + fname, lineno, func, text = info[-1] + text = str(self.test_case) + err = "Expected Test Case:\n{0}\n" \ + "Error - {1} {2} in: {3}\n".format( + self.test_case, + type.__name__, + str(value), + text + ) + except TimeoutException: + raise + except Exception: + msg = traceback.format_exc(limit=0) + err = "Error in Test case: {0}".format(msg) + del tb + return success, err, mark_fraction + \ No newline at end of file diff --git a/yaksh/models.py b/yaksh/models.py index d65970b..cc65b9c 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1185,3 +1185,10 @@ class McqTestCase(TestCase): class HookTestCase(TestCase): code = models.TextField() weight = models.FloatField(default=1.0) + + def get_field_value(self): + return {"test_case_type": "hooktestcase", "code": self.code} + + def __str__(self): + return u'Hook Testcase | Correct: {0}'.format(self.code) + diff --git a/yaksh/settings.py b/yaksh/settings.py index 0e432cf..72f9fda 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -21,18 +21,26 @@ URL_ROOT = '' code_evaluators = { "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator" + "stdiobasedtestcase": "yaksh.python_stdio_evaluator.PythonStdIOEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" }, "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" }, "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator" + "stdiobasedtestcase": "yaksh.cpp_stdio_evaluator.CppStdIOEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" }, "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator", - "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator"}, + "stdiobasedtestcase": "yaksh.java_stdio_evaluator.JavaStdIOEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" + }, "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator", - "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator" + "stdiobasedtestcase": "yaksh.bash_stdio_evaluator.BashStdIOEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" }, - "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, + "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator", + "hooktestcase": "yaksh.hook_evaluator.HookEvaluator" + }, } -- cgit From f4b0ea30960ffe8e3ca4ffdd757c5df2f28d3371 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 6 Jan 2017 12:28:38 +0530 Subject: modified variable names in models and hook_evaluator --- yaksh/hook_evaluator.py | 34 +++++----------------------------- yaksh/models.py | 7 ++++--- 2 files changed, 9 insertions(+), 32 deletions(-) (limited to 'yaksh') diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index 5480849..0ccdd5e 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -23,7 +23,7 @@ class HookEvaluator(BaseEvaluator): self.partial_grading = metadata.get('partial_grading') # Set test case data values - self.test_case = test_case_data.get('code') + self.hook_code = test_case_data.get('hook_code') self.weight = test_case_data.get('weight') def teardown(self): @@ -31,21 +31,9 @@ class HookEvaluator(BaseEvaluator): if self.files: delete_files(self.files) - def compile_code(self): - if self.file_paths: - self.files = copy_files(self.file_paths) - if self.exec_scope: - return None - else: - submitted = compile(self.user_answer, '', mode='exec') - self.exec_scope = {} - exec(submitted, self.exec_scope) - return self.exec_scope - def check_code(self): - """ Function validates user answer by running an assertion based test case - against it - + """ Function evaluates user answer by running a python based hook code + against it. Returns -------- Returns a tuple (success, error, test_case_weight) @@ -67,21 +55,9 @@ class HookEvaluator(BaseEvaluator): mark_fraction = 0.0 try: tb = None - _tests = compile(self.test_case, '', mode='exec') - exec(_tests, globals()) + _tests = compile(self.hook_code, '', mode='exec') + exec(_tests, locals()) success, err, mark_fraction = check_answer(self.user_answer) - except AssertionError: - type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(self.test_case) - err = "Expected Test Case:\n{0}\n" \ - "Error - {1} {2} in: {3}\n".format( - self.test_case, - type.__name__, - str(value), - text - ) except TimeoutException: raise except Exception: diff --git a/yaksh/models.py b/yaksh/models.py index cc65b9c..b859faa 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1183,12 +1183,13 @@ class McqTestCase(TestCase): class HookTestCase(TestCase): - code = models.TextField() + hook_code = models.TextField() weight = models.FloatField(default=1.0) def get_field_value(self): - return {"test_case_type": "hooktestcase", "code": self.code} + return {"test_case_type": "hooktestcase", "hook_code": self.hook_code, + "weight": self.weight} def __str__(self): - return u'Hook Testcase | Correct: {0}'.format(self.code) + return u'Hook Testcase | Correct: {0}'.format(self.hook_code) -- cgit From ec5cb67b3314d05675c501cd78ce92d02ae8fde4 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 6 Jan 2017 12:33:26 +0530 Subject: added test cases for hook evaluator --- yaksh/evaluator_tests/test_bash_evaluation.py | 259 +++++++++++++++++++ yaksh/evaluator_tests/test_c_cpp_evaluation.py | 330 ++++++++++++++++++++++++ yaksh/evaluator_tests/test_code_evaluation.py | 49 ---- yaksh/evaluator_tests/test_grader_evaluation.py | 42 +++ yaksh/evaluator_tests/test_java_evaluation.py | 329 +++++++++++++++++++++++ yaksh/evaluator_tests/test_python_evaluation.py | 214 ++++++++++++++- 6 files changed, 1172 insertions(+), 51 deletions(-) delete mode 100644 yaksh/evaluator_tests/test_code_evaluation.py create mode 100644 yaksh/evaluator_tests/test_grader_evaluation.py (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 4b551d7..0662831 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -269,5 +269,264 @@ class BashStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertTrue(result.get('success')) + +class BashHookEvaluationTestCases(EvaluatorBaseTest): + + def setUp(self): + self.f_path = os.path.join(tempfile.gettempdir(), "test.txt") + with open(self.f_path, 'wb') as f: + f.write('2'.encode('ascii')) + self.in_dir = tempfile.mkdtemp() + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) + self.file_paths = None + + def tearDown(self): + os.remove(self.f_path) + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Hello, world! + """ + ) + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Goodbye, world! + """ + ) + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output('Incorrect Answer', result.get('error')) + + def test_assert_with_hook(self): + # Given + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" + " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + ) + assert_test_case = dedent(""" + #!/bin/bash + [[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 )) + """) + + assert_test_case_args = "1 2\n2 1" + + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "echo $(( $1 + $2 ))" in user_answer: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": assert_test_case, + "test_case_args":assert_test_case_args, + 'weight': 1.0 + }, + {"test_case_type": "hooktestcase", + "hook_code": hook_code, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 2.0) + + def test_multiple_hooks(self): + # Given + user_answer = dedent(""" #!/bin/bash + echo -n Hello, world! + """ + ) + + hook_code_1 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "echo -n Hello, world!" in user_answer: + success, err, mark_fraction = True, "", 0.5 + return success, err, mark_fraction + """ + ) + hook_code_2 = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code_1, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code_2, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 1.5) + + def test_infinite_loop(self): + # Given + user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" + " do echo "" > /dev/null ; done") + + hook_code = dedent("""\ + def check_answer(user_answer): + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + proc = subprocess.Popen(user_answer, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'bash' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output(self.timeout_msg, result.get('error')) + + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d734cf2..bba7bc7 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -639,5 +639,335 @@ class CppStdIOEvaluationTestCases(EvaluatorBaseTest): # Then self.assertTrue(result.get('success')) +class CppHookEvaluationTestCases(EvaluatorBaseTest): + + def setUp(self): + self.f_path = os.path.join(tempfile.gettempdir(), "test.txt") + with open(self.f_path, 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) + self.file_paths = None + + def tearDown(self): + os.remove(self.f_path) + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = dedent("""\ + #include + main() + { + printf("Hello, world!"); + } + """) + hook_code = dedent("""\ + def check_answer(user_answer): + with open("Test.c", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["gcc Test.c", "./a.out"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'cpp' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + # Given + user_answer = dedent("""\ + #include + main() + { + printf("Goodbye, world!"); + } + """) + hook_code = dedent("""\ + def check_answer(user_answer): + with open("Test.c", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["gcc Test.c", "./a.out"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'cpp' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output('Incorrect Answer', result.get('error')) + + def test_assert_with_hook(self): + # Given + user_answer = "int add(int a, int b)\n{return a+b;}" + + + assert_test_case = dedent("""\ + #include + #include + + extern int add(int, int); + + template + + void check(T expect, T result) + { + if (expect == result) + { + printf("Correct: Expected %d got %d ",expect,result); + } + else + { + printf("Incorrect: Expected %d got %d ",expect,result); + exit (1); + } + } + + int main(void) + { + int result; + result = add(0,0); + printf("Input submitted to the function: 0, 0"); + check(0, result); + result = add(2,3); + printf("Input submitted to the function: 2 3"); + check(5,result); + printf("All Correct"); + return 0; + } + """) + + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "return a+b;" in user_answer: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": assert_test_case, + 'weight': 1.0 + }, + {"test_case_type": "hooktestcase", + "hook_code": hook_code, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'cpp' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 2.0) + + def test_multiple_hooks(self): + # Given + user_answer = dedent("""\ + #include + main() + { + printf("Hello, world!"); + } + """) + + hook_code_1 = dedent("""\ + def check_answer(user_answer): + with open("Test.c", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["gcc Test.c", "./a.out"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + hook_code_2 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if 'printf("Hello, world!");' in user_answer: + success, err, mark_fraction = True, "", 0.5 + return success, err, mark_fraction + """ + ) + + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code_1, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code_2, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'cpp' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 1.5) + + def test_infinite_loop(self): + # Given + user_answer = dedent("""\ + #include + int main(void){ + while(0==0){ + printf("abc");} + }""") + + hook_code= dedent("""\ + def check_answer(user_answer): + with open("Test.c", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["gcc Test.c", "./a.out"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'cpp' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output(self.timeout_msg, result.get('error')) + + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py deleted file mode 100644 index cb783b0..0000000 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import unicode_literals -import unittest -import os -from yaksh import python_assertion_evaluator -from yaksh.language_registry import _LanguageRegistry, get_registry -from yaksh.settings import SERVER_TIMEOUT, code_evaluators - - -class RegistryTestCase(unittest.TestCase): - def setUp(self): - self.registry_object = get_registry() - self.language_registry = _LanguageRegistry() - assertion_evaluator_path = ("yaksh.python_assertion_evaluator" - ".PythonAssertionEvaluator" - ) - stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdIOEvaluator" - ) - code_evaluators['python'] = \ - {"standardtestcase": assertion_evaluator_path, - "stdiobasedtestcase": stdio_evaluator_path - } - - def test_set_register(self): - evaluator_class = self.registry_object.get_class("python", - "standardtestcase" - ) - assertion_evaluator_path = ("yaksh.python_assertion_evaluator" - ".PythonAssertionEvaluator" - ) - stdio_evaluator_path = ("yaksh.python_stdio_evaluator." - "PythonStdIOEvaluator" - ) - class_name = getattr(python_assertion_evaluator, - 'PythonAssertionEvaluator' - ) - self.registry_object.register("python", - {"standardtestcase": assertion_evaluator_path, - "stdiobasedtestcase": stdio_evaluator_path - } - ) - self.assertEqual(evaluator_class, class_name) - - def tearDown(self): - self.registry_object = None - - -if __name__ == '__main__': - unittest.main() diff --git a/yaksh/evaluator_tests/test_grader_evaluation.py b/yaksh/evaluator_tests/test_grader_evaluation.py new file mode 100644 index 0000000..d11f4a0 --- /dev/null +++ b/yaksh/evaluator_tests/test_grader_evaluation.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals +import unittest +import os +from yaksh import python_assertion_evaluator +from yaksh.language_registry import _LanguageRegistry, get_registry +from yaksh.settings import SERVER_TIMEOUT, code_evaluators + + +class RegistryTestCase(unittest.TestCase): + def setUp(self): + self.registry_object = get_registry() + self.language_registry = _LanguageRegistry() + assertion_evaluator_path = ("yaksh.python_assertion_evaluator" + ".PythonAssertionEvaluator" + ) + stdio_evaluator_path = ("yaksh.python_stdio_evaluator." + "PythonStdIOEvaluator" + ) + + hook_evaluator_path = ("yaksh.hook_evaluator." + "HookEvaluator" + ) + code_evaluators['python'] = \ + {"standardtestcase": assertion_evaluator_path, + "stdiobasedtestcase": stdio_evaluator_path, + "hooktestcase": hook_evaluator_path + } + + def test_set_register(self): + evaluator_class = self.registry_object.get_class("python", + "standardtestcase" + ) + class_name = getattr(python_assertion_evaluator, + 'PythonAssertionEvaluator' + ) + self.assertEqual(evaluator_class, class_name) + + def tearDown(self): + self.registry_object = None + +if __name__ == '__main__': + unittest.main() diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index b53d8aa..8c9fe90 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -507,5 +507,334 @@ class JavaStdIOEvaluationTestCases(EvaluatorBaseTest): self.assertTrue(result.get("success")) +class JavaHookEvaluationTestCases(EvaluatorBaseTest): + + def setUp(self): + self.f_path = os.path.join(tempfile.gettempdir(), "test.txt") + with open(self.f_path, 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path + self.file_paths = None + gd.SERVER_TIMEOUT = 9 + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(gd.SERVER_TIMEOUT) + + def tearDown(self): + gd.SERVER_TIMEOUT = 4 + os.remove(self.f_path) + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = dedent("""\ + class Test + {public static void main(String[] args){ + System.out.print("Hello, world!"); + }} + """) + hook_code = dedent("""\ + def check_answer(user_answer): + with open("Test.java", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["javac Test.java", "java Test"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + # Given + user_answer = dedent("""\ + class Test + {public static void main(String[] args){ + System.out.print("Goodbye, world!"); + }} + """) + hook_code = dedent("""\ + def check_answer(user_answer): + with open("Test.java", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["javac Test.java", "java Test"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output('Incorrect Answer', result.get('error')) + + def test_assert_with_hook(self): + # Given + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" + assert_test_case = dedent(""" + class main + { + public static void check(E expect, E result) + { + if(result.equals(expect)) + { + System.out.println("Correct:Output expected "+expect+" and got "+result); + } + else + { + System.out.println("Incorrect:Output expected "+expect+" but got "+result); + System.exit(1); + } + } + public static void main(String arg[]) + { + Test t = new Test(); + int result, input, output; + input = 0; output = 0; + result = t.square_num(input); + System.out.println("Input submitted to the function: "+input); + check(output, result); + input = 5; output = 25; + result = t.square_num(input); + System.out.println("Input submitted to the function: "+input); + check(output, result); + input = 6; output = 36; + result = t.square_num(input); + System.out.println("Input submitted to the function: "+input); + check(output, result); + } + } + """) + + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "return a*a" in user_answer: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": assert_test_case, + 'weight': 1.0 + }, + {"test_case_type": "hooktestcase", + "hook_code": hook_code, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'java' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 2.0) + + def test_multiple_hooks(self): + # Given + user_answer = dedent("""\ + class Test + {public static void main(String[] args){ + System.out.print("Hello, world!"); + }} + """) + + hook_code_1 = dedent("""\ + def check_answer(user_answer): + with open("Test.java", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["javac Test.java", "java Test"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + hook_code_2 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if 'System.out.print("Hello, world!");' in user_answer: + success, err, mark_fraction = True, "", 0.5 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code_1, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code_2, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'java' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 1.5) + + def test_infinite_loop(self): + # Given + user_answer = dedent("""\ + class Test + {public static void main(String[] args){ + while(0==0) + { + System.out.print("a");} + }}""") + + hook_code = dedent("""\ + def check_answer(user_answer): + with open("Test.java", "w+") as f: + f.write(user_answer) + import subprocess + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + def _run_command(cmd): + proc = subprocess.Popen("{}".format(cmd), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout,stderr = proc.communicate() + return stdout,stderr + cmds = ["javac Test.java", "java Test"] + for cmd in cmds: + stdout, stderr = _run_command(cmd) + if stdout == "Hello, world!": + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'java' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output(self.timeout_msg, result.get('error')) + + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index c58d7f1..a9f8122 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -7,8 +7,6 @@ from textwrap import dedent # Local import from yaksh.grader import Grader -from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdio_evaluator import PythonStdIOEvaluator from yaksh.settings import SERVER_TIMEOUT @@ -643,5 +641,217 @@ class PythonStdIOEvaluationTestCases(EvaluatorBaseTest): self.assertFalse(result.get('success')) +class PythonHookEvaluationTestCases(EvaluatorBaseTest): + + def setUp(self): + with open('/tmp/test.txt', 'wb') as f: + f.write('2'.encode('ascii')) + tmp_in_dir_path = tempfile.mkdtemp() + self.in_dir = tmp_in_dir_path + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + self.file_paths = None + + def tearDown(self): + os.remove('/tmp/test.txt') + shutil.rmtree(self.in_dir) + + def test_correct_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn a + b" + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + exec(user_answer) + if add(1,2) == 3: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + # Given + user_answer = "def add(a,b):\n\treturn a - b" + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + exec user_answer + if add(1,2) == 3: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output('Incorrect Answer', result.get('error')) + + def test_assert_with_hook(self): + # Given + user_answer = "def add(a,b):\n\treturn a + b" + assert_test_case = "assert add(1,2) == 3" + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "return a + b" in user_answer: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + test_case_data = [{"test_case_type": "standardtestcase", + "test_case": assert_test_case, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 2.0) + + def test_multiple_hooks(self): + # Given + user_answer = "def add(a,b):\n\treturn a + b" + hook_code_1 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + if "return a + b" in user_answer: + success, err, mark_fraction = True, "", 0.5 + return success, err, mark_fraction + """ + ) + hook_code_2 = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + exec(user_answer) + if add(1,2) == 3: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + + + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code_1, 'weight': 1.0}, + {"test_case_type": "hooktestcase", + "hook_code": hook_code_2, 'weight': 1.0}, + ] + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': True, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertTrue(result.get('success')) + self.assertEqual(result.get("weight"), 1.5) + + def test_infinite_loop(self): + # Given + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + hook_code = dedent("""\ + def check_answer(user_answer): + success = False + err = "Incorrect Answer" + mark_fraction = 0.0 + exec(user_answer) + if add(1,2) == 3: + success, err, mark_fraction = True, "", 1.0 + return success, err, mark_fraction + """ + ) + test_case_data = [{"test_case_type": "hooktestcase", + "hook_code": hook_code,"weight": 1.0 + }] + + kwargs = { + 'metadata': { + 'user_answer': user_answer, + 'file_paths': self.file_paths, + 'partial_grading': False, + 'language': 'python' + }, + 'test_case_data': test_case_data, + } + + # When + grader = Grader(self.in_dir) + result = grader.evaluate(kwargs) + + # Then + self.assertFalse(result.get('success')) + self.assert_correct_output(self.timeout_msg, result.get('error')) + + if __name__ == '__main__': unittest.main() -- cgit From 6b8f8f53d7110e27bc44f0d50b8b3155155932f6 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Fri, 6 Jan 2017 14:43:06 +0530 Subject: added hook evaluator type tuple in forms and fixed limit=0 mistake in grader --- yaksh/forms.py | 7 +++++-- yaksh/grader.py | 2 +- yaksh/hook_evaluator.py | 5 +---- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'yaksh') diff --git a/yaksh/forms.py b/yaksh/forms.py index 1d18d29..8a90dee 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,6 +1,7 @@ from django import forms from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course,\ - QuestionPaper, StandardTestCase, StdIOBasedTestCase + QuestionPaper, StandardTestCase, StdIOBasedTestCase, \ + HookTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType @@ -40,7 +41,8 @@ test_case_types = ( ("standardtestcase", "Standard Testcase"), ("stdiobasedtestcase", "StdIO Based Testcase"), ("mcqtestcase", "MCQ Testcase"), - ) + ("hooktestcase", "Hook Testcase"), + ) UNAME_CHARS = letters + "._" + digits PWD_CHARS = letters + punctuation + digits @@ -296,3 +298,4 @@ class QuestionPaperForm(forms.ModelForm): class Meta: model = QuestionPaper fields = ['shuffle_questions'] + diff --git a/yaksh/grader.py b/yaksh/grader.py index 1d4e61e..a9a3738 100644 --- a/yaksh/grader.py +++ b/yaksh/grader.py @@ -156,7 +156,7 @@ class Grader(object): except TimeoutException: error.append(self.timeout_msg) except OSError: - msg = traceback.format_exc() + msg = traceback.format_exc(limit=0) error.append("Error: {0}".format(msg)) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index 0ccdd5e..6c8b2c3 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -3,8 +3,6 @@ from __future__ import unicode_literals import sys import traceback import os -from os.path import join -import importlib # Local imports from .file_utils import copy_files, delete_files @@ -14,7 +12,6 @@ from .grader import TimeoutException class HookEvaluator(BaseEvaluator): def __init__(self, metadata, test_case_data): - self.exec_scope = None self.files = [] # Set metadata values @@ -62,7 +59,7 @@ class HookEvaluator(BaseEvaluator): raise except Exception: msg = traceback.format_exc(limit=0) - err = "Error in Test case: {0}".format(msg) + err = "Error in Hook code: {0}".format(msg) del tb return success, err, mark_fraction \ No newline at end of file -- cgit From cf4f207f710b9f67e0973a99f5894c4a866bca4c Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Mon, 9 Jan 2017 12:48:37 +0530 Subject: modified tests and hook evaluator for python 3 --- yaksh/evaluator_tests/test_bash_evaluation.py | 10 ++++++---- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 8 ++++---- yaksh/evaluator_tests/test_java_evaluation.py | 8 ++++---- yaksh/evaluator_tests/test_python_evaluation.py | 8 ++++---- yaksh/hook_evaluator.py | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 0662831..482d45e 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -303,7 +303,7 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): stderr=subprocess.PIPE ) stdout,stderr = proc.communicate() - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -346,7 +346,7 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): stderr=subprocess.PIPE ) stdout,stderr = proc.communicate() - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -444,6 +444,7 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): hook_code_2 = dedent("""\ def check_answer(user_answer): import subprocess + import sys success = False err = "Incorrect Answer" mark_fraction = 0.0 @@ -452,7 +453,8 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): stderr=subprocess.PIPE ) stdout,stderr = proc.communicate() - if stdout == "Hello, world!": + + if stdout.decode('utf-8') == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -498,7 +500,7 @@ class BashHookEvaluationTestCases(EvaluatorBaseTest): stderr=subprocess.PIPE ) stdout,stderr = proc.communicate() - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index bba7bc7..304f1cb 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -684,7 +684,7 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["gcc Test.c", "./a.out"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -738,7 +738,7 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["gcc Test.c", "./a.out"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -871,7 +871,7 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["gcc Test.c", "./a.out"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -940,7 +940,7 @@ class CppHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["gcc Test.c", "./a.out"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 8c9fe90..3d127af 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -553,7 +553,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["javac Test.java", "java Test"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -606,7 +606,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["javac Test.java", "java Test"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -736,7 +736,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["javac Test.java", "java Test"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ @@ -806,7 +806,7 @@ class JavaHookEvaluationTestCases(EvaluatorBaseTest): cmds = ["javac Test.java", "java Test"] for cmd in cmds: stdout, stderr = _run_command(cmd) - if stdout == "Hello, world!": + if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index a9f8122..ef155e8 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -665,7 +665,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success = False err = "Incorrect Answer" mark_fraction = 0.0 - exec(user_answer) + exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction @@ -700,7 +700,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success = False err = "Incorrect Answer" mark_fraction = 0.0 - exec user_answer + exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction @@ -785,7 +785,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success = False err = "Incorrect Answer" mark_fraction = 0.0 - exec(user_answer) + exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction @@ -824,7 +824,7 @@ class PythonHookEvaluationTestCases(EvaluatorBaseTest): success = False err = "Incorrect Answer" mark_fraction = 0.0 - exec(user_answer) + exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index 6c8b2c3..c125b21 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -53,7 +53,7 @@ class HookEvaluator(BaseEvaluator): try: tb = None _tests = compile(self.hook_code, '', mode='exec') - exec(_tests, locals()) + exec(_tests, globals()) success, err, mark_fraction = check_answer(self.user_answer) except TimeoutException: raise -- cgit From 1a8838084cea43525575521e88017f3106d44379 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Mon, 9 Jan 2017 15:05:18 +0530 Subject: added default value to hook code in models --- yaksh/hook_evaluator.py | 2 +- yaksh/models.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'yaksh') diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index c125b21..891192c 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -36,7 +36,7 @@ class HookEvaluator(BaseEvaluator): Returns a tuple (success, error, test_case_weight) success - Boolean, indicating if code was executed successfully, correctly - weight - Float, indicating total weight of all successful test cases + mark_fraction - Float, indicating fraction of the weight to a test case error - String, error message if success is false returns (True, "Correct answer", 1.0) : If the student script passes all diff --git a/yaksh/models.py b/yaksh/models.py index b859faa..7787e17 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -3,6 +3,7 @@ import json from random import sample, shuffle from itertools import islice, cycle from collections import Counter +from textwrap import dedent from django.db import models from django.db.models import Q from django.contrib.auth.models import User @@ -1183,7 +1184,25 @@ class McqTestCase(TestCase): class HookTestCase(TestCase): - hook_code = models.TextField() + hook_code = models.TextField(default=dedent\ + ("""\ + def check_answer(user_answer): + ''' Evaluates user answer to return - + success - Boolean, indicating if code was executed correctly + mark_fraction - Float, indicating fraction of the + weight to a test case + error - String, error message if success is false''' + success = False + err = "Incorrect Answer" # Please make the error message more specific + mark_fraction = 0.0 + + # write your code here + + return success, err, mark_fraction + + """) + + ) weight = models.FloatField(default=1.0) def get_field_value(self): -- cgit From db4b15ada4b078910442984776dbfc20e5955a8c Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Wed, 11 Jan 2017 14:37:25 +0530 Subject: added docs to hook_evaluator --- yaksh/documentation/images/hook_testcase.jpg | Bin 0 -> 63419 bytes .../moderator_docs/creating_question.rst | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 yaksh/documentation/images/hook_testcase.jpg (limited to 'yaksh') diff --git a/yaksh/documentation/images/hook_testcase.jpg b/yaksh/documentation/images/hook_testcase.jpg new file mode 100644 index 0000000..3018050 Binary files /dev/null and b/yaksh/documentation/images/hook_testcase.jpg differ diff --git a/yaksh/documentation/moderator_docs/creating_question.rst b/yaksh/documentation/moderator_docs/creating_question.rst index ec273f5..f99bf7f 100644 --- a/yaksh/documentation/moderator_docs/creating_question.rst +++ b/yaksh/documentation/moderator_docs/creating_question.rst @@ -234,6 +234,24 @@ How to write Test cases For MCC based question, check the correct checkbox for multiple correct options. + * **Create Hook based Test Case** + + Select Hook from Add Test Case field. + + In Hook based test case type, moderator is provided with a evaluator function + called **check_answer** which is provided with a parameter called **user_answer**. + + **user_answer** is the code of the student in string format. + + A moderator can check the string for specific words in the user answer + and/or compile and execute the user answer (using standard python libraries) to + evaluate and hence return the mark fraction. + + + .. image:: ../images/hook_testcase.jpg + :width: 80% + + Features in Question -------------------- -- cgit From 987efbcba6e9976d1351a35454a62d6b8305009d Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Wed, 11 Jan 2017 23:46:26 +0530 Subject: check_answer executed in custom scope instead of globals --- yaksh/hook_evaluator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yaksh') diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index 891192c..3956da1 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -53,8 +53,10 @@ class HookEvaluator(BaseEvaluator): try: tb = None _tests = compile(self.hook_code, '', mode='exec') - exec(_tests, globals()) - success, err, mark_fraction = check_answer(self.user_answer) + hook_scope = {} + exec(_tests, hook_scope) + check = hook_scope["check_answer"] + success, err, mark_fraction = check(self.user_answer) except TimeoutException: raise except Exception: -- cgit