1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
#!/usr/bin/env python
from __future__ import unicode_literals
import sys
import traceback
import os
from os.path import join
import importlib
# Local imports
from .file_utils import copy_files, delete_files
from .base_evaluator import BaseEvaluator
from .code_evaluator import TimeoutException
class PythonAssertionEvaluator(BaseEvaluator):
"""Tests the Python code obtained from Code Server"""
def __init__(self, metadata, test_case_data):
self.exec_scope = None
self.files = []
# Set metadata values
self.user_answer = metadata.get('user_answer')
self.file_paths = metadata.get('file_paths')
self.partial_grading = metadata.get('partial_grading')
# Set test case data values
self.test_case = test_case_data.get('test_case')
self.weight = test_case_data.get('weight')
# def __del__(self):
# # Delete the created file.
# if self.files:
# delete_files(self.files)
# def setup(self):
# super(PythonAssertionEvaluator, self).setup()
# self.exec_scope = None
# self.files = []
# def teardown(self):
# # Delete the created file.
# if self.files:
# delete_files(self.files)
# super(PythonAssertionEvaluator, self).teardown()
# def teardown(self):
# # Delete the created file.
# if self.files:
# delete_files(self.files)
# # Cancel the signal
# delete_signal_handler()
# self._change_dir(dirname(MY_DIR))
def compile_code(self):
if self.file_paths:
self.files = copy_files(self.file_paths)
if self.exec_scope:
return None
else:
submitted = compile(self.user_answer, '<string>', mode='exec')
self.exec_scope = {}
exec(submitted, self.exec_scope)
return self.exec_scope
def check_code(self):
""" Function validates user answer by running an assertion based test case
against it
Returns
--------
Returns a tuple (success, error, test_case_weight)
success - Boolean, indicating if code was executed successfully, correctly
weight - Float, indicating total weight of all successful test cases
error - String, error message if success is false
returns (True, "Correct answer", 1.0) : If the student script passes all
test cases/have same output, when compared to the instructor script
returns (False, error_msg, 0.0): If the student script fails a single
test/have dissimilar output, when compared to the instructor script.
Returns (False, error_msg, 0.0): If mandatory arguments are not files or if
the required permissions are not given to the file(s).
"""
success = False
test_case_weight = 0.0
try:
tb = None
_tests = compile(self.test_case, '<string>', mode='exec')
exec(_tests, self.exec_scope)
except AssertionError:
type, value, tb = sys.exc_info()
info = traceback.extract_tb(tb)
fname, lineno, func, text = info[-1]
text = str(self.test_case).splitlines()[lineno-1]
err = ("-----\nExpected Test Case:\n{0}\n"
"Error - {1} {2} in: {3}\n-----").format(self.test_case, type.__name__, str(value), text)
except TimeoutException:
raise
except Exception:
msg = traceback.format_exc(limit=0)
err = "Error in Test case: {0}".format(msg)
else:
success = True
err = '-----\nCorrect answer\nTest Case: {0}\n-----'.format(self.test_case)
test_case_weight = float(self.weight) if self.partial_grading else 0.0
del tb
return success, err, test_case_weight
|
