summaryrefslogtreecommitdiff
path: root/yaksh/python_assertion_evaluator.py
blob: 35a08ec6329c8d2c21368956b4d67e5b8d4c548f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
#!/usr/bin/env python
import sys
import traceback
import os
import re
from os.path import join
from textwrap import dedent
import importlib

# Local imports
from .file_utils import copy_files, delete_files
from .base_evaluator import BaseEvaluator
from .grader import TimeoutException


class PythonAssertionEvaluator(BaseEvaluator):
    """Tests the Python code obtained from Code Server"""

    def __init__(self, metadata, test_case_data):
        self.exec_scope = None
        self.files = []

        # Set metadata values
        self.user_answer = metadata.get('user_answer')
        self.file_paths = metadata.get('file_paths')
        self.partial_grading = metadata.get('partial_grading')

        # Set test case data values
        self.test_case = test_case_data.get('test_case')
        self.weight = test_case_data.get('weight')

    def teardown(self):
        # Delete the created file.
        if self.files:
            delete_files(self.files)

    def compile_code(self):
        if self.file_paths:
            self.files = copy_files(self.file_paths)
        if self.exec_scope:
            return None
        else:
            submitted = compile(self.user_answer, '<string>', mode='exec')
            self.exec_scope = {}
            exec(submitted, self.exec_scope)
            return self.exec_scope

    def check_code(self):
        """ Function validates user answer by running an assertion based test case
        against it

        Returns
        --------
        Returns a tuple (success, error, test_case_weight)

        success - Boolean, indicating if code was executed successfully, correctly
        weight - Float, indicating total weight of all successful test cases
        error - String, error message if success is false

        returns (True, "Correct answer", 1.0) : If the student script passes all
        test cases/have same output, when compared to the instructor script

        returns (False, error_msg, 0.0): If the student script fails a single
        test/have dissimilar output, when compared to the instructor script.

        Returns (False, error_msg, 0.0): If mandatory arguments are not files or if
        the required permissions are not given to the file(s).
        """
        success = False
        mark_fraction = 0.0
        try:
            tb = None
            _tests = compile(self.test_case, '<string>', mode='exec')
            exec(_tests, self.exec_scope)
        except TimeoutException:
            raise
        except AssertionError:
            type, _, tb = sys.exc_info()
            tb_info = traceback.extract_tb(tb)
            filename, line, func, text = tb_info[-1]
            value = "Expected answer from the test case didnt match the output"
            err = {"type": "assertion",
                    "test_case": self.test_case,
                    "exception": type.__name__,
                    "message": value
                    }
        except Exception:
            type, value, tb = sys.exc_info()
            tb_info = traceback.extract_tb(tb)
            filename, line, func, text = tb_info[-1]
            err = {"type": "assertion",
                    "test_case": self.test_case,
                    "exception": type.__name__,
                    "message": str(value)
                    }   

        else:
            success = True
            err = None
            mark_fraction = 1.0 if self.partial_grading else 0.0
        del tb
        return success, err, mark_fraction