diff options
| -rw-r--r-- | yaksh/code_evaluator.py | 6 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 159 | ||||
| -rw-r--r-- | yaksh/python_assertion_evaluator.py | 41 | ||||
| -rw-r--r-- | yaksh/python_code_evaluator.py | 98 | ||||
| -rw-r--r-- | yaksh/python_stdout_evaluator.py | 15 | ||||
| -rw-r--r-- | yaksh/settings.py | 2 |
6 files changed, 149 insertions, 172 deletions
diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 7e2a729..535daa3 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -52,9 +52,7 @@ class CodeEvaluator(object): """Tests the code obtained from Code Server""" # def __init__(self, test_case_data, test, language, user_answer, # ref_code_path=None, in_dir=None): - def __init__(self, in_dir, **kwargs): - - + def __init__(self, in_dir=None): msg = 'Code took more than %s seconds to run. You probably '\ 'have an infinite loop in your code.' % SERVER_TIMEOUT self.timeout_msg = msg @@ -64,7 +62,7 @@ class CodeEvaluator(object): # self.ref_code_path = ref_code_path #@@@specific to check-code # self.test = test #@@@specific to check-code self.in_dir = in_dir #@@@Common for all, no change - self.test_case_args = None #@@@no change + # self.test_case_args = None #@@@no change # Public Protocol ########## # @classmethod diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index c55f04f..e157cde 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -1,56 +1,47 @@ import unittest import os -from yaksh.python_code_evaluator import PythonCodeEvaluator +from yaksh.python_assertion_evaluator import PythonAssertionEvaluator +from yaksh.python_stdout_evaluator import PythonStdoutEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent -class PythonEvaluationTestCases(unittest.TestCase): +class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "Python" - self.test = None - self.test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} - }] + self.test_case_data = ['assert(add(1,2)==3)'] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = dedent(""" - def add(a, b): - return a + b - """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + # {u'user_answer': u'def adder(a,b):\r\n return a', u'test_case_data': [u'assert(adder(1,2)==3)']} + user_answer = "def add(a,b):\n\treturn a + b" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEqual(result.get('error'), "Correct answer") def test_incorrect_answer(self): - user_answer = dedent(""" - def add(a, b): - return a - b - """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertFalse(result.get("success")) - self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") + user_answer = "def add(a,b):\n\treturn a - b" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)") def test_infinite_loop(self): - user_answer = dedent(""" - def add(a, b): - while True: - pass - """) - timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), timeout_msg) + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) def test_syntax_error(self): user_answer = dedent(""" @@ -59,9 +50,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) syntax_error_msg = ["Traceback", "call", "File", "line", "<string>", "SyntaxError", "invalid syntax"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) @@ -75,9 +68,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) indent_error_msg = ["Traceback", "call", "File", "line", "<string>", "IndentationError", "indented block"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) @@ -87,9 +82,11 @@ class PythonEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -103,9 +100,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) recursion_error_msg = ["Traceback", "call", "RuntimeError", "maximum recursion depth exceeded"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -118,9 +117,11 @@ class PythonEvaluationTestCases(unittest.TestCase): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -134,14 +135,54 @@ class PythonEvaluationTestCases(unittest.TestCase): return int(a) + int(b) + int(c) """) value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) +class PythonStdoutEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.output = ['Hello World\\n'] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertEqual(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = "a = 'Goodbye'\nb = 'Name'\nprint '{0} {1}'.format(a, b)" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), "Incorrect Answer") + + def test_infinite_loop(self): + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), 'Incorrect Answer') + +>>>>>>> - Add test cases for multiple python evaluators + if __name__ == '__main__': - unittest.main()
\ No newline at end of file + unittest.main() diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py new file mode 100644 index 0000000..0615d84 --- /dev/null +++ b/yaksh/python_assertion_evaluator.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib + +# local imports +from code_evaluator import CodeEvaluator, TimeoutException + + +class PythonAssertionEvaluator(CodeEvaluator): + """Tests the Python code obtained from Code Server""" + + # def check_code(self, test, user_answer, ref_code_path): + def check_code(self, user_answer, test_case_data): + success = False + try: + tb = None + submitted = compile(user_answer, '<string>', mode='exec') + g = {} + exec submitted in g + for test_code in test_case_data: + _tests = compile(test_code, '<string>', mode='exec') + exec _tests in g + except AssertionError: + type, value, tb = sys.exc_info() + info = traceback.extract_tb(tb) + fname, lineno, func, text = info[-1] + text = str(test_code).splitlines()[lineno-1] + err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + except TimeoutException: + raise + except Exception: + err = traceback.format_exc(limit=0) + else: + success = True + err = 'Correct answer' + + del tb + return success, err diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py deleted file mode 100644 index a131a0e..0000000 --- a/yaksh/python_code_evaluator.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -import sys -import traceback -import os -from os.path import join -import importlib - -# local imports -from code_evaluator import CodeEvaluator, TimeoutException - - -class PythonCodeEvaluator(CodeEvaluator): - """Tests the Python code obtained from Code Server""" - - # def check_code(self, test, user_answer, ref_code_path): - def check_code(self, user_answer, test_cases): - success = False - - try: - tb = None - submitted = compile(user_answer, '<string>', mode='exec') - g = {} - exec submitted in g - for test_code in test_cases: - _tests = compile(test_code, '<string>', mode='exec') - exec _tests in g - except AssertionError: - type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(test_code).splitlines()[lineno-1] - err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - except TimeoutException: - raise - except Exception: - err = traceback.format_exc(limit=0) - else: - success = True - err = 'Correct answer' - - del tb - return success, err - - # def unpack_test_case_data(self, test_case_data): - # test_cases = [] - # for t in test_case_data: - # test_case = t.get('test_case') - # test_cases.append(test_case) - - # return test_cases - - # def check_code(self): - # success = False - - # try: - # tb = None - # test_code = self._create_test_case() - # submitted = compile(self.user_answer, '<string>', mode='exec') - # g = {} - # exec submitted in g - # _tests = compile(test_code, '<string>', mode='exec') - # exec _tests in g - # except AssertionError: - # type, value, tb = sys.exc_info() - # info = traceback.extract_tb(tb) - # fname, lineno, func, text = info[-1] - # text = str(test_code).splitlines()[lineno-1] - # err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - # else: - # success = True - # err = 'Correct answer' - - # del tb - # return success, err - - # def _create_test_case(self): - # """ - # Create assert based test cases in python - # """ - # test_code = "" - # if self.test: - # return self.test - # elif self.test_case_data: - # for test_case in self.test_case_data: - # pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \ - # if test_case.get('pos_args') else "" - # kw_args = ", ".join(str(k+"="+a) for k, a - # in test_case.get('kw_args').iteritems()) \ - # if test_case.get('kw_args') else "" - # args = pos_args + ", " + kw_args if pos_args and kw_args \ - # else pos_args or kw_args - # function_name = test_case.get('func_name') - # expected_answer = test_case.get('expected_answer') - - # tcode = "assert {0}({1}) == {2}".format(function_name, args, - # expected_answer) - # test_code += tcode + "\n" - # return test_code diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 28c3372..8f3eb65 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -25,29 +25,24 @@ def redirect_stdout(): class PythonStdoutEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self, user_answer, test_cases): + def check_code(self, user_answer, test_case_data): success = False tb = None - expected_output = test_cases[0] + expected_output = test_case_data[0] submitted = compile(user_answer, '<string>', mode='exec') with redirect_stdout() as output_buffer: g = {} exec submitted in g raw_output_value = output_buffer.getvalue() output_value = raw_output_value.encode('string_escape').strip() - if output_value == str(test_code): + if output_value == expected_output: success = True - err = 'Correct answer' + err = "Correct answer" + else: success = False err = "Incorrect Answer" del tb return success, err - - # def unpack_test_case_data(self, test_case_data): - # for t in test_case_data: - # test_case = t.get('output') - - # return test_case diff --git a/yaksh/settings.py b/yaksh/settings.py index a4e78db..30fab0a 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,7 +20,7 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"standardtestcase": "python_code_evaluator.PythonCodeEvaluator", + "python": {"standardtestcase": "python_assertion_evaluator.PythonStandardEvaluator", "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" }, "c": "cpp_code_evaluator.CppCodeEvaluator", |