diff options
Diffstat (limited to 'yaksh/evaluator_tests')
| -rw-r--r-- | yaksh/evaluator_tests/test_bash_evaluation.py | 17 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 7 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_code_evaluation.py | 35 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_java_evaluation.py | 7 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_python_evaluation.py | 107 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_scilab_evaluation.py | 15 |
6 files changed, 131 insertions, 57 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 7c58c43..4ff3e0a 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -5,13 +5,18 @@ from yaksh.settings import SERVER_TIMEOUT class BashEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = [{"test_case": "bash_files/sample.sh,bash_files/sample.args"}] + self.test_case_data = [ + {"test_case": "bash_files/sample.sh,bash_files/sample.args"} + ] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" + " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + ) get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -21,7 +26,8 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct answer") def test_error(self): - user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " + "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -31,7 +37,8 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): - user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" + " do echo "" > /dev/null ; done") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index a07150d..71fb843 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -8,14 +8,15 @@ class CEvaluationTestCases(unittest.TestCase): self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEquals(result.get('error'), "Correct answer") diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index ace6115..51c0c51 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -2,20 +2,43 @@ import unittest import os from yaksh import python_assertion_evaluator from yaksh.language_registry import _LanguageRegistry, get_registry -from yaksh.settings import SERVER_TIMEOUT +from yaksh.settings import SERVER_TIMEOUT, code_evaluators class RegistryTestCase(unittest.TestCase): def setUp(self): self.registry_object = get_registry() self.language_registry = _LanguageRegistry() + assertion_evaluator_path = ("yaksh.python_assertion_evaluator" + ".PythonAssertionEvaluator" + ) + stdout_evaluator_path = ("yaksh.python_stdout_evaluator." + "PythonStdoutEvaluator" + ) + code_evaluators['python'] = \ + {"standardtestcase": assertion_evaluator_path, + "stdoutbasedtestcase": stdout_evaluator_path + } def test_set_register(self): - class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator') - self.registry_object.register("python", {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" - }) - self.assertEquals(self.registry_object.get_class("python", "standardtestcase"), class_name) + evaluator_class = self.registry_object.get_class("python", + "standardtestcase" + ) + assertion_evaluator_path = ("yaksh.python_assertion_evaluator" + ".PythonAssertionEvaluator" + ) + stdout_evaluator_path = ("yaksh.python_stdout_evaluator." + "PythonStdoutEvaluator" + ) + class_name = getattr(python_assertion_evaluator, + 'PythonAssertionEvaluator' + ) + self.registry_object.register("python", + {"standardtestcase": assertion_evaluator_path, + "stdoutbasedtestcase": stdout_evaluator_path + } + ) + self.assertEquals(evaluator_class, class_name) def tearDown(self): self.registry_object = None diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 76a3fcf..801277f 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -7,11 +7,14 @@ from yaksh.settings import SERVER_TIMEOUT class JavaEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = [{"test_case": "java_files/main_square.java"}] + self.test_case_data = [ + {"test_case": "java_files/main_square.java"} + ] self.in_dir = "/tmp" evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(evaluator.SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(evaluator.SERVER_TIMEOUT) def tearDown(self): evaluator.SERVER_TIMEOUT = 2 diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index f6ac0bf..1e867a3 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -9,11 +9,12 @@ from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, - ] + {"test_case": 'assert(add(-1,2)==1)'}, + {"test_case": 'assert(add(-1,-2)==-3)'}, + ] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "def add(a,b):\n\treturn a + b" @@ -33,14 +34,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)") + self.assertEqual(result.get('error'), + "AssertionError in: assert(add(1,2)==3)" + ) def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg) @@ -50,12 +53,18 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b); return a + b """) - syntax_error_msg = ["Traceback", "call", "File", "line", "<string>", - "SyntaxError", "invalid syntax"] + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "<string>", + "SyntaxError", + "invalid syntax" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -68,12 +77,17 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b): return a + b """) - indent_error_msg = ["Traceback", "call", "File", "line", "<string>", - "IndentationError", "indented block"] + indent_error_msg = ["Traceback", "call", + "File", + "line", + "<string>", + "IndentationError", + "indented block" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -83,11 +97,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" - name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] + name_error_msg = ["Traceback", + "call", + "NameError", + "name", + "defined" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -100,12 +119,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b): return add(3, 3) """) - recursion_error_msg = ["Traceback", "call", "RuntimeError", - "maximum recursion depth exceeded"] + recursion_error_msg = ["Traceback", + "call", + "RuntimeError", + "maximum recursion depth exceeded" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -118,11 +140,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a): return a + b """) - type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] + type_error_msg = ["Traceback", + "call", + "TypeError", + "exactly", + "argument" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -136,7 +163,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): c = 'a' return int(a) + int(b) + int(c) """) - value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] + value_error_msg = ["Traceback", + "call", + "ValueError", + "invalid literal", + "base" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -152,14 +184,15 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"expected_output": "0 1 1 2 3"}] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -168,19 +201,21 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), "Incorrect Answer") def test_direct_printed_answer(self): user_answer = "print '0 1 1 2 3'" - error_msg = "Incorrect Answer: Please avoid printing the expected output directly" + error_msg = ("Incorrect Answer: Please avoid printing" + " the expected output directly" + ) get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), error_msg) @@ -189,8 +224,8 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), 'Incorrect Answer') diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 24d6740..242f260 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -9,10 +9,12 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a+b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -22,7 +24,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(result.get('success')) def test_error(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a+b;\ndis(\tendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -33,7 +36,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a-b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -44,7 +48,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(result.get('error').splitlines > 1) def test_infinite_loop(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a;\nwhile(1==1)\nend\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a;\nwhile(1==1)\nend\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data |