summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py74
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py13
-rw-r--r--yaksh/evaluator_tests/test_code_evaluation.py38
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py96
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py70
5 files changed, 227 insertions, 64 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index c6a5408..5fa3d2d 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -5,37 +5,81 @@ from yaksh.settings import SERVER_TIMEOUT
class BashEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.language = "bash"
- self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args"
+ self.test_case_data = ["bash_files/sample.sh,bash_files/sample.args"]
self.in_dir = "/tmp"
- self.test_case_data = []
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
- self.test = None
+
+ # def setUp(self):
+ # self.language = "bash"
+ # self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args"
+ # self.in_dir = "/tmp"
+ # self.test_case_data = []
+ # self.timeout_msg = ("Code took more than {0} seconds to run. "
+ # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
+ # self.test = None
def test_correct_answer(self):
user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
- get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
- self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer")
+ get_class = BashCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertEquals(result.get('error'), "Correct answer")
def test_error(self):
user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))"
- get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = BashCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("Error" in result.get("error"))
def test_infinite_loop(self):
user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done"
- get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = BashCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
+ # def test_infinite_loop(self):
+ # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done"
+ # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertFalse(result.get("success"))
+ # self.assertEquals(result.get("error"), self.timeout_msg)
+
+ # def test_correct_answer(self):
+ # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))"
+ # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertTrue(result.get("success"))
+ # self.assertEqual(result.get("error"), "Correct answer")
+
+ # def test_error(self):
+ # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))"
+ # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertFalse(result.get("success"))
+ # self.assertTrue("Error" in result.get("error"))
+
+ # def test_infinite_loop(self):
+ # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done"
+ # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertFalse(result.get("success"))
+ # self.assertEquals(result.get("error"), self.timeout_msg)
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index c5f8775..9d37fb4 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -18,7 +18,18 @@ class CEvaluationTestCases(unittest.TestCase):
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
- self.assertEqual(result.get('error'), "Correct answer")
+ self.assertEquals(result.get('error'), "Correct answer")
+
+ def test_incorrect_answer(self):
+ user_answer = "int add(int a, int b)\n{return a-b;}"
+ get_class = CppCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect:", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
def test_compilation_error(self):
user_answer = "int add(int a, int b)\n{return a+b}"
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py
index 84701fb..e4f129c 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_code_evaluation.py
@@ -1,20 +1,21 @@
import unittest
import os
-from yaksh import python_code_evaluator
-from yaksh.language_registry import _LanguageRegistry, set_registry, get_registry
+from yaksh import python_assertion_evaluator
+from yaksh.language_registry import _LanguageRegistry, get_registry
from yaksh.settings import SERVER_TIMEOUT
class RegistryTestCase(unittest.TestCase):
def setUp(self):
- set_registry()
self.registry_object = get_registry()
self.language_registry = _LanguageRegistry()
def test_set_register(self):
- class_name = getattr(python_code_evaluator, 'PythonCodeEvaluator')
- self.registry_object.register("python", "yaksh.python_code_evaluator.PythonCodeEvaluator")
- self.assertEquals(self.registry_object.get_class("python"), class_name)
+ class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator')
+ self.registry_object.register("python", {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator",
+ "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator"
+ })
+ self.assertEquals(self.registry_object.get_class("python", "standardtestcase"), class_name)
def tearDown(self):
self.registry_object = None
@@ -22,3 +23,28 @@ class RegistryTestCase(unittest.TestCase):
if __name__ == '__main__':
unittest.main()
+
+
+# import unittest
+# import os
+# from yaksh import cpp_code_evaluator
+# from yaksh.language_registry import _LanguageRegistry, get_registry
+# from yaksh.settings import SERVER_TIMEOUT
+
+
+# class RegistryTestCase(unittest.TestCase):
+# def setUp(self):
+# self.registry_object = get_registry()
+# self.language_registry = _LanguageRegistry()
+
+# def test_set_register(self):
+# class_name = getattr(cpp_code_evaluator, 'CppCodeEvaluator')
+# self.registry_object.register("c", {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"})
+# self.assertEquals(self.registry_object.get_class("c", "standardtestcase"), class_name)
+
+# def tearDown(self):
+# self.registry_object = None
+
+
+# if __name__ == '__main__':
+# unittest.main()
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index dfa1066..a5852a3 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -2,52 +2,104 @@ import unittest
import os
from yaksh import code_evaluator as evaluator
from yaksh.java_code_evaluator import JavaCodeEvaluator
+from yaksh.settings import SERVER_TIMEOUT
class JavaEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.language = "java"
- self.ref_code_path = "java_files/main_square.java"
+ self.test_case_data = ["java_files/main_square.java"]
self.in_dir = "/tmp"
- self.test_case_data = []
evaluator.SERVER_TIMEOUT = 9
self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop in "
- "your code.").format(evaluator.SERVER_TIMEOUT)
- self.test = None
+ "You probably have an infinite loop in your code.").format(evaluator.SERVER_TIMEOUT)
def tearDown(self):
evaluator.SERVER_TIMEOUT = 2
def test_correct_answer(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
- get_class = JavaCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
+ get_class = JavaCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct answer")
+ self.assertTrue(result.get('success'))
- self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer")
+ def test_incorrect_answer(self):
+ user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
+ get_class = JavaCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect:", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
def test_error(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
- get_class = JavaCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = JavaCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("Error" in result.get("error"))
def test_infinite_loop(self):
user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
- get_class = JavaCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = JavaCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
+ # def setUp(self):
+ # self.language = "java"
+ # self.ref_code_path = "java_files/main_square.java"
+ # self.in_dir = "/tmp"
+ # self.test_case_data = []
+ # evaluator.SERVER_TIMEOUT = 9
+ # self.timeout_msg = ("Code took more than {0} seconds to run. "
+ # "You probably have an infinite loop in "
+ # "your code.").format(evaluator.SERVER_TIMEOUT)
+ # self.test = None
+
+ # def tearDown(self):
+ # evaluator.SERVER_TIMEOUT = 2
+
+ # def test_correct_answer(self):
+ # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
+ # get_class = JavaCodeEvaluator(self.test_case_data, self.test,
+ # self.language, user_answer,
+ # self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertTrue(result.get("success"))
+ # self.assertEqual(result.get("error"), "Correct answer")
+
+ # def test_error(self):
+ # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
+ # get_class = JavaCodeEvaluator(self.test_case_data, self.test,
+ # self.language, user_answer,
+ # self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertFalse(result.get("success"))
+ # self.assertTrue("Error" in result.get("error"))
+
+ # def test_infinite_loop(self):
+ # user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
+ # get_class = JavaCodeEvaluator(self.test_case_data, self.test,
+ # self.language, user_answer,
+ # self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
+
+ # self.assertFalse(result.get("success"))
+ # self.assertEquals(result.get("error"), self.timeout_msg)
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 30af041..2b459fc 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -1,45 +1,75 @@
import unittest
import os
+from yaksh import code_evaluator as evaluator
from yaksh.scilab_code_evaluator import ScilabCodeEvaluator
from yaksh.settings import SERVER_TIMEOUT
class ScilabEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.language = "scilab"
- self.ref_code_path = "scilab_files/test_add.sce"
+ self.test_case_data = ["scilab_files/test_add.sce"]
self.in_dir = "/tmp"
- self.test_case_data = []
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
- self.test = None
+
+ # def setUp(self):
+ # self.language = "scilab"
+ # self.ref_code_path = "scilab_files/test_add.sce"
+ # self.in_dir = "/tmp"
+ # self.test_case_data = []
+ # self.timeout_msg = ("Code took more than {0} seconds to run. "
+ # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
+ # self.test = None
+
def test_correct_answer(self):
user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction"
- get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
+ get_class = ScilabCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct answer")
+ self.assertTrue(result.get('success'))
+
+ # def test_correct_answer(self):
+ # user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction"
+ # get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
+ # self.language, user_answer,
+ # self.ref_code_path, self.in_dir)
+ # result = get_class.evaluate()
- self.assertTrue(result.get("success"))
- self.assertEqual(result.get("error"), "Correct answer")
+ # self.assertTrue(result.get("success"))
+ # self.assertEqual(result.get("error"), "Correct answer")
def test_error(self):
user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction"
- get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = ScilabCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertTrue("error" in result.get("error"))
+
+ def test_incorrect_answer(self):
+ user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction"
+ get_class = ScilabCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Message", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
+
def test_infinite_loop(self):
user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a;\nwhile(1==1)\nend\nendfunction"
- get_class = ScilabCodeEvaluator(self.test_case_data, self.test,
- self.language, user_answer,
- self.ref_code_path, self.in_dir)
- result = get_class.evaluate()
-
+ get_class = ScilabCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
generated by cgit (git 2.34.1) at 2024-12-21 17:47:56 +0000