From d8876ae789197de4d977d28f0c0ff04cf39c2c35 Mon Sep 17 00:00:00 2001 From: adityacp Date: Mon, 20 Jun 2016 14:57:21 +0530 Subject: added test cases for file based questions --- yaksh/evaluator_tests/test_bash_evaluation.py | 26 ++++++-- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 41 +++++++++++-- yaksh/evaluator_tests/test_java_evaluation.py | 48 +++++++++++++-- yaksh/evaluator_tests/test_python_evaluation.py | 79 ++++++++++++++++++++----- yaksh/evaluator_tests/test_scilab_evaluation.py | 15 +++-- 5 files changed, 173 insertions(+), 36 deletions(-) (limited to 'yaksh/evaluator_tests') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 4ff3e0a..1070c6a 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -8,10 +8,11 @@ class BashEvaluationTestCases(unittest.TestCase): self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args"} ] - self.in_dir = "/tmp" + self.in_dir = os.getcwd() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" @@ -19,7 +20,8 @@ class BashEvaluationTestCases(unittest.TestCase): ) get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) @@ -30,7 +32,8 @@ class BashEvaluationTestCases(unittest.TestCase): "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) @@ -41,12 +44,27 @@ class BashEvaluationTestCases(unittest.TestCase): " do echo "" > /dev/null ; done") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + def test_file_based_assert(self): + self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.test_case_data = [ + {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"} + ] + user_answer = ("#!/bin/bash\ncat $1") + get_class = BashCodeEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get("success")) + self.assertEquals(result.get("error"), "Correct answer") if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 71fb843..71af177 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -2,20 +2,23 @@ import unittest import os from yaksh.cpp_code_evaluator import CppCodeEvaluator from yaksh.settings import SERVER_TIMEOUT +from textwrap import dedent class CEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] - self.in_dir = "/tmp" + self.in_dir = os.getcwd() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) @@ -25,7 +28,8 @@ class CEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a-b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -36,7 +40,8 @@ class CEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{return a+b}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) @@ -46,11 +51,37 @@ class CEvaluationTestCases(unittest.TestCase): user_answer = "int add(int a, int b)\n{while(1>0){}}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + def test_file_based_assert(self): + self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}] + user_answer = dedent(""" + #include + char ans() + { + FILE *fp; + char buff[255]; + fp = fopen("test.txt", "r"); + fscanf(fp, "%s", buff); + fclose(fp); + return buff[0]; + } + """) + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEquals(result.get('error'), "Correct answer") + + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 801277f..ed28745 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -3,18 +3,19 @@ import os from yaksh import code_evaluator as evaluator from yaksh.java_code_evaluator import JavaCodeEvaluator from yaksh.settings import SERVER_TIMEOUT - +from textwrap import dedent class JavaEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [ {"test_case": "java_files/main_square.java"} ] - self.in_dir = "/tmp" + self.in_dir = os.getcwd() evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(evaluator.SERVER_TIMEOUT) + self.file_paths = None def tearDown(self): evaluator.SERVER_TIMEOUT = 2 @@ -23,7 +24,8 @@ class JavaEvaluationTestCases(unittest.TestCase): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertEquals(result.get('error'), "Correct answer") @@ -33,7 +35,8 @@ class JavaEvaluationTestCases(unittest.TestCase): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -44,7 +47,8 @@ class JavaEvaluationTestCases(unittest.TestCase): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) @@ -54,12 +58,44 @@ class JavaEvaluationTestCases(unittest.TestCase): user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" get_class = JavaCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + def test_file_based_assert(self): + self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + self.test_case_data = [ + {"test_case": "java_files/read_file.java"} + ] + user_answer = dedent(""" + import java.io.BufferedReader; + import java.io.FileReader; + import java.io.IOException; + class Test{ + String readFile() throws IOException { + BufferedReader br = new BufferedReader(new FileReader("test.txt")); + try { + StringBuilder sb = new StringBuilder(); + String line = br.readLine(); + while (line != null) { + sb.append(line); + line = br.readLine();} + return sb.toString(); + } finally { + br.close(); + }}} + """) + get_class = JavaCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get("success")) + self.assertEquals(result.get("error"), "Correct answer") if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 1e867a3..b432630 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -15,13 +15,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = "def add(a,b):\n\treturn a + b" get_class = PythonAssertionEvaluator() - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer") @@ -30,8 +32,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): user_answer = "def add(a,b):\n\treturn a - b" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), @@ -42,7 +45,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -63,7 +67,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -86,7 +91,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -105,7 +111,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -126,7 +133,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -148,7 +156,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -171,7 +180,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() @@ -180,18 +190,37 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): for msg in value_error_msg: self.assertIn(msg, result.get("error")) + def test_file_based_assert(self): + self.test_case_data = [{"test_case": "assert(ans()=='2')"}] + self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + user_answer = dedent(""" + def ans(): + with open("test.txt") as f: + return f.read()[0] + """) + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + result = get_class.evaluate(**kwargs) + self.assertEqual(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) + class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"expected_output": "0 1 1 2 3"}] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") @@ -201,7 +230,8 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -214,7 +244,8 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): ) get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -224,12 +255,28 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), 'Incorrect Answer') + def test_file_based_answer(self): + self.test_case_data = [{"expected_output": "2\n\n"}] + self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] + user_answer = dedent(""" + with open("test.txt") as f: + print f.read() + """) + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths + } + result = get_class.evaluate(**kwargs) + self.assertEqual(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 242f260..f5e3767 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -7,17 +7,19 @@ from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] - self.in_dir = "/tmp" + self.in_dir = os.getcwd() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop" " in your code.").format(SERVER_TIMEOUT) + self.file_paths = None def test_correct_answer(self): user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" "\n\tc=a+b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertEquals(result.get('error'), "Correct answer") @@ -28,7 +30,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): "\n\tc=a+b;\ndis(\tendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) @@ -40,7 +43,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): "\n\tc=a-b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -52,7 +56,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): "\n\tc=a;\nwhile(1==1)\nend\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data + 'test_case_data': self.test_case_data, + 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) -- cgit