diff options
| -rw-r--r-- | yaksh/cpp_stdio_evaluator.py | 105 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_c_cpp_evaluation.py | 262 | ||||
| -rw-r--r-- | yaksh/evaluator_tests/test_java_evaluation.py | 125 | ||||
| -rw-r--r-- | yaksh/java_stdio_evaluator.py | 77 | ||||
| -rw-r--r-- | yaksh/models.py | 12 | ||||
| -rw-r--r-- | yaksh/settings.py | 11 | ||||
| -rw-r--r-- | yaksh/stdio_evaluator.py | 23 |
7 files changed, 606 insertions, 9 deletions
diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py new file mode 100644 index 0000000..28b0a0b --- /dev/null +++ b/yaksh/cpp_stdio_evaluator.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +import subprocess +import os +from os.path import isfile + +#local imports +from code_evaluator import CodeEvaluator +from stdio_evaluator import Evaluator + + +class CppStdioEvaluator(CodeEvaluator): + """Evaluates C StdIO based code""" + + def setup(self): + super(CppStdioEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('main.c') + + def teardown(self): + super(CppStdioEvaluator, self).teardown() + os.remove(self.submit_code_path) + + def set_file_paths(self): + user_output_path = os.getcwd() + '/output' + ref_output_path = os.getcwd() + '/executable' + + return user_output_path, ref_output_path + + def get_commands(self, user_output_path, ref_output_path): + compile_command = 'g++ {0} -c -o {1}'.format(self.submit_code_path, + user_output_path) + compile_main = 'g++ {0} -o {1}'.format(user_output_path, + ref_output_path) + + return compile_command, compile_main + + def compile_code(self, user_answer, expected_input, expected_output): + + if not isfile(self.submit_code_path): + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg + self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_output_path, self.ref_output_path = self.set_file_paths() + self.compile_command, self.compile_main = self.get_commands( + self.user_output_path, + self.ref_output_path + ) + self.compiled_user_answer = self._run_command(self.compile_command, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + self.compiled_test_code = self._run_command(self.compile_main, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + return self.compiled_user_answer, self.compiled_test_code + + def check_code(self, user_answer, expected_input, expected_output): + success = False + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + if stdnt_stderr == '': + proc, main_out, main_err = self.compiled_test_code + main_err = self._remove_null_substitute_char(main_err) + if main_err == '': + proc = subprocess.Popen("./executable", + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + evaluator = Evaluator() + success, err = evaluator.evaluate(user_answer, proc, + expected_input, + expected_output + ) + os.remove(self.ref_output_path) + else: + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(self.user_output_path) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 71fb843..7a87d87 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -1,7 +1,10 @@ import unittest import os from yaksh.cpp_code_evaluator import CppCodeEvaluator +from yaksh.cpp_stdio_evaluator import CppStdioEvaluator from yaksh.settings import SERVER_TIMEOUT +from textwrap import dedent + class CEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -52,5 +55,264 @@ class CEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + +class CStdioEvaluationTestCases(unittest.TestCase): + + def setUp(self): + self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + int a,b; + scanf("%d%d",&a,&b); + printf("%d",a+b); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_array_input(self): + self.test_case_data = [{'expected_output': '561', + 'expected_input': '5,6,1'}] + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + int a[3],i; + for(i=0;i<3;i++){ + scanf("%d",&a[i]);} + for(i=0;i<3;i++){ + printf("%d",a[i]);} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_string_input(self): + self.test_case_data = [{'expected_output': 'abc', + 'expected_input': 'abc'}] + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + char a[4]; + scanf("%s",a); + printf("%s",a); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + int a=10; + printf("%d",a); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) + + def test_error(self): + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + int a=10; + printf("%d",a) + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + while(0==0){ + printf("abc");} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + #include<stdio.h> + int main(void){ + int a=5,b=6; + printf("%d",a+b); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + +class CppStdioEvaluationTestCases(unittest.TestCase): + + def setUp(self): + self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + int a,b; + cin>>a>>b; + cout<<a+b; + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_array_input(self): + self.test_case_data = [{'expected_output': '561', + 'expected_input': '5,6,1'}] + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + int a[3],i; + for(i=0;i<3;i++){ + cin>>a[i];} + for(i=0;i<3;i++){ + cout<<a[i];} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_string_input(self): + self.test_case_data = [{'expected_output': 'abc', + 'expected_input': 'abc'}] + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + char a[4]; + cin>>a; + cout<<a; + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + int a=10; + cout<<a; + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) + + def test_error(self): + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + int a=10; + cout<<a + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + while(0==0){ + cout<<"abc";} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + #include<iostream> + using namespace std; + int main(void){ + int a=5,b=6; + cout<<a+b; + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 801277f..8cfccb9 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -2,7 +2,9 @@ import unittest import os from yaksh import code_evaluator as evaluator from yaksh.java_code_evaluator import JavaCodeEvaluator +from yaksh.java_stdio_evaluator import JavaStdioEvaluator from yaksh.settings import SERVER_TIMEOUT +from textwrap import dedent class JavaEvaluationTestCases(unittest.TestCase): @@ -61,5 +63,128 @@ class JavaEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get("error"), self.timeout_msg) +class JavaStdioEvaluationTestCases(unittest.TestCase): + + def setUp(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': '5\n6'}] + evaluator.SERVER_TIMEOUT = 9 + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(evaluator.SERVER_TIMEOUT) + + def teardown(self): + evaluator.SERVER_TIMEOUT = 2 + + def test_correct_answer(self): + user_answer = dedent(""" + import java.util.Scanner; + class Test + {public static void main(String[] args){ + Scanner s = new Scanner(System.in); + int a = s.nextInt(); + int b = s.nextInt(); + System.out.print(a+b); + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_array_input(self): + + self.test_case_data = [{'expected_output': '561', + 'expected_input': '5,6,1'}] + user_answer = dedent(""" + import java.util.Scanner; + class Test + {public static void main(String[] args){ + Scanner s = new Scanner(System.in); + int a[] = new int[3]; + for (int i=0;i<3;i++){ + a[i] = s.nextInt(); + System.out.print(a[i]);} + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + + user_answer = dedent(""" + import java.util.Scanner; + class Test + {public static void main(String[] args){ + Scanner s = new Scanner(System.in); + int a = s.nextInt(); + int b = s.nextInt(); + System.out.print(a); + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) + + def test_error(self): + + user_answer = dedent(""" + class Test + { + System.out.print("a"); + }""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + + user_answer = dedent(""" + class Test + {public static void main(String[] args){ + while(0==0) + { + System.out.print("a");} + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + class Test + {public static void main(String[] args){ + int a = 5; + int b = 6; + System.out.print(a+b); + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py new file mode 100644 index 0000000..2f6ef40 --- /dev/null +++ b/yaksh/java_stdio_evaluator.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +import subprocess +import os +from os.path import isfile + +#local imports +from code_evaluator import CodeEvaluator +from stdio_evaluator import Evaluator + + +class JavaStdioEvaluator(CodeEvaluator): + """Evaluates Java StdIO based code""" + + def setup(self): + super(JavaStdioEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('Test.java') + + def teardown(self): + super(JavaStdioEvaluator, self).teardown() + os.remove(self.submit_code_path) + + def set_file_paths(self, directory, file_name): + output_path = "{0}{1}.class".format(directory, file_name) + return output_path + + def get_commands(self): + compile_command = 'javac {0}'.format(self.submit_code_path) + + return compile_command + + def compile_code(self, user_answer, expected_input, expected_output): + if not isfile(self.submit_code_path): + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg + user_code_directory = os.getcwd() + '/' + self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_output_path = self.set_file_paths(user_code_directory, + 'Test' + ) + self.compile_command = self.get_commands() + self.compiled_user_answer = self._run_command(self.compile_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + return self.compiled_user_answer + + def check_code(self, user_answer, expected_input, expected_output): + success = False + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + if stdnt_stderr == '' or "error" not in stdnt_stderr: + proc = subprocess.Popen("java Test", + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + evaluator = Evaluator() + success, err = evaluator.evaluate(user_answer, proc, + expected_input, + expected_output + ) + os.remove(self.user_output_path) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/yaksh/models.py b/yaksh/models.py index 4ee6141..e7ae52f 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -36,7 +36,7 @@ enrollment_methods = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - ("stdoutbasedtestcase", "Stdout Based Testcase"), + ("stdiobasedtestcase", "StdIO Based Testcase"), ("mcqtestcase", "MCQ Testcase"), ) @@ -888,15 +888,17 @@ class StandardTestCase(TestCase): ) -class StdoutBasedTestCase(TestCase): +class StdIOBasedTestCase(TestCase): + expected_input = models.TextField(blank=True) expected_output = models.TextField(blank=True) def get_field_value(self): - return {"expected_output": self.expected_output} + return {"expected_output": self.expected_output, + "expected_input": self.expected_input} def __unicode__(self): - return u'Question: {0} | Exp. Output: {1}'.format(self.question, - self.expected_output + return u'Question: {0} | Exp. Output: {1} | Exp. Input: {2}'.format(self.question, + self.expected_output, self.expected_input ) diff --git a/yaksh/settings.py b/yaksh/settings.py index 70e5471..13860b6 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -11,7 +11,7 @@ SERVER_PORTS = [8001] # range(8001, 8026) SERVER_POOL_PORT = 53579 # Timeout for the code to run in seconds. This is an integer! -SERVER_TIMEOUT = 2 +SERVER_TIMEOUT = 4 # The root of the URL, for example you might be in the situation where you # are not hosted as host.org/exam/ but as host.org/foo/exam/ for whatever @@ -23,9 +23,12 @@ code_evaluators = { "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" }, - "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"}, + "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator", + "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"}, "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py new file mode 100644 index 0000000..037ad3d --- /dev/null +++ b/yaksh/stdio_evaluator.py @@ -0,0 +1,23 @@ +class Evaluator(object): + + def evaluate(self, user_answer, proc, expected_input, expected_output): + success = False + if expected_input: + ip = expected_input.replace(",", " ") + proc.stdin.write('{0}\n'.format(ip)) + error_msg = " Given Input is {0} \n Expected Output is {1} ".\ + format(expected_input, expected_output) + else: + error_msg = "Expected output is {0}".format(expected_output) + output_err = proc.stderr.read() + user_output = proc.stdout.read() + expected_output = expected_output.replace("\r", "") + if output_err == '': + if user_output == expected_output: + success, err = True, "Correct Answer" + else: + err = " Incorrect Answer\n" + error_msg +\ + "\n Your output is {0}".format(user_output) + else: + err = "Error:"+"\n"+output_err + return success, err |