From f5ea9ecf903360c2860293639d1c6ba7a78b2339 Mon Sep 17 00:00:00 2001 From: adityacp Date: Thu, 2 Jun 2016 14:24:57 +0530 Subject: c,cpp,java evaluators with test cases --- yaksh/cpp_stdio_evaluator.py | 105 ++++++++++ yaksh/evaluator_tests/test_c_cpp_evaluation.py | 262 +++++++++++++++++++++++++ yaksh/evaluator_tests/test_java_evaluation.py | 125 ++++++++++++ yaksh/java_stdio_evaluator.py | 77 ++++++++ yaksh/models.py | 12 +- yaksh/settings.py | 11 +- yaksh/stdio_evaluator.py | 23 +++ 7 files changed, 606 insertions(+), 9 deletions(-) create mode 100644 yaksh/cpp_stdio_evaluator.py create mode 100644 yaksh/java_stdio_evaluator.py create mode 100644 yaksh/stdio_evaluator.py (limited to 'yaksh') diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py new file mode 100644 index 0000000..28b0a0b --- /dev/null +++ b/yaksh/cpp_stdio_evaluator.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +import subprocess +import os +from os.path import isfile + +#local imports +from code_evaluator import CodeEvaluator +from stdio_evaluator import Evaluator + + +class CppStdioEvaluator(CodeEvaluator): + """Evaluates C StdIO based code""" + + def setup(self): + super(CppStdioEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('main.c') + + def teardown(self): + super(CppStdioEvaluator, self).teardown() + os.remove(self.submit_code_path) + + def set_file_paths(self): + user_output_path = os.getcwd() + '/output' + ref_output_path = os.getcwd() + '/executable' + + return user_output_path, ref_output_path + + def get_commands(self, user_output_path, ref_output_path): + compile_command = 'g++ {0} -c -o {1}'.format(self.submit_code_path, + user_output_path) + compile_main = 'g++ {0} -o {1}'.format(user_output_path, + ref_output_path) + + return compile_command, compile_main + + def compile_code(self, user_answer, expected_input, expected_output): + + if not isfile(self.submit_code_path): + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg + self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_output_path, self.ref_output_path = self.set_file_paths() + self.compile_command, self.compile_main = self.get_commands( + self.user_output_path, + self.ref_output_path + ) + self.compiled_user_answer = self._run_command(self.compile_command, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + self.compiled_test_code = self._run_command(self.compile_main, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + return self.compiled_user_answer, self.compiled_test_code + + def check_code(self, user_answer, expected_input, expected_output): + success = False + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + if stdnt_stderr == '': + proc, main_out, main_err = self.compiled_test_code + main_err = self._remove_null_substitute_char(main_err) + if main_err == '': + proc = subprocess.Popen("./executable", + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + evaluator = Evaluator() + success, err = evaluator.evaluate(user_answer, proc, + expected_input, + expected_output + ) + os.remove(self.ref_output_path) + else: + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(self.user_output_path) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 71fb843..7a87d87 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -1,7 +1,10 @@ import unittest import os from yaksh.cpp_code_evaluator import CppCodeEvaluator +from yaksh.cpp_stdio_evaluator import CppStdioEvaluator from yaksh.settings import SERVER_TIMEOUT +from textwrap import dedent + class CEvaluationTestCases(unittest.TestCase): def setUp(self): @@ -52,5 +55,264 @@ class CEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + +class CStdioEvaluationTestCases(unittest.TestCase): + + def setUp(self): + self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = dedent(""" + #include + int main(void){ + int a,b; + scanf("%d%d",&a,&b); + printf("%d",a+b); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_array_input(self): + self.test_case_data = [{'expected_output': '561', + 'expected_input': '5,6,1'}] + user_answer = dedent(""" + #include + int main(void){ + int a[3],i; + for(i=0;i<3;i++){ + scanf("%d",&a[i]);} + for(i=0;i<3;i++){ + printf("%d",a[i]);} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_string_input(self): + self.test_case_data = [{'expected_output': 'abc', + 'expected_input': 'abc'}] + user_answer = dedent(""" + #include + int main(void){ + char a[4]; + scanf("%s",a); + printf("%s",a); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = dedent(""" + #include + int main(void){ + int a=10; + printf("%d",a); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) + + def test_error(self): + user_answer = dedent(""" + #include + int main(void){ + int a=10; + printf("%d",a) + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + user_answer = dedent(""" + #include + int main(void){ + while(0==0){ + printf("abc");} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + #include + int main(void){ + int a=5,b=6; + printf("%d",a+b); + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + +class CppStdioEvaluationTestCases(unittest.TestCase): + + def setUp(self): + self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = dedent(""" + #include + using namespace std; + int main(void){ + int a,b; + cin>>a>>b; + cout< + using namespace std; + int main(void){ + int a[3],i; + for(i=0;i<3;i++){ + cin>>a[i];} + for(i=0;i<3;i++){ + cout< + using namespace std; + int main(void){ + char a[4]; + cin>>a; + cout< + using namespace std; + int main(void){ + int a=10; + cout< 1) + + def test_error(self): + user_answer = dedent(""" + #include + using namespace std; + int main(void){ + int a=10; + cout< + using namespace std; + int main(void){ + while(0==0){ + cout<<"abc";} + }""") + get_class = CppStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + #include + using namespace std; + int main(void){ + int a=5,b=6; + cout< 1) + + def test_error(self): + + user_answer = dedent(""" + class Test + { + System.out.print("a"); + }""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertTrue("Compilation Error" in result.get("error")) + + def test_infinite_loop(self): + + user_answer = dedent(""" + class Test + {public static void main(String[] args){ + while(0==0) + { + System.out.print("a");} + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get("success")) + self.assertEquals(result.get("error"), self.timeout_msg) + + def test_only_stdout(self): + self.test_case_data = [{'expected_output': '11', + 'expected_input': ''}] + user_answer = dedent(""" + class Test + {public static void main(String[] args){ + int a = 5; + int b = 6; + System.out.print(a+b); + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py new file mode 100644 index 0000000..2f6ef40 --- /dev/null +++ b/yaksh/java_stdio_evaluator.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +import subprocess +import os +from os.path import isfile + +#local imports +from code_evaluator import CodeEvaluator +from stdio_evaluator import Evaluator + + +class JavaStdioEvaluator(CodeEvaluator): + """Evaluates Java StdIO based code""" + + def setup(self): + super(JavaStdioEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('Test.java') + + def teardown(self): + super(JavaStdioEvaluator, self).teardown() + os.remove(self.submit_code_path) + + def set_file_paths(self, directory, file_name): + output_path = "{0}{1}.class".format(directory, file_name) + return output_path + + def get_commands(self): + compile_command = 'javac {0}'.format(self.submit_code_path) + + return compile_command + + def compile_code(self, user_answer, expected_input, expected_output): + if not isfile(self.submit_code_path): + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg + user_code_directory = os.getcwd() + '/' + self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_output_path = self.set_file_paths(user_code_directory, + 'Test' + ) + self.compile_command = self.get_commands() + self.compiled_user_answer = self._run_command(self.compile_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + return self.compiled_user_answer + + def check_code(self, user_answer, expected_input, expected_output): + success = False + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + if stdnt_stderr == '' or "error" not in stdnt_stderr: + proc = subprocess.Popen("java Test", + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + evaluator = Evaluator() + success, err = evaluator.evaluate(user_answer, proc, + expected_input, + expected_output + ) + os.remove(self.user_output_path) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr + + return success, err diff --git a/yaksh/models.py b/yaksh/models.py index 4ee6141..e7ae52f 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -36,7 +36,7 @@ enrollment_methods = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - ("stdoutbasedtestcase", "Stdout Based Testcase"), + ("stdiobasedtestcase", "StdIO Based Testcase"), ("mcqtestcase", "MCQ Testcase"), ) @@ -888,15 +888,17 @@ class StandardTestCase(TestCase): ) -class StdoutBasedTestCase(TestCase): +class StdIOBasedTestCase(TestCase): + expected_input = models.TextField(blank=True) expected_output = models.TextField(blank=True) def get_field_value(self): - return {"expected_output": self.expected_output} + return {"expected_output": self.expected_output, + "expected_input": self.expected_input} def __unicode__(self): - return u'Question: {0} | Exp. Output: {1}'.format(self.question, - self.expected_output + return u'Question: {0} | Exp. Output: {1} | Exp. Input: {2}'.format(self.question, + self.expected_output, self.expected_input ) diff --git a/yaksh/settings.py b/yaksh/settings.py index 70e5471..13860b6 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -11,7 +11,7 @@ SERVER_PORTS = [8001] # range(8001, 8026) SERVER_POOL_PORT = 53579 # Timeout for the code to run in seconds. This is an integer! -SERVER_TIMEOUT = 2 +SERVER_TIMEOUT = 4 # The root of the URL, for example you might be in the situation where you # are not hosted as host.org/exam/ but as host.org/foo/exam/ for whatever @@ -23,9 +23,12 @@ code_evaluators = { "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" }, - "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"}, + "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator", + "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"}, "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py new file mode 100644 index 0000000..037ad3d --- /dev/null +++ b/yaksh/stdio_evaluator.py @@ -0,0 +1,23 @@ +class Evaluator(object): + + def evaluate(self, user_answer, proc, expected_input, expected_output): + success = False + if expected_input: + ip = expected_input.replace(",", " ") + proc.stdin.write('{0}\n'.format(ip)) + error_msg = " Given Input is {0} \n Expected Output is {1} ".\ + format(expected_input, expected_output) + else: + error_msg = "Expected output is {0}".format(expected_output) + output_err = proc.stderr.read() + user_output = proc.stdout.read() + expected_output = expected_output.replace("\r", "") + if output_err == '': + if user_output == expected_output: + success, err = True, "Correct Answer" + else: + err = " Incorrect Answer\n" + error_msg +\ + "\n Your output is {0}".format(user_output) + else: + err = "Error:"+"\n"+output_err + return success, err -- cgit From bb76e0a25b113b0f40b47f19eaa2c053e0b4e423 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:00:27 +0530 Subject: changed model class name Stdout to Stdio --- yaksh/admin.py | 4 ++-- yaksh/evaluator_tests/test_java_evaluation.py | 4 ++-- yaksh/forms.py | 6 +++--- yaksh/models.py | 4 ++-- yaksh/settings.py | 12 +++++++----- yaksh/tests.py | 6 +++--- 6 files changed, 19 insertions(+), 17 deletions(-) (limited to 'yaksh') diff --git a/yaksh/admin.py b/yaksh/admin.py index c31b99b..ddd5e62 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -1,9 +1,9 @@ from yaksh.models import Question, Quiz -from yaksh.models import TestCase, StandardTestCase, StdoutBasedTestCase +from yaksh.models import TestCase, StandardTestCase, StdioBasedTestCase from django.contrib import admin admin.site.register(Question) admin.site.register(TestCase) admin.site.register(StandardTestCase) -admin.site.register(StdoutBasedTestCase) +admin.site.register(StdioBasedTestCase) admin.site.register(Quiz) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 8cfccb9..fa8d68c 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -68,13 +68,13 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] - evaluator.SERVER_TIMEOUT = 9 + evaluator.SERVER_TIMEOUT = 4 self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(evaluator.SERVER_TIMEOUT) def teardown(self): - evaluator.SERVER_TIMEOUT = 2 + evaluator.SERVER_TIMEOUT = 4 def test_correct_answer(self): user_answer = dedent(""" diff --git a/yaksh/forms.py b/yaksh/forms.py index a443e34..6bea0c8 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,9 +1,9 @@ from django import forms -from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdoutBasedTestCase +from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdioBasedTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User -from django.contrib.contenttypes.models import ContentType +from django.contrib.contenttypes.models import ContentType from taggit.managers import TaggableManager from taggit.forms import TagField @@ -33,7 +33,7 @@ question_types = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - ("stdoutbasedtestcase", "Stdout Based Testcase"), + ("stdiobasedtestcase", "Stdio Based Testcase"), ("mcqtestcase", "MCQ Testcase"), ) diff --git a/yaksh/models.py b/yaksh/models.py index e7ae52f..3fd7508 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -888,9 +888,9 @@ class StandardTestCase(TestCase): ) -class StdIOBasedTestCase(TestCase): +class StdioBasedTestCase(TestCase): expected_input = models.TextField(blank=True) - expected_output = models.TextField(blank=True) + expected_output = models.TextField() def get_field_value(self): return {"expected_output": self.expected_output, diff --git a/yaksh/settings.py b/yaksh/settings.py index 13860b6..f4122b6 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -21,14 +21,16 @@ URL_ROOT = '' code_evaluators = { "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", - "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" - }, + "stdiobasedtestcase": "python_stdio_evaluator.PythonStdioEvaluator" + }, "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator" + }, "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator", - "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"}, + "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator" + }, "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator", - "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"}, + "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"}, "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/tests.py b/yaksh/tests.py index 8bd2dda..5b87fc1 100644 --- a/yaksh/tests.py +++ b/yaksh/tests.py @@ -1,7 +1,7 @@ import unittest from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdoutBasedTestCase + StdioBasedTestCase import json from datetime import datetime, timedelta from django.utils import timezone @@ -632,7 +632,7 @@ class TestCaseTestCases(unittest.TestCase): active=True, description='Write to standard output', points=1.0, - test_case_type="stdoutbasedtestcase", + test_case_type="stdiobasedtestcase", user=self.user, snippet='def myfunc()' ) @@ -642,7 +642,7 @@ class TestCaseTestCases(unittest.TestCase): question=self.question1, test_case='assert myfunc(12, 13) == 15' ) - self.stdout_based_testcase = StdoutBasedTestCase( + self.stdout_based_testcase = StdioBasedTestCase( question=self.question2, expected_output='Hello World' ) -- cgit From bdebcc790e6f0ac0f53795c017568ad44c8f01dd Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:01:03 +0530 Subject: changed model class name Stdout to Stdio --- yaksh/evaluator_tests/test_code_evaluation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index 51c0c51..cbca32d 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -17,7 +17,7 @@ class RegistryTestCase(unittest.TestCase): ) code_evaluators['python'] = \ {"standardtestcase": assertion_evaluator_path, - "stdoutbasedtestcase": stdout_evaluator_path + "stdiobasedtestcase": stdout_evaluator_path } def test_set_register(self): @@ -35,7 +35,7 @@ class RegistryTestCase(unittest.TestCase): ) self.registry_object.register("python", {"standardtestcase": assertion_evaluator_path, - "stdoutbasedtestcase": stdout_evaluator_path + "stdiobasedtestcase": stdout_evaluator_path } ) self.assertEquals(evaluator_class, class_name) -- cgit From de63766dad6ed0755af0171cdfdae916e0e8676b Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:01:59 +0530 Subject: added test cases for python stdio evaluator --- yaksh/evaluator_tests/test_python_evaluation.py | 159 +++++++++++++++++++----- 1 file changed, 127 insertions(+), 32 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 1e867a3..dfdc6ec 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -1,7 +1,7 @@ import unittest import os from yaksh.python_assertion_evaluator import PythonAssertionEvaluator -from yaksh.python_stdout_evaluator import PythonStdoutEvaluator +from yaksh.python_stdio_evaluator import PythonStdioEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent @@ -164,15 +164,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): return int(a) + int(b) + int(c) """) value_error_msg = ["Traceback", - "call", - "ValueError", - "invalid literal", - "base" - ] + "call", + "ValueError", + "invalid literal", + "base" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -180,55 +180,150 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): for msg in value_error_msg: self.assertIn(msg, result.get("error")) -class PythonStdoutEvaluationTestCases(unittest.TestCase): + +class PythonStdioEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = [{"expected_output": "0 1 1 2 3"}] + self.test_case_data = [{"expected_input": None, + "expected_output": "0 1 1 2 3" + }] + self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop" - " in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" - get_class = PythonStdoutEvaluator() + get_class = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEqual(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) def test_incorrect_answer(self): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" - get_class = PythonStdoutEvaluator() - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + get_class = PythonStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), "Incorrect Answer") + self.assertIn("Incorrect Answer", result.get('error')) def test_direct_printed_answer(self): user_answer = "print '0 1 1 2 3'" error_msg = ("Incorrect Answer: Please avoid printing" - " the expected output directly" - ) - get_class = PythonStdoutEvaluator() - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + " the expected output directly" + ) + get_class = PythonStdioEvaluator() + + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), error_msg) def test_infinite_loop(self): - user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" - get_class = PythonStdoutEvaluator() + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)" + get_class = PythonStdioEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) + + +class PythonStdIOEvaluator(unittest.TestCase): + + def setUp(self): + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) + + def test_add_two_integers_correct(self): + self.test_case_data = [{"expected_input": "1\n2", + "expected_output": "3" + }] + user_answer = dedent(""" + a = int(raw_input()) + b = int(raw_input()) + print a+b + """ + ) + + get_class = PythonStdioEvaluator() + + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertIn("Correct Answer", result.get('error')) + + def test_add_two_lists(self): + self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]", + "expected_output": "[1, 2, 3, 5, 6, 7]" + }] + user_answer = dedent(""" + from ast import literal_eval + a = literal_eval(raw_input()) + b = literal_eval(raw_input()) + print a+b + """ + ) + + get_class = PythonStdioEvaluator() + + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertIn("Correct Answer", result.get('error')) + + def test_find_string_in_sentence(self): + self.test_case_data = [{"expected_input": """the quick brown fox jumps\ + over the lazy dog\nthe""", + "expected_output": "2" + }] + user_answer = dedent(""" + a = raw_input() + b = raw_input() + print (a.count(b)) + """ + ) + + get_class = PythonStdioEvaluator() + + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertIn("Correct Answer", result.get('error')) + + def test_add_two_integers_incorrect(self): + self.test_case_data = [{"expected_input": "1\n2", + "expected_output": "4" + }] + user_answer = dedent(""" + a = int(raw_input()) + b = int(raw_input()) + print a+b + """ + ) + + get_class = PythonStdioEvaluator() + + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), 'Incorrect Answer') + self.assertIn("Incorrect Answer", result.get('error')) if __name__ == '__main__': -- cgit From cf52baf30e5902c2f3c79cfbd8df88789ccd09ce Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:03:21 +0530 Subject: added python stdio evaluator --- yaksh/python_stdio_evaluator.py | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 yaksh/python_stdio_evaluator.py (limited to 'yaksh') diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py new file mode 100644 index 0000000..5db80f5 --- /dev/null +++ b/yaksh/python_stdio_evaluator.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib +from contextlib import contextmanager +from ast import literal_eval +# local imports +from code_evaluator import CodeEvaluator +from StringIO import StringIO + + +@contextmanager +def redirect_stdout(): + new_target = StringIO() + + old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout + try: + yield new_target # run some code with the replaced stdout + finally: + sys.stdout = old_target # restore to the previous value + + +class PythonStdioEvaluator(CodeEvaluator): + """Tests the Python code obtained from Code Server""" + + def compile_code(self, user_answer, expected_input, expected_output): + submitted = compile(user_answer, '', mode='exec') + if expected_input: + input_buffer = StringIO() + try: + literal_input = literal_eval(expected_input.split("\n")) + except ValueError: + literal_input = expected_input.split("\n") + for inputs in literal_input: + input_buffer.write(str(inputs)+'\n') + input_buffer.seek(0) + sys.stdin = input_buffer + + with redirect_stdout() as output_buffer: + exec_scope = {} + exec submitted in exec_scope + self.output_value = output_buffer.getvalue().rstrip("\n") + return self.output_value + + def check_code(self, user_answer, expected_input, expected_output): + success = False + + tb = None + if expected_output in user_answer: + success = False + err = ("Incorrect Answer: Please avoid " + "printing the expected output directly" + ) + elif self.output_value == expected_output: + + success = True + err = "Correct Answer" + + else: + success = False + err = """Incorrect Answer:\nExpected output - {0} + and your output - {1}"""\ + .format(expected_output, self.output_value) + + del tb + return success, err -- cgit From e46bd90fb2907a7a43786bee95985072e256cd56 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:30:44 +0530 Subject: removed literal_eval from python_stdio_evaluator --- yaksh/python_stdio_evaluator.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'yaksh') diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 5db80f5..003337d 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -29,12 +29,7 @@ class PythonStdioEvaluator(CodeEvaluator): submitted = compile(user_answer, '', mode='exec') if expected_input: input_buffer = StringIO() - try: - literal_input = literal_eval(expected_input.split("\n")) - except ValueError: - literal_input = expected_input.split("\n") - for inputs in literal_input: - input_buffer.write(str(inputs)+'\n') + input_buffer.write(expected_input) input_buffer.seek(0) sys.stdin = input_buffer -- cgit From d14bf131d60af6eae6c3ff6ce5537af25420c314 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 2 Jun 2016 16:31:54 +0530 Subject: renamed python_stdout_evaluation.py to python_stdio_evaluation.py --- yaksh/python_stdout_evaluator.py | 58 ---------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 yaksh/python_stdout_evaluator.py (limited to 'yaksh') diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py deleted file mode 100644 index 6606581..0000000 --- a/yaksh/python_stdout_evaluator.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -import sys -import traceback -import os -from os.path import join -import importlib -from contextlib import contextmanager - -# local imports -from code_evaluator import CodeEvaluator - - -@contextmanager -def redirect_stdout(): - from StringIO import StringIO - new_target = StringIO() - - old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout - try: - yield new_target # run some code with the replaced stdout - finally: - sys.stdout = old_target # restore to the previous value - - -class PythonStdoutEvaluator(CodeEvaluator): - """Tests the Python code obtained from Code Server""" - - def compile_code(self, user_answer, expected_output): - if hasattr(self, 'output_value'): - return None - else: - submitted = compile(user_answer, '', mode='exec') - with redirect_stdout() as output_buffer: - exec_scope = {} - exec submitted in exec_scope - self.output_value = output_buffer.getvalue() - return self.output_value - - def check_code(self, user_answer, expected_output): - success = False - - tb = None - if expected_output in user_answer: - success = False - err = ("Incorrect Answer: Please avoid " - "printing the expected output directly" - ) - elif self.output_value == expected_output: - success = True - err = "Correct answer" - - else: - success = False - err = "Incorrect Answer" - - del tb - return success, err - -- cgit From 53192a5f757064b9f8873ea3fcfc7f9cd01e587c Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Wed, 8 Jun 2016 11:18:52 +0530 Subject: made changes to stdio evaluator and added bash evaluator with test cases --- yaksh/evaluator_tests/test_bash_evaluation.py | 81 ++++++++++++++++++++++++- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 2 +- yaksh/evaluator_tests/test_java_evaluation.py | 2 +- yaksh/evaluator_tests/test_python_evaluation.py | 2 +- yaksh/settings.py | 6 +- yaksh/stdio_evaluator.py | 21 +++---- 6 files changed, 98 insertions(+), 16 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 4ff3e0a..f954ef0 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -1,9 +1,12 @@ import unittest import os from yaksh.bash_code_evaluator import BashCodeEvaluator +from yaksh.bash_stdio_evaluator import BashStdioEvaluator from yaksh.settings import SERVER_TIMEOUT +from textwrap import dedent -class BashEvaluationTestCases(unittest.TestCase): + +class BashAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [ {"test_case": "bash_files/sample.sh,bash_files/sample.args"} @@ -48,5 +51,81 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get("error"), self.timeout_msg) +class BashStdioEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = dedent(""" #!/bin/bash + read A + read B + echo -n `expr $A + $B` + """ + ) + test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] + get_class = BashStdioEvaluator() + kwargs = {"user_answer": user_answer, + "test_case_data": test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_array_input(self): + user_answer = dedent(""" readarray arr; + COUNTER=0 + while [ $COUNTER -lt 3 ]; do + echo -n "${arr[$COUNTER]}" + let COUNTER=COUNTER+1 + done + """ + ) + test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', + 'expected_input': '1,2,3\n4,5,6\n7,8,9' + }] + get_class = BashStdioEvaluator() + kwargs = {"user_answer": user_answer, + "test_case_data": test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = dedent(""" #!/bin/bash + read A + read B + echo -n `expr $A + $B` + """ + ) + test_case_data = [{'expected_output': '12', 'expected_input': '5\n6'}] + get_class = BashStdioEvaluator() + kwargs = {"user_answer": user_answer, + "test_case_data": test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertIn("Incorrect", result.get('error')) + self.assertFalse(result.get('success')) + + def test_stdout_only(self): + user_answer = dedent(""" #!/bin/bash + A=6 + B=4 + echo -n `expr $A + $B` + """ + ) + test_case_data = [{'expected_output': '10', + 'expected_input': '' + }] + get_class = BashStdioEvaluator() + kwargs = {"user_answer": user_answer, + "test_case_data": test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 7a87d87..3d2e9fe 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -6,7 +6,7 @@ from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent -class CEvaluationTestCases(unittest.TestCase): +class CAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] self.in_dir = "/tmp" diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index fa8d68c..211b324 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -7,7 +7,7 @@ from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent -class JavaEvaluationTestCases(unittest.TestCase): +class JavaAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [ {"test_case": "java_files/main_square.java"} diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index dfdc6ec..a935f1d 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -181,7 +181,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): self.assertIn(msg, result.get("error")) -class PythonStdioEvaluationTestCases(unittest.TestCase): +class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"expected_input": None, "expected_output": "0 1 1 2 3" diff --git a/yaksh/settings.py b/yaksh/settings.py index f4122b6..b1336f4 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -31,6 +31,10 @@ code_evaluators = { }, "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator", "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"}, - "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, + + "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator", + "stdiobasedtestcase": "bash_stdio_evaluator.BashStdioEvaluator" + }, + "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py index 037ad3d..236c0ab 100644 --- a/yaksh/stdio_evaluator.py +++ b/yaksh/stdio_evaluator.py @@ -1,23 +1,22 @@ class Evaluator(object): - def evaluate(self, user_answer, proc, expected_input, expected_output): + success = False - if expected_input: - ip = expected_input.replace(",", " ") - proc.stdin.write('{0}\n'.format(ip)) - error_msg = " Given Input is {0} \n Expected Output is {1} ".\ - format(expected_input, expected_output) - else: - error_msg = "Expected output is {0}".format(expected_output) - output_err = proc.stderr.read() - user_output = proc.stdout.read() + ip = expected_input.replace(",", " ") + user_output, output_err = proc.communicate(input='{0}\n'.format(ip)) expected_output = expected_output.replace("\r", "") + if not expected_input: + error_msg = "Expected Output is {0} ".\ + format(repr(expected_output)) + else: + error_msg = " Given Input is {0} \n Expected Output is {1} ".\ + format(expected_input, repr(expected_output)) if output_err == '': if user_output == expected_output: success, err = True, "Correct Answer" else: err = " Incorrect Answer\n" + error_msg +\ - "\n Your output is {0}".format(user_output) + "\n Your output is {0}".format(repr(user_output)) else: err = "Error:"+"\n"+output_err return success, err -- cgit From 379acc5439c3aaa7f659e6f718db8baf8cfc5fdc Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Wed, 8 Jun 2016 11:24:47 +0530 Subject: added bash stdio evaluator --- yaksh/bash_stdio_evaluator.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 yaksh/bash_stdio_evaluator.py (limited to 'yaksh') diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py new file mode 100644 index 0000000..25f35a1 --- /dev/null +++ b/yaksh/bash_stdio_evaluator.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +import subprocess +import os +from os.path import isfile + +#local imports +from code_evaluator import CodeEvaluator +from stdio_evaluator import Evaluator + + +class BashStdioEvaluator(CodeEvaluator): + """Evaluates Bash StdIO based code""" + + def setup(self): + super(BashStdioEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('Test.sh') + + def teardown(self): + super(BashStdioEvaluator, self).teardown() + os.remove(self.submit_code_path) + + def compile_code(self, user_answer, expected_input, expected_output): + if not isfile(self.submit_code_path): + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg + user_code_directory = os.getcwd() + '/' + user_answer = user_answer.replace("\r", "") + self.write_to_submit_code_file(self.submit_code_path, user_answer) + + def check_code(self, user_answer, expected_input, expected_output): + success = False + expected_input = str(expected_input).replace('\r', '') + proc = subprocess.Popen(["bash ./Test.sh"], + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + evaluator = Evaluator() + success, err = evaluator.evaluate(user_answer, proc, + expected_input, + expected_output + ) + return success, err -- cgit From bca8dd59b6d796af680cac3d7f65fc3ac6ed8420 Mon Sep 17 00:00:00 2001 From: adityacp Date: Wed, 27 Jul 2016 22:49:05 +0530 Subject: added testcase for string input --- yaksh/evaluator_tests/test_java_evaluation.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 211b324..c98a938 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -97,7 +97,7 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5,6,1'}] + 'expected_input': '5\n6\n1'}] user_answer = dedent(""" import java.util.Scanner; class Test @@ -186,5 +186,25 @@ class JavaStdioEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) + def test_string_input(self): + self.test_case_data = [{'expected_output': 'HelloWorld', + 'expected_input': 'Hello\nWorld'}] + user_answer = dedent(""" + import java.util.Scanner; + class Test + {public static void main(String[] args){ + Scanner s = new Scanner(System.in); + String a = s.nextLine(); + String b = s.nextLine(); + System.out.print(a+b); + }}""") + get_class = JavaStdioEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct Answer") + self.assertTrue(result.get('success')) + if __name__ == '__main__': unittest.main() -- cgit From bdbacc0a24970cbe27acd1aa1dbee0bd82157bdb Mon Sep 17 00:00:00 2001 From: adityacp Date: Wed, 27 Jul 2016 22:50:14 +0530 Subject: combined c and cpp testcase classes into single class --- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 27 +++++++++----------------- 1 file changed, 9 insertions(+), 18 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 3d2e9fe..a2fc51f 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -56,7 +56,7 @@ class CAssertionEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get("error"), self.timeout_msg) -class CStdioEvaluationTestCases(unittest.TestCase): +class CppStdioEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] @@ -181,16 +181,7 @@ class CStdioEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) - -class CppStdioEvaluationTestCases(unittest.TestCase): - - def setUp(self): - self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) - - def test_correct_answer(self): + def test_cpp_correct_answer(self): user_answer = dedent(""" #include using namespace std; @@ -207,9 +198,9 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) - def test_array_input(self): + def test_cpp_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5,6,1'}] + 'expected_input': '5\n6\n1'}] user_answer = dedent(""" #include using namespace std; @@ -228,7 +219,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) - def test_string_input(self): + def test_cpp_string_input(self): self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc'}] user_answer = dedent(""" @@ -247,7 +238,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct Answer") self.assertTrue(result.get('success')) - def test_incorrect_answer(self): + def test_cpp_incorrect_answer(self): user_answer = dedent(""" #include using namespace std; @@ -264,7 +255,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertIn("Incorrect", result.get('error')) self.assertTrue(result.get('error').splitlines > 1) - def test_error(self): + def test_cpp_error(self): user_answer = dedent(""" #include using namespace std; @@ -280,7 +271,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) - def test_infinite_loop(self): + def test_cpp_infinite_loop(self): user_answer = dedent(""" #include using namespace std; @@ -296,7 +287,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) - def test_only_stdout(self): + def test_cpp_only_stdout(self): self.test_case_data = [{'expected_output': '11', 'expected_input': ''}] user_answer = dedent(""" -- cgit From e340b564d886501d36fa738e135ccb8e07645f4d Mon Sep 17 00:00:00 2001 From: adityacp Date: Wed, 27 Jul 2016 22:51:30 +0530 Subject: removed extra new lines --- yaksh/cpp_stdio_evaluator.py | 3 --- yaksh/java_stdio_evaluator.py | 2 -- yaksh/stdio_evaluator.py | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) (limited to 'yaksh') diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py index 28b0a0b..db49adf 100644 --- a/yaksh/cpp_stdio_evaluator.py +++ b/yaksh/cpp_stdio_evaluator.py @@ -22,7 +22,6 @@ class CppStdioEvaluator(CodeEvaluator): def set_file_paths(self): user_output_path = os.getcwd() + '/output' ref_output_path = os.getcwd() + '/executable' - return user_output_path, ref_output_path def get_commands(self, user_output_path, ref_output_path): @@ -30,7 +29,6 @@ class CppStdioEvaluator(CodeEvaluator): user_output_path) compile_main = 'g++ {0} -o {1}'.format(user_output_path, ref_output_path) - return compile_command, compile_main def compile_code(self, user_answer, expected_input, expected_output): @@ -101,5 +99,4 @@ class CppStdioEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - return success, err diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py index 2f6ef40..2ab2981 100644 --- a/yaksh/java_stdio_evaluator.py +++ b/yaksh/java_stdio_evaluator.py @@ -25,7 +25,6 @@ class JavaStdioEvaluator(CodeEvaluator): def get_commands(self): compile_command = 'javac {0}'.format(self.submit_code_path) - return compile_command def compile_code(self, user_answer, expected_input, expected_output): @@ -73,5 +72,4 @@ class JavaStdioEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + stdnt_stderr - return success, err diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py index 236c0ab..86b124c 100644 --- a/yaksh/stdio_evaluator.py +++ b/yaksh/stdio_evaluator.py @@ -1,6 +1,6 @@ class Evaluator(object): + def evaluate(self, user_answer, proc, expected_input, expected_output): - success = False ip = expected_input.replace(",", " ") user_output, output_err = proc.communicate(input='{0}\n'.format(ip)) -- cgit From 9411ab221e007a6c3e2901ce34f38f2267288b6e Mon Sep 17 00:00:00 2001 From: adityacp Date: Thu, 28 Jul 2016 00:12:51 +0530 Subject: rebase changes and changed expected input format in cpp tests --- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 2 +- yaksh/test_views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index a2fc51f..ff3cddf 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -82,7 +82,7 @@ class CppStdioEvaluationTestCases(unittest.TestCase): def test_array_input(self): self.test_case_data = [{'expected_output': '561', - 'expected_input': '5,6,1'}] + 'expected_input': '5\n6\n1'}] user_answer = dedent(""" #include int main(void){ diff --git a/yaksh/test_views.py b/yaksh/test_views.py index 6e59e26..2544276 100644 --- a/yaksh/test_views.py +++ b/yaksh/test_views.py @@ -8,7 +8,7 @@ from django.test import Client from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ - StdoutBasedTestCase, has_profile + StdioBasedTestCase, has_profile class TestProfile(TestCase): -- cgit From 1b71abc9437d721a41f017db406f312755f5a4c4 Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 28 Jul 2016 18:24:18 +0530 Subject: added expected input in traceback for python and made minor changes in bash_stio_evaluator and stdio_evaluator --- yaksh/bash_stdio_evaluator.py | 2 +- yaksh/evaluator_tests/test_bash_evaluation.py | 4 ++-- yaksh/evaluator_tests/test_python_evaluation.py | 14 ------------- yaksh/python_stdio_evaluator.py | 26 +++++++++++-------------- yaksh/stdio_evaluator.py | 2 +- 5 files changed, 15 insertions(+), 33 deletions(-) (limited to 'yaksh') diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py index 25f35a1..5431e5d 100644 --- a/yaksh/bash_stdio_evaluator.py +++ b/yaksh/bash_stdio_evaluator.py @@ -30,7 +30,7 @@ class BashStdioEvaluator(CodeEvaluator): def check_code(self, user_answer, expected_input, expected_output): success = False expected_input = str(expected_input).replace('\r', '') - proc = subprocess.Popen(["bash ./Test.sh"], + proc = subprocess.Popen("bash ./Test.sh", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index f954ef0..ee6c1f0 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -97,10 +97,10 @@ class BashStdioEvaluationTestCases(unittest.TestCase): user_answer = dedent(""" #!/bin/bash read A read B - echo -n `expr $A + $B` + echo -n `expr $A - $B` """ ) - test_case_data = [{'expected_output': '12', 'expected_input': '5\n6'}] + test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}] get_class = BashStdioEvaluator() kwargs = {"user_answer": user_answer, "test_case_data": test_case_data diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index a935f1d..40562db 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -211,20 +211,6 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get('success')) self.assertIn("Incorrect Answer", result.get('error')) - def test_direct_printed_answer(self): - user_answer = "print '0 1 1 2 3'" - error_msg = ("Incorrect Answer: Please avoid printing" - " the expected output directly" - ) - get_class = PythonStdioEvaluator() - - kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } - result = get_class.evaluate(**kwargs) - self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), error_msg) - def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)" get_class = PythonStdioEvaluator() diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py index 003337d..aeec744 100644 --- a/yaksh/python_stdio_evaluator.py +++ b/yaksh/python_stdio_evaluator.py @@ -10,11 +10,10 @@ from ast import literal_eval from code_evaluator import CodeEvaluator from StringIO import StringIO - +from textwrap import dedent @contextmanager def redirect_stdout(): new_target = StringIO() - old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout try: yield new_target # run some code with the replaced stdout @@ -32,7 +31,6 @@ class PythonStdioEvaluator(CodeEvaluator): input_buffer.write(expected_input) input_buffer.seek(0) sys.stdin = input_buffer - with redirect_stdout() as output_buffer: exec_scope = {} exec submitted in exec_scope @@ -43,21 +41,19 @@ class PythonStdioEvaluator(CodeEvaluator): success = False tb = None - if expected_output in user_answer: - success = False - err = ("Incorrect Answer: Please avoid " - "printing the expected output directly" - ) - elif self.output_value == expected_output: - + if self.output_value == expected_output: success = True err = "Correct Answer" - else: success = False - err = """Incorrect Answer:\nExpected output - {0} - and your output - {1}"""\ - .format(expected_output, self.output_value) - + err = dedent(""" + Incorrect Answer: + Given input - {0}, + Expected output - {1} and your output - {2} + """ + .format(expected_input, + expected_output, self.output_value + ) + ) del tb return success, err diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py index 86b124c..4f5cfaf 100644 --- a/yaksh/stdio_evaluator.py +++ b/yaksh/stdio_evaluator.py @@ -9,7 +9,7 @@ class Evaluator(object): error_msg = "Expected Output is {0} ".\ format(repr(expected_output)) else: - error_msg = " Given Input is {0} \n Expected Output is {1} ".\ + error_msg = " Given Input is\n {0} \n Expected Output is {1} ".\ format(expected_input, repr(expected_output)) if output_err == '': if user_output == expected_output: -- cgit From a4b636508d572cbe00ae8d0336c7b30494bad4a6 Mon Sep 17 00:00:00 2001 From: adityacp Date: Wed, 31 Aug 2016 18:10:57 +0530 Subject: change in incorrect answer stdio testcase --- yaksh/evaluator_tests/test_python_evaluation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yaksh') diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index eea9403..e46b21f 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -318,12 +318,12 @@ class PythonStdIOEvaluator(unittest.TestCase): def test_add_two_integers_incorrect(self): self.test_case_data = [{"expected_input": "1\n2", - "expected_output": "4" + "expected_output": "3" }] user_answer = dedent(""" a = int(raw_input()) b = int(raw_input()) - print a+b + print a-b """ ) -- cgit