summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--yaksh/admin.py8
-rw-r--r--yaksh/bash_stdio_evaluator.py49
-rw-r--r--yaksh/cpp_stdio_evaluator.py108
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py81
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py254
-rw-r--r--yaksh/evaluator_tests/test_code_evaluation.py4
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py147
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py179
-rw-r--r--yaksh/forms.py6
-rw-r--r--yaksh/java_stdio_evaluator.py81
-rw-r--r--yaksh/models.py14
-rw-r--r--yaksh/python_stdio_evaluator.py70
-rw-r--r--yaksh/settings.py23
-rw-r--r--yaksh/stdio_evaluator.py22
-rw-r--r--yaksh/test_models.py6
-rw-r--r--yaksh/test_views.py2
16 files changed, 973 insertions, 81 deletions
diff --git a/yaksh/admin.py b/yaksh/admin.py
index 4ef2f3d..b5c8933 100644
--- a/yaksh/admin.py
+++ b/yaksh/admin.py
@@ -1,13 +1,11 @@
-from yaksh.models import Question, Quiz
-from yaksh.models import TestCase, StandardTestCase, StdoutBasedTestCase, Course
-from yaksh.models import Question, Quiz, Course, QuestionPaper
-from yaksh.models import TestCase, StandardTestCase, StdoutBasedTestCase
+from yaksh.models import Question, Quiz, QuestionPaper
+from yaksh.models import TestCase, StandardTestCase, StdioBasedTestCase, Course
from django.contrib import admin
admin.site.register(Question)
admin.site.register(TestCase)
admin.site.register(StandardTestCase)
-admin.site.register(StdoutBasedTestCase)
+admin.site.register(StdioBasedTestCase)
admin.site.register(Course)
admin.site.register(Quiz)
admin.site.register(QuestionPaper)
diff --git a/yaksh/bash_stdio_evaluator.py b/yaksh/bash_stdio_evaluator.py
new file mode 100644
index 0000000..56f2e35
--- /dev/null
+++ b/yaksh/bash_stdio_evaluator.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+import subprocess
+import os
+from os.path import isfile
+
+#local imports
+from code_evaluator import CodeEvaluator
+from stdio_evaluator import Evaluator
+from file_utils import copy_files, delete_files
+
+class BashStdioEvaluator(CodeEvaluator):
+ """Evaluates Bash StdIO based code"""
+
+ def setup(self):
+ super(BashStdioEvaluator, self).setup()
+ self.submit_code_path = self.create_submit_code_file('Test.sh')
+
+ def teardown(self):
+ super(BashStdioEvaluator, self).teardown()
+ os.remove(self.submit_code_path)
+ if self.files:
+ delete_files(self.files)
+
+ def compile_code(self, user_answer, file_paths, expected_input, expected_output):
+ self.files = []
+ if file_paths:
+ self.files = copy_files(file_paths)
+ if not isfile(self.submit_code_path):
+ msg = "No file at %s or Incorrect path" % self.submit_code_path
+ return False, msg
+ user_code_directory = os.getcwd() + '/'
+ user_answer = user_answer.replace("\r", "")
+ self.write_to_submit_code_file(self.submit_code_path, user_answer)
+
+ def check_code(self, user_answer, file_paths, expected_input, expected_output):
+ success = False
+ expected_input = str(expected_input).replace('\r', '')
+ proc = subprocess.Popen("bash ./Test.sh",
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ evaluator = Evaluator()
+ success, err = evaluator.evaluate(user_answer, proc,
+ expected_input,
+ expected_output
+ )
+ return success, err
diff --git a/yaksh/cpp_stdio_evaluator.py b/yaksh/cpp_stdio_evaluator.py
new file mode 100644
index 0000000..4ea1bbf
--- /dev/null
+++ b/yaksh/cpp_stdio_evaluator.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+import subprocess
+import os
+from os.path import isfile
+
+#local imports
+from code_evaluator import CodeEvaluator
+from stdio_evaluator import Evaluator
+from file_utils import copy_files, delete_files
+
+
+class CppStdioEvaluator(CodeEvaluator):
+ """Evaluates C StdIO based code"""
+
+ def setup(self):
+ super(CppStdioEvaluator, self).setup()
+ self.submit_code_path = self.create_submit_code_file('main.c')
+
+ def teardown(self):
+ super(CppStdioEvaluator, self).teardown()
+ os.remove(self.submit_code_path)
+ if self.files:
+ delete_files(self.files)
+
+ def set_file_paths(self):
+ user_output_path = os.getcwd() + '/output'
+ ref_output_path = os.getcwd() + '/executable'
+ return user_output_path, ref_output_path
+
+ def get_commands(self, user_output_path, ref_output_path):
+ compile_command = 'g++ {0} -c -o {1}'.format(self.submit_code_path,
+ user_output_path)
+ compile_main = 'g++ {0} -o {1}'.format(user_output_path,
+ ref_output_path)
+ return compile_command, compile_main
+
+ def compile_code(self, user_answer, file_paths, expected_input, expected_output):
+
+ self.files = []
+ if file_paths:
+ self.files = copy_files(file_paths)
+ if not isfile(self.submit_code_path):
+ msg = "No file at %s or Incorrect path" % self.submit_code_path
+ return False, msg
+ self.write_to_submit_code_file(self.submit_code_path, user_answer)
+ self.user_output_path, self.ref_output_path = self.set_file_paths()
+ self.compile_command, self.compile_main = self.get_commands(
+ self.user_output_path,
+ self.ref_output_path
+ )
+ self.compiled_user_answer = self._run_command(self.compile_command,
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ self.compiled_test_code = self._run_command(self.compile_main,
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ return self.compiled_user_answer, self.compiled_test_code
+
+ def check_code(self, user_answer, file_paths, expected_input, expected_output):
+ success = False
+ proc, stdnt_out, stdnt_stderr = self.compiled_user_answer
+ stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr)
+ if stdnt_stderr == '':
+ proc, main_out, main_err = self.compiled_test_code
+ main_err = self._remove_null_substitute_char(main_err)
+ if main_err == '':
+ proc = subprocess.Popen("./executable",
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ evaluator = Evaluator()
+ success, err = evaluator.evaluate(user_answer, proc,
+ expected_input,
+ expected_output
+ )
+ os.remove(self.ref_output_path)
+ else:
+ err = "Error:"
+ try:
+ error_lines = main_err.splitlines()
+ for e in error_lines:
+ if ':' in e:
+ err = err + "\n" + e.split(":", 1)[1]
+ else:
+ err = err + "\n" + e
+ except:
+ err = err + "\n" + main_err
+ os.remove(self.user_output_path)
+ else:
+ err = "Compilation Error:"
+ try:
+ error_lines = stdnt_stderr.splitlines()
+ for e in error_lines:
+ if ':' in e:
+ err = err + "\n" + e.split(":", 1)[1]
+ else:
+ err = err + "\n" + e
+ except:
+ err = err + "\n" + stdnt_stderr
+ return success, err
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index 1070c6a..addc5e6 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -1,9 +1,12 @@
import unittest
import os
from yaksh.bash_code_evaluator import BashCodeEvaluator
+from yaksh.bash_stdio_evaluator import BashStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
+from textwrap import dedent
-class BashEvaluationTestCases(unittest.TestCase):
+
+class BashAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [
{"test_case": "bash_files/sample.sh,bash_files/sample.args"}
@@ -66,5 +69,81 @@ class BashEvaluationTestCases(unittest.TestCase):
self.assertTrue(result.get("success"))
self.assertEquals(result.get("error"), "Correct answer")
+class BashStdioEvaluationTestCases(unittest.TestCase):
+ def setUp(self):
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in your"
+ " code.").format(SERVER_TIMEOUT)
+
+ def test_correct_answer(self):
+ user_answer = dedent(""" #!/bin/bash
+ read A
+ read B
+ echo -n `expr $A + $B`
+ """
+ )
+ test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ get_class = BashStdioEvaluator()
+ kwargs = {"user_answer": user_answer,
+ "test_case_data": test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_array_input(self):
+ user_answer = dedent(""" readarray arr;
+ COUNTER=0
+ while [ $COUNTER -lt 3 ]; do
+ echo -n "${arr[$COUNTER]}"
+ let COUNTER=COUNTER+1
+ done
+ """
+ )
+ test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
+ 'expected_input': '1,2,3\n4,5,6\n7,8,9'
+ }]
+ get_class = BashStdioEvaluator()
+ kwargs = {"user_answer": user_answer,
+ "test_case_data": test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ user_answer = dedent(""" #!/bin/bash
+ read A
+ read B
+ echo -n `expr $A - $B`
+ """
+ )
+ test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ get_class = BashStdioEvaluator()
+ kwargs = {"user_answer": user_answer,
+ "test_case_data": test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertIn("Incorrect", result.get('error'))
+ self.assertFalse(result.get('success'))
+
+ def test_stdout_only(self):
+ user_answer = dedent(""" #!/bin/bash
+ A=6
+ B=4
+ echo -n `expr $A + $B`
+ """
+ )
+ test_case_data = [{'expected_output': '10',
+ 'expected_input': ''
+ }]
+ get_class = BashStdioEvaluator()
+ kwargs = {"user_answer": user_answer,
+ "test_case_data": test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index 71af177..0042d0f 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -1,10 +1,12 @@
import unittest
import os
from yaksh.cpp_code_evaluator import CppCodeEvaluator
+from yaksh.cpp_stdio_evaluator import CppStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
-class CEvaluationTestCases(unittest.TestCase):
+
+class CAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
self.in_dir = os.getcwd()
@@ -83,5 +85,255 @@ class CEvaluationTestCases(unittest.TestCase):
self.assertEquals(result.get('error'), "Correct answer")
+class CppStdioEvaluationTestCases(unittest.TestCase):
+
+ def setUp(self):
+ self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ self.in_dir = os.getcwd()
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in"
+ " your code.").format(SERVER_TIMEOUT)
+
+ def test_correct_answer(self):
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ int a,b;
+ scanf("%d%d",&a,&b);
+ printf("%d",a+b);
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_array_input(self):
+ self.test_case_data = [{'expected_output': '561',
+ 'expected_input': '5\n6\n1'}]
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ int a[3],i;
+ for(i=0;i<3;i++){
+ scanf("%d",&a[i]);}
+ for(i=0;i<3;i++){
+ printf("%d",a[i]);}
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_string_input(self):
+ self.test_case_data = [{'expected_output': 'abc',
+ 'expected_input': 'abc'}]
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ char a[4];
+ scanf("%s",a);
+ printf("%s",a);
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ int a=10;
+ printf("%d",a);
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
+
+ def test_error(self):
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ int a=10;
+ printf("%d",a)
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertTrue("Compilation Error" in result.get("error"))
+
+ def test_infinite_loop(self):
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ while(0==0){
+ printf("abc");}
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertEquals(result.get("error"), self.timeout_msg)
+
+ def test_only_stdout(self):
+ self.test_case_data = [{'expected_output': '11',
+ 'expected_input': ''}]
+ user_answer = dedent("""
+ #include<stdio.h>
+ int main(void){
+ int a=5,b=6;
+ printf("%d",a+b);
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_cpp_correct_answer(self):
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ int a,b;
+ cin>>a>>b;
+ cout<<a+b;
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_cpp_array_input(self):
+ self.test_case_data = [{'expected_output': '561',
+ 'expected_input': '5\n6\n1'}]
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ int a[3],i;
+ for(i=0;i<3;i++){
+ cin>>a[i];}
+ for(i=0;i<3;i++){
+ cout<<a[i];}
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_cpp_string_input(self):
+ self.test_case_data = [{'expected_output': 'abc',
+ 'expected_input': 'abc'}]
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ char a[4];
+ cin>>a;
+ cout<<a;
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_cpp_incorrect_answer(self):
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ int a=10;
+ cout<<a;
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
+
+ def test_cpp_error(self):
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ int a=10;
+ cout<<a
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertTrue("Compilation Error" in result.get("error"))
+
+ def test_cpp_infinite_loop(self):
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ while(0==0){
+ cout<<"abc";}
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertEquals(result.get("error"), self.timeout_msg)
+
+ def test_cpp_only_stdout(self):
+ self.test_case_data = [{'expected_output': '11',
+ 'expected_input': ''}]
+ user_answer = dedent("""
+ #include<iostream>
+ using namespace std;
+ int main(void){
+ int a=5,b=6;
+ cout<<a+b;
+ }""")
+ get_class = CppStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py
index 51c0c51..cbca32d 100644
--- a/yaksh/evaluator_tests/test_code_evaluation.py
+++ b/yaksh/evaluator_tests/test_code_evaluation.py
@@ -17,7 +17,7 @@ class RegistryTestCase(unittest.TestCase):
)
code_evaluators['python'] = \
{"standardtestcase": assertion_evaluator_path,
- "stdoutbasedtestcase": stdout_evaluator_path
+ "stdiobasedtestcase": stdout_evaluator_path
}
def test_set_register(self):
@@ -35,7 +35,7 @@ class RegistryTestCase(unittest.TestCase):
)
self.registry_object.register("python",
{"standardtestcase": assertion_evaluator_path,
- "stdoutbasedtestcase": stdout_evaluator_path
+ "stdiobasedtestcase": stdout_evaluator_path
}
)
self.assertEquals(evaluator_class, class_name)
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index ed28745..74ac677 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -2,10 +2,12 @@ import unittest
import os
from yaksh import code_evaluator as evaluator
from yaksh.java_code_evaluator import JavaCodeEvaluator
+from yaksh.java_stdio_evaluator import JavaStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
-class JavaEvaluationTestCases(unittest.TestCase):
+
+class JavaAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [
{"test_case": "java_files/main_square.java"}
@@ -97,5 +99,148 @@ class JavaEvaluationTestCases(unittest.TestCase):
self.assertTrue(result.get("success"))
self.assertEquals(result.get("error"), "Correct answer")
+class JavaStdioEvaluationTestCases(unittest.TestCase):
+
+ def setUp(self):
+ self.test_case_data = [{'expected_output': '11',
+ 'expected_input': '5\n6'}]
+ evaluator.SERVER_TIMEOUT = 4
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop in"
+ " your code.").format(evaluator.SERVER_TIMEOUT)
+
+ def teardown(self):
+ evaluator.SERVER_TIMEOUT = 4
+
+ def test_correct_answer(self):
+ user_answer = dedent("""
+ import java.util.Scanner;
+ class Test
+ {public static void main(String[] args){
+ Scanner s = new Scanner(System.in);
+ int a = s.nextInt();
+ int b = s.nextInt();
+ System.out.print(a+b);
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_array_input(self):
+
+ self.test_case_data = [{'expected_output': '561',
+ 'expected_input': '5\n6\n1'}]
+ user_answer = dedent("""
+ import java.util.Scanner;
+ class Test
+ {public static void main(String[] args){
+ Scanner s = new Scanner(System.in);
+ int a[] = new int[3];
+ for (int i=0;i<3;i++){
+ a[i] = s.nextInt();
+ System.out.print(a[i]);}
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_incorrect_answer(self):
+
+ user_answer = dedent("""
+ import java.util.Scanner;
+ class Test
+ {public static void main(String[] args){
+ Scanner s = new Scanner(System.in);
+ int a = s.nextInt();
+ int b = s.nextInt();
+ System.out.print(a);
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get('success'))
+ self.assertIn("Incorrect", result.get('error'))
+ self.assertTrue(result.get('error').splitlines > 1)
+
+ def test_error(self):
+
+ user_answer = dedent("""
+ class Test
+ {
+ System.out.print("a");
+ }""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertTrue("Compilation Error" in result.get("error"))
+
+ def test_infinite_loop(self):
+
+ user_answer = dedent("""
+ class Test
+ {public static void main(String[] args){
+ while(0==0)
+ {
+ System.out.print("a");}
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertFalse(result.get("success"))
+ self.assertEquals(result.get("error"), self.timeout_msg)
+
+ def test_only_stdout(self):
+ self.test_case_data = [{'expected_output': '11',
+ 'expected_input': ''}]
+ user_answer = dedent("""
+ class Test
+ {public static void main(String[] args){
+ int a = 5;
+ int b = 6;
+ System.out.print(a+b);
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
+ def test_string_input(self):
+ self.test_case_data = [{'expected_output': 'HelloWorld',
+ 'expected_input': 'Hello\nWorld'}]
+ user_answer = dedent("""
+ import java.util.Scanner;
+ class Test
+ {public static void main(String[] args){
+ Scanner s = new Scanner(System.in);
+ String a = s.nextLine();
+ String b = s.nextLine();
+ System.out.print(a+b);
+ }}""")
+ get_class = JavaStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEquals(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
+
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index b432630..e46b21f 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -1,7 +1,7 @@
import unittest
import os
from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
-from yaksh.python_stdout_evaluator import PythonStdoutEvaluator
+from yaksh.python_stdio_evaluator import PythonStdioEvaluator
from yaksh.settings import SERVER_TIMEOUT
from textwrap import dedent
@@ -52,7 +52,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
- def test_syntax_error(self):
+
user_answer = dedent("""
def add(a, b);
return a + b
@@ -173,16 +173,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
return int(a) + int(b) + int(c)
""")
value_error_msg = ["Traceback",
- "call",
- "ValueError",
- "invalid literal",
- "base"
- ]
+ "call",
+ "ValueError",
+ "invalid literal",
+ "base"
+ ]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
@@ -209,73 +209,150 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
class PythonStdoutEvaluationTestCases(unittest.TestCase):
def setUp(self):
- self.test_case_data = [{"expected_output": "0 1 1 2 3"}]
+ self.test_case_data = [{"expected_input": None,
+ "expected_output": "0 1 1 2 3"
+ }]
+
self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop"
- " in your code.").format(SERVER_TIMEOUT)
+ "You probably have an infinite loop"
+ " in your code.").format(SERVER_TIMEOUT)
self.file_paths = None
def test_correct_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
- get_class = PythonStdoutEvaluator()
+ get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
- self.assertEqual(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct Answer")
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
- get_class = PythonStdoutEvaluator()
+ get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), "Incorrect Answer")
+ self.assertIn("Incorrect Answer", result.get('error'))
- def test_direct_printed_answer(self):
- user_answer = "print '0 1 1 2 3'"
- error_msg = ("Incorrect Answer: Please avoid printing"
- " the expected output directly"
- )
- get_class = PythonStdoutEvaluator()
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ def test_infinite_loop(self):
+ user_answer = "def add(a, b):\n\twhile True:\n\t\tpass\nadd(1,2)"
+ get_class = PythonStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+
+class PythonStdIOEvaluator(unittest.TestCase):
+
+ def setUp(self):
+ self.timeout_msg = ("Code took more than {0} seconds to run. "
+ "You probably have an infinite loop"
+ " in your code.").format(SERVER_TIMEOUT)
+
+ def test_add_two_integers_correct(self):
+ self.test_case_data = [{"expected_input": "1\n2",
+ "expected_output": "3"
+ }]
+ user_answer = dedent("""
+ a = int(raw_input())
+ b = int(raw_input())
+ print a+b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
result = get_class.evaluate(**kwargs)
- self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), error_msg)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_add_two_lists(self):
+ self.test_case_data = [{"expected_input": "[1,2,3]\n[5,6,7]",
+ "expected_output": "[1, 2, 3, 5, 6, 7]"
+ }]
+ user_answer = dedent("""
+ from ast import literal_eval
+ a = literal_eval(raw_input())
+ b = literal_eval(raw_input())
+ print a+b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
- def test_infinite_loop(self):
- user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
- get_class = PythonStdoutEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_find_string_in_sentence(self):
+ self.test_case_data = [{"expected_input": """the quick brown fox jumps\
+ over the lazy dog\nthe""",
+ "expected_output": "2"
+ }]
+ user_answer = dedent("""
+ a = raw_input()
+ b = raw_input()
+ print (a.count(b))
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertIn("Correct Answer", result.get('error'))
+
+ def test_add_two_integers_incorrect(self):
+ self.test_case_data = [{"expected_input": "1\n2",
+ "expected_output": "3"
+ }]
+ user_answer = dedent("""
+ a = int(raw_input())
+ b = int(raw_input())
+ print a-b
+ """
+ )
+
+ get_class = PythonStdioEvaluator()
+
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), 'Incorrect Answer')
+ self.assertIn("Incorrect Answer", result.get('error'))
def test_file_based_answer(self):
- self.test_case_data = [{"expected_output": "2\n\n"}]
+ self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+
user_answer = dedent("""
- with open("test.txt") as f:
- print f.read()
- """)
- get_class = PythonStdoutEvaluator()
+ with open("test.txt") as f:
+ a = f.read()
+ print a[0]
+ """
+ )
+ get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths
- }
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
- self.assertEqual(result.get('error'), "Correct answer")
+ self.assertEqual(result.get('error'), "Correct Answer")
self.assertTrue(result.get('success'))
if __name__ == '__main__':
diff --git a/yaksh/forms.py b/yaksh/forms.py
index 1226fe2..4a20102 100644
--- a/yaksh/forms.py
+++ b/yaksh/forms.py
@@ -1,9 +1,9 @@
from django import forms
-from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdoutBasedTestCase
+from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdioBasedTestCase
from django.contrib.auth import authenticate
from django.contrib.auth.models import User
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.models import ContentType
from taggit.managers import TaggableManager
from taggit.forms import TagField
@@ -33,7 +33,7 @@ question_types = (
test_case_types = (
("standardtestcase", "Standard Testcase"),
- ("stdoutbasedtestcase", "Stdout Based Testcase"),
+ ("stdiobasedtestcase", "Stdio Based Testcase"),
("mcqtestcase", "MCQ Testcase"),
)
diff --git a/yaksh/java_stdio_evaluator.py b/yaksh/java_stdio_evaluator.py
new file mode 100644
index 0000000..27dd4a9
--- /dev/null
+++ b/yaksh/java_stdio_evaluator.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+import subprocess
+import os
+from os.path import isfile
+
+#local imports
+from code_evaluator import CodeEvaluator
+from stdio_evaluator import Evaluator
+from file_utils import copy_files, delete_files
+
+
+class JavaStdioEvaluator(CodeEvaluator):
+ """Evaluates Java StdIO based code"""
+
+ def setup(self):
+ super(JavaStdioEvaluator, self).setup()
+ self.submit_code_path = self.create_submit_code_file('Test.java')
+
+ def teardown(self):
+ super(JavaStdioEvaluator, self).teardown()
+ os.remove(self.submit_code_path)
+ if self.files:
+ delete_files(self.files)
+
+ def set_file_paths(self, directory, file_name):
+ output_path = "{0}{1}.class".format(directory, file_name)
+ return output_path
+
+ def get_commands(self):
+ compile_command = 'javac {0}'.format(self.submit_code_path)
+ return compile_command
+
+ def compile_code(self, user_answer, file_paths, expected_input, expected_output):
+ self.files = []
+ if not isfile(self.submit_code_path):
+ msg = "No file at %s or Incorrect path" % self.submit_code_path
+ return False, msg
+ if file_paths:
+ self.files = copy_files(file_paths)
+ user_code_directory = os.getcwd() + '/'
+ self.write_to_submit_code_file(self.submit_code_path, user_answer)
+ self.user_output_path = self.set_file_paths(user_code_directory,
+ 'Test'
+ )
+ self.compile_command = self.get_commands()
+ self.compiled_user_answer = self._run_command(self.compile_command,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ return self.compiled_user_answer
+
+ def check_code(self, user_answer, file_paths, expected_input, expected_output):
+ success = False
+ proc, stdnt_out, stdnt_stderr = self.compiled_user_answer
+ stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr)
+ if stdnt_stderr == '' or "error" not in stdnt_stderr:
+ proc = subprocess.Popen("java Test",
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ evaluator = Evaluator()
+ success, err = evaluator.evaluate(user_answer, proc,
+ expected_input,
+ expected_output
+ )
+ os.remove(self.user_output_path)
+ else:
+ err = "Compilation Error:"
+ try:
+ error_lines = stdnt_stderr.splitlines()
+ for e in error_lines:
+ if ':' in e:
+ err = err + "\n" + e.split(":", 1)[1]
+ else:
+ err = err + "\n" + e
+ except:
+ err = err + "\n" + stdnt_stderr
+ return success, err
diff --git a/yaksh/models.py b/yaksh/models.py
index 81926e6..0f801b8 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -38,7 +38,7 @@ enrollment_methods = (
test_case_types = (
("standardtestcase", "Standard Testcase"),
- ("stdoutbasedtestcase", "Stdout Based Testcase"),
+ ("stdiobasedtestcase", "StdIO Based Testcase"),
("mcqtestcase", "MCQ Testcase"),
)
@@ -930,15 +930,17 @@ class StandardTestCase(TestCase):
)
-class StdoutBasedTestCase(TestCase):
- expected_output = models.TextField(blank=True)
+class StdioBasedTestCase(TestCase):
+ expected_input = models.TextField(blank=True)
+ expected_output = models.TextField()
def get_field_value(self):
- return {"expected_output": self.expected_output}
+ return {"expected_output": self.expected_output,
+ "expected_input": self.expected_input}
def __unicode__(self):
- return u'Question: {0} | Exp. Output: {1}'.format(self.question,
- self.expected_output
+ return u'Question: {0} | Exp. Output: {1} | Exp. Input: {2}'.format(self.question,
+ self.expected_output, self.expected_input
)
diff --git a/yaksh/python_stdio_evaluator.py b/yaksh/python_stdio_evaluator.py
new file mode 100644
index 0000000..4a02267
--- /dev/null
+++ b/yaksh/python_stdio_evaluator.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+import sys
+import traceback
+import os
+from os.path import join
+import importlib
+from contextlib import contextmanager
+from ast import literal_eval
+# local imports
+from code_evaluator import CodeEvaluator
+from StringIO import StringIO
+from file_utils import copy_files, delete_files
+from textwrap import dedent
+@contextmanager
+def redirect_stdout():
+ new_target = StringIO()
+ old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+ try:
+ yield new_target # run some code with the replaced stdout
+ finally:
+ sys.stdout = old_target # restore to the previous value
+
+
+class PythonStdioEvaluator(CodeEvaluator):
+ """Tests the Python code obtained from Code Server"""
+
+ def teardown(self):
+ super(PythonStdioEvaluator, self).teardown()
+ # Delete the created file.
+ if self.files:
+ delete_files(self.files)
+
+
+ def compile_code(self, user_answer, file_paths, expected_input, expected_output):
+ self.files = []
+ if file_paths:
+ self.files = copy_files(file_paths)
+ submitted = compile(user_answer, '<string>', mode='exec')
+ if expected_input:
+ input_buffer = StringIO()
+ input_buffer.write(expected_input)
+ input_buffer.seek(0)
+ sys.stdin = input_buffer
+ with redirect_stdout() as output_buffer:
+ exec_scope = {}
+ exec submitted in exec_scope
+ self.output_value = output_buffer.getvalue().rstrip("\n")
+ return self.output_value
+
+ def check_code(self, user_answer, file_paths, expected_input, expected_output):
+ success = False
+
+ tb = None
+ if self.output_value == expected_output:
+ success = True
+ err = "Correct Answer"
+ else:
+ success = False
+ err = dedent("""
+ Incorrect Answer:
+ Given input - {0}
+ Expected output - {1}
+ Your output - {2}
+ """
+ .format(expected_input,
+ expected_output, self.output_value
+ )
+ )
+ del tb
+ return success, err
diff --git a/yaksh/settings.py b/yaksh/settings.py
index 70e5471..b1336f4 100644
--- a/yaksh/settings.py
+++ b/yaksh/settings.py
@@ -11,7 +11,7 @@ SERVER_PORTS = [8001] # range(8001, 8026)
SERVER_POOL_PORT = 53579
# Timeout for the code to run in seconds. This is an integer!
-SERVER_TIMEOUT = 2
+SERVER_TIMEOUT = 4
# The root of the URL, for example you might be in the situation where you
# are not hosted as host.org/exam/ but as host.org/foo/exam/ for whatever
@@ -21,11 +21,20 @@ URL_ROOT = ''
code_evaluators = {
"python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator",
- "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator"
- },
- "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"},
- "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"},
- "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"},
- "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"},
+ "stdiobasedtestcase": "python_stdio_evaluator.PythonStdioEvaluator"
+ },
+ "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator",
+ "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"
+ },
+ "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator",
+ "stdiobasedtestcase": "cpp_stdio_evaluator.CppStdioEvaluator"
+ },
+ "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator",
+ "stdiobasedtestcase": "java_stdio_evaluator.JavaStdioEvaluator"},
+
+ "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator",
+ "stdiobasedtestcase": "bash_stdio_evaluator.BashStdioEvaluator"
+ },
+
"scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"},
}
diff --git a/yaksh/stdio_evaluator.py b/yaksh/stdio_evaluator.py
new file mode 100644
index 0000000..4f5cfaf
--- /dev/null
+++ b/yaksh/stdio_evaluator.py
@@ -0,0 +1,22 @@
+class Evaluator(object):
+
+ def evaluate(self, user_answer, proc, expected_input, expected_output):
+ success = False
+ ip = expected_input.replace(",", " ")
+ user_output, output_err = proc.communicate(input='{0}\n'.format(ip))
+ expected_output = expected_output.replace("\r", "")
+ if not expected_input:
+ error_msg = "Expected Output is {0} ".\
+ format(repr(expected_output))
+ else:
+ error_msg = " Given Input is\n {0} \n Expected Output is {1} ".\
+ format(expected_input, repr(expected_output))
+ if output_err == '':
+ if user_output == expected_output:
+ success, err = True, "Correct Answer"
+ else:
+ err = " Incorrect Answer\n" + error_msg +\
+ "\n Your output is {0}".format(repr(user_output))
+ else:
+ err = "Error:"+"\n"+output_err
+ return success, err
diff --git a/yaksh/test_models.py b/yaksh/test_models.py
index 2136987..ca64f8c 100644
--- a/yaksh/test_models.py
+++ b/yaksh/test_models.py
@@ -1,7 +1,7 @@
import unittest
from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\
QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\
- StdoutBasedTestCase
+ StdioBasedTestCase
import json
from datetime import datetime, timedelta
from django.utils import timezone
@@ -709,7 +709,7 @@ class TestCaseTestCases(unittest.TestCase):
active=True,
description='Write to standard output',
points=1.0,
- test_case_type="stdoutbasedtestcase",
+ test_case_type="stdiobasedtestcase",
user=self.user,
snippet='def myfunc()'
)
@@ -719,7 +719,7 @@ class TestCaseTestCases(unittest.TestCase):
question=self.question1,
test_case='assert myfunc(12, 13) == 15'
)
- self.stdout_based_testcase = StdoutBasedTestCase(
+ self.stdout_based_testcase = StdioBasedTestCase(
question=self.question2,
expected_output='Hello World'
)
diff --git a/yaksh/test_views.py b/yaksh/test_views.py
index 4dd905e..e232bc0 100644
--- a/yaksh/test_views.py
+++ b/yaksh/test_views.py
@@ -9,7 +9,7 @@ from django.utils import timezone
from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\
QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\
- StdoutBasedTestCase, has_profile
+ StdioBasedTestCase, has_profile
class TestProfile(TestCase):
generated by cgit (git 2.34.1) at 2024-12-22 13:33:35 +0000