from __future__ import absolute_import import unittest import os import shutil import tempfile from textwrap import dedent # Local import from yaksh.grader import Grader from yaksh.cpp_code_evaluator import CppCodeEvaluator from yaksh.cpp_stdio_evaluator import CppStdIOEvaluator from yaksh.evaluator_tests.test_python_evaluation import EvaluatorBaseTest from yaksh.settings import SERVER_TIMEOUT class CAssertionEvaluationTestCases(EvaluatorBaseTest): def setUp(self): self.f_path = os.path.join(tempfile.gettempdir(), "test.txt") with open(self.f_path, 'wb') as f: f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.tc_data = dedent(""" #include #include extern int add(int, int); template void check(T expect, T result) { if (expect == result) { printf("Correct: Expected %d got %d ",expect,result); } else { printf("Incorrect: Expected %d got %d ",expect,result); exit (1); } } int main(void) { int result; result = add(0,0); printf("Input submitted to the function: 0, 0"); check(0, result); result = add(2,3); printf("Input submitted to the function: 2 3"); check(5,result); printf("All Correct"); return 0; } """) self.test_case_data = [{"test_case": self.tc_data, "test_case_type": "standardtestcase", "weight": 0.0 }] self.in_dir = tmp_in_dir_path self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your" " code.").format(SERVER_TIMEOUT) self.file_paths = None def tearDown(self): os.remove(self.f_path) shutil.rmtree(self.in_dir) def test_correct_answer(self): # Given user_answer = "int add(int a, int b)\n{return a+b;}" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) def test_incorrect_answer(self): # Given user_answer = "int add(int a, int b)\n{return a-b;}" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then lines_of_error = len(result.get('error')[0].splitlines()) self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect:", result.get('error')) self.assertTrue(lines_of_error > 1) def test_compilation_error(self): # Given user_answer = "int add(int a, int b)\n{return a+b}" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output("Compilation Error", result.get("error")) def test_infinite_loop(self): # Given user_answer = "int add(int a, int b)\n{while(1>0){}}" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")) def test_file_based_assert(self): # Given self.file_paths = [(self.f_path, False)] self.tc_data = dedent(""" #include #include extern int ans(); template void check(T expect,T result) { if (expect == result) { printf("Correct: Expected %d got %d ",expect,result); } else { printf("Incorrect: Expected %d got %d ",expect,result); exit (0); } } int main(void) { int result; result = ans(); check(50, result); } """) self.test_case_data = [{"test_case": self.tc_data, "test_case_type": "standardtestcase", "weight": 0.0 }] user_answer = dedent(""" #include char ans() { FILE *fp; char buff[255]; fp = fopen("test.txt", "r"); fscanf(fp, "%s", buff); fclose(fp); return buff[0]; } """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) class CppStdIOEvaluationTestCases(EvaluatorBaseTest): def setUp(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] self.in_dir = tempfile.mkdtemp() self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) self.file_paths = None def tearDown(self): shutil.rmtree(self.in_dir) def test_correct_answer(self): # Given user_answer = dedent(""" #include int main(void){ int a,b; scanf("%d%d",&a,&b); printf("%d",a+b); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) def test_array_input(self): # Given self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include int main(void){ int a[3],i; for(i=0;i<3;i++){ scanf("%d",&a[i]);} for(i=0;i<3;i++){ printf("%d",a[i]);} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) def test_string_input(self): # Given self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include int main(void){ char a[4]; scanf("%s",a); printf("%s",a); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) def test_incorrect_answer(self): # Given user_answer = dedent(""" #include int main(void){ int a=10; printf("%d",a); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then lines_of_error = len(result.get('error')[0].splitlines()) self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect", result.get('error')) self.assertTrue(lines_of_error > 1) def test_error(self): # Given user_answer = dedent(""" #include int main(void){ int a=10; printf("%d",a) }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output("Compilation Error", result.get("error")) def test_infinite_loop(self): # Given user_answer = dedent(""" #include int main(void){ while(0==0){ printf("abc");} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")) def test_only_stdout(self): # Given self.test_case_data = [{'expected_output': '11', 'expected_input': '', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include int main(void){ int a=5,b=6; printf("%d",a+b); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success')) def test_cpp_correct_answer(self): # Given user_answer = dedent(""" #include using namespace std; int main(void){ int a,b; cin>>a>>b; cout< using namespace std; int main(void){ int a[3],i; for(i=0;i<3;i++){ cin>>a[i];} for(i=0;i<3;i++){ cout< using namespace std; int main(void){ char a[4]; cin>>a; cout< using namespace std; int main(void){ int a=10; cout< 1) def test_cpp_error(self): # Given user_answer = dedent(""" #include using namespace std; int main(void){ int a=10; cout< using namespace std; int main(void){ while(0==0){ cout<<"abc";} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")) def test_cpp_only_stdout(self): # Given self.test_case_data = [{'expected_output': '11', 'expected_input': '', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include using namespace std; int main(void){ int a=5,b=6; cout<