summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests')
-rw-r--r--yaksh/evaluator_tests/test_bash_evaluation.py26
-rw-r--r--yaksh/evaluator_tests/test_c_cpp_evaluation.py40
-rw-r--r--yaksh/evaluator_tests/test_java_evaluation.py46
-rw-r--r--yaksh/evaluator_tests/test_python_evaluation.py87
-rw-r--r--yaksh/evaluator_tests/test_scilab_evaluation.py15
5 files changed, 173 insertions, 41 deletions
diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py
index ee6c1f0..addc5e6 100644
--- a/yaksh/evaluator_tests/test_bash_evaluation.py
+++ b/yaksh/evaluator_tests/test_bash_evaluation.py
@@ -11,10 +11,11 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
self.test_case_data = [
{"test_case": "bash_files/sample.sh,bash_files/sample.args"}
]
- self.in_dir = "/tmp"
+ self.in_dir = os.getcwd()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
def test_correct_answer(self):
user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]"
@@ -22,7 +23,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
)
get_class = BashCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
@@ -33,7 +35,8 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
"&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))")
get_class = BashCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
@@ -44,12 +47,27 @@ class BashAssertionEvaluationTestCases(unittest.TestCase):
" do echo "" > /dev/null ; done")
get_class = BashCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
+ def test_file_based_assert(self):
+ self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.test_case_data = [
+ {"test_case": "bash_files/sample1.sh,bash_files/sample1.args"}
+ ]
+ user_answer = ("#!/bin/bash\ncat $1")
+ get_class = BashCodeEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get("success"))
+ self.assertEquals(result.get("error"), "Correct answer")
class BashStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
index ff3cddf..0042d0f 100644
--- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py
+++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py
@@ -9,16 +9,18 @@ from textwrap import dedent
class CAssertionEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}]
- self.in_dir = "/tmp"
+ self.in_dir = os.getcwd()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in your"
" code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
def test_correct_answer(self):
user_answer = "int add(int a, int b)\n{return a+b;}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
@@ -28,7 +30,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "int add(int a, int b)\n{return a-b;}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
@@ -39,7 +42,8 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "int add(int a, int b)\n{return a+b}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
@@ -49,17 +53,43 @@ class CAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "int add(int a, int b)\n{while(1>0){}}"
get_class = CppCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
+ def test_file_based_assert(self):
+ self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.test_case_data = [{"test_case": "c_cpp_files/file_data.c"}]
+ user_answer = dedent("""
+ #include<stdio.h>
+ char ans()
+ {
+ FILE *fp;
+ char buff[255];
+ fp = fopen("test.txt", "r");
+ fscanf(fp, "%s", buff);
+ fclose(fp);
+ return buff[0];
+ }
+ """)
+ get_class = CppCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get('success'))
+ self.assertEquals(result.get('error'), "Correct answer")
+
class CppStdioEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{'expected_output': '11', 'expected_input': '5\n6'}]
+ self.in_dir = os.getcwd()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py
index c98a938..74ac677 100644
--- a/yaksh/evaluator_tests/test_java_evaluation.py
+++ b/yaksh/evaluator_tests/test_java_evaluation.py
@@ -12,11 +12,12 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
self.test_case_data = [
{"test_case": "java_files/main_square.java"}
]
- self.in_dir = "/tmp"
+ self.in_dir = os.getcwd()
evaluator.SERVER_TIMEOUT = 9
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(evaluator.SERVER_TIMEOUT)
+ self.file_paths = None
def tearDown(self):
evaluator.SERVER_TIMEOUT = 2
@@ -25,7 +26,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertEquals(result.get('error'), "Correct answer")
@@ -35,7 +37,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
@@ -46,7 +49,8 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a"
get_class = JavaCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
@@ -56,12 +60,44 @@ class JavaAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
get_class = JavaCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
self.assertEquals(result.get("error"), self.timeout_msg)
+ def test_file_based_assert(self):
+ self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ self.test_case_data = [
+ {"test_case": "java_files/read_file.java"}
+ ]
+ user_answer = dedent("""
+ import java.io.BufferedReader;
+ import java.io.FileReader;
+ import java.io.IOException;
+ class Test{
+ String readFile() throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader("test.txt"));
+ try {
+ StringBuilder sb = new StringBuilder();
+ String line = br.readLine();
+ while (line != null) {
+ sb.append(line);
+ line = br.readLine();}
+ return sb.toString();
+ } finally {
+ br.close();
+ }}}
+ """)
+ get_class = JavaCodeEvaluator(self.in_dir)
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertTrue(result.get("success"))
+ self.assertEquals(result.get("error"), "Correct answer")
class JavaStdioEvaluationTestCases(unittest.TestCase):
diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py
index 40562db..eea9403 100644
--- a/yaksh/evaluator_tests/test_python_evaluation.py
+++ b/yaksh/evaluator_tests/test_python_evaluation.py
@@ -15,13 +15,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
def test_correct_answer(self):
user_answer = "def add(a,b):\n\treturn a + b"
get_class = PythonAssertionEvaluator()
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer")
@@ -30,8 +32,9 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "def add(a,b):\n\treturn a - b"
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
- }
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'),
@@ -42,13 +45,14 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
- def test_syntax_error(self):
+
user_answer = dedent("""
def add(a, b);
return a + b
@@ -63,7 +67,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -86,7 +91,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -105,7 +111,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -126,7 +133,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -148,7 +156,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -171,7 +180,8 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
@@ -180,6 +190,22 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase):
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
+ def test_file_based_assert(self):
+ self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
+ self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+ user_answer = dedent("""
+ def ans():
+ with open("test.txt") as f:
+ return f.read()[0]
+ """)
+ get_class = PythonAssertionEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEqual(result.get('error'), "Correct answer")
+ self.assertTrue(result.get('success'))
class PythonStdoutEvaluationTestCases(unittest.TestCase):
def setUp(self):
@@ -190,12 +216,14 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop"
" in your code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
def test_correct_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b"
get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct Answer")
@@ -204,8 +232,9 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
def test_incorrect_answer(self):
user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b"
get_class = PythonStdioEvaluator()
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
@@ -217,11 +246,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase):
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
- result = get_class.evaluate(**kwargs)
- self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), self.timeout_msg)
-
-
+
class PythonStdIOEvaluator(unittest.TestCase):
def setUp(self):
@@ -305,12 +330,30 @@ class PythonStdIOEvaluator(unittest.TestCase):
get_class = PythonStdioEvaluator()
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertIn("Incorrect Answer", result.get('error'))
+ def test_file_based_answer(self):
+ self.test_case_data = [{"expected_input": "", "expected_output": "2"}]
+ self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
+
+ user_answer = dedent("""
+ with open("test.txt") as f:
+ a = f.read()
+ print a[0]
+ """
+ )
+ get_class = PythonStdioEvaluator()
+ kwargs = {'user_answer': user_answer,
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
+ }
+ result = get_class.evaluate(**kwargs)
+ self.assertEqual(result.get('error'), "Correct Answer")
+ self.assertTrue(result.get('success'))
if __name__ == '__main__':
unittest.main()
diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py
index 242f260..f5e3767 100644
--- a/yaksh/evaluator_tests/test_scilab_evaluation.py
+++ b/yaksh/evaluator_tests/test_scilab_evaluation.py
@@ -7,17 +7,19 @@ from yaksh.settings import SERVER_TIMEOUT
class ScilabEvaluationTestCases(unittest.TestCase):
def setUp(self):
self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}]
- self.in_dir = "/tmp"
+ self.in_dir = os.getcwd()
self.timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop"
" in your code.").format(SERVER_TIMEOUT)
+ self.file_paths = None
def test_correct_answer(self):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a+b;\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertEquals(result.get('error'), "Correct answer")
@@ -28,7 +30,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
"\n\tc=a+b;\ndis(\tendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
@@ -40,7 +43,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
"\n\tc=a-b;\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
@@ -52,7 +56,8 @@ class ScilabEvaluationTestCases(unittest.TestCase):
"\n\tc=a;\nwhile(1==1)\nend\nendfunction")
get_class = ScilabCodeEvaluator(self.in_dir)
kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data
+ 'test_case_data': self.test_case_data,
+ 'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get("success"))
generated by cgit (git 2.34.1) at 2024-12-20 20:19:49 +0000