summaryrefslogtreecommitdiff
path: root/yaksh/evaluator_tests/old_test_python_evaluation.py
diff options
context:
space:
mode:
Diffstat (limited to 'yaksh/evaluator_tests/old_test_python_evaluation.py')
-rw-r--r--yaksh/evaluator_tests/old_test_python_evaluation.py549
1 files changed, 0 insertions, 549 deletions
diff --git a/yaksh/evaluator_tests/old_test_python_evaluation.py b/yaksh/evaluator_tests/old_test_python_evaluation.py
deleted file mode 100644
index 9796fa2..0000000
--- a/yaksh/evaluator_tests/old_test_python_evaluation.py
+++ /dev/null
@@ -1,549 +0,0 @@
-from __future__ import unicode_literals
-import unittest
-import os
-import tempfile
-import shutil
-from textwrap import dedent
-
-# Local import
-from yaksh.python_assertion_evaluator import PythonAssertionEvaluator
-from yaksh.python_stdio_evaluator import PythonStdioEvaluator
-from yaksh.settings import SERVER_TIMEOUT
-
-
-class PythonAssertionEvaluationTestCases(unittest.TestCase):
- def setUp(self):
- with open('/tmp/test.txt', 'wb') as f:
- f.write('2'.encode('ascii'))
- tmp_in_dir_path = tempfile.mkdtemp()
- self.in_dir = tmp_in_dir_path
- self.test_case_data = [{"test_case": 'assert(add(1,2)==3)', 'weight': 0.0},
- {"test_case": 'assert(add(-1,2)==1)', 'weight': 0.0},
- {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0},
- ]
- self.timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop in"
- " your code.").format(SERVER_TIMEOUT)
- self.file_paths = None
-
- def tearDown(self):
- os.remove('/tmp/test.txt')
- shutil.rmtree(self.in_dir)
-
- def test_correct_answer(self):
- # Given
- user_answer = "def add(a,b):\n\treturn a + b"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
-
- def test_incorrect_answer(self):
- # Given
- user_answer = "def add(a,b):\n\treturn a - b"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertFalse(result.get('success'))
- self.assertIn('AssertionError in: assert(add(1,2)==3)',
- result.get('error')
- )
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
- result.get('error')
- )
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
- result.get('error')
- )
-
- def test_partial_incorrect_answer(self):
- # Given
- user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
- test_case_data = [{"test_case": 'assert(add(-1,2)==1)', 'weight': 1.0},
- {"test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0},
- {"test_case": 'assert(add(1,2)==3)', 'weight': 2.0}
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': True
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertFalse(result.get('success'))
- self.assertEqual(result.get('weight'), 2.0)
- self.assertIn('AssertionError in: assert(add(-1,2)==1)',
- result.get('error')
- )
- self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
- result.get('error')
- )
-
- def test_infinite_loop(self):
- # Given
- user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertFalse(result.get('success'))
- self.assertEqual(result.get('error'), self.timeout_msg)
-
- def test_syntax_error(self):
- # Given
- user_answer = dedent("""
- def add(a, b);
- return a + b
- """)
- syntax_error_msg = ["Traceback",
- "call",
- "File",
- "line",
- "<string>",
- "SyntaxError",
- "invalid syntax"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(5, len(err))
- for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_indent_error(self):
- # Given
- user_answer = dedent("""
- def add(a, b):
- return a + b
- """)
- indent_error_msg = ["Traceback", "call",
- "File",
- "line",
- "<string>",
- "IndentationError",
- "indented block"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
-
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(5, len(err))
- for msg in indent_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_name_error(self):
- # Given
- user_answer = ""
- name_error_msg = ["Traceback",
- "call",
- "NameError",
- "name",
- "defined"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(3, len(err))
- for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_recursion_error(self):
- # Given
- user_answer = dedent("""
- def add(a, b):
- return add(3, 3)
- """)
- recursion_error_msg = ["Traceback",
- "call",
- "maximum recursion depth exceeded"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- for msg in recursion_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_type_error(self):
- # Given
- user_answer = dedent("""
- def add(a):
- return a + b
- """)
- type_error_msg = ["Traceback",
- "call",
- "TypeError",
- "argument"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(3, len(err))
- for msg in type_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_value_error(self):
- # Given
- user_answer = dedent("""
- def add(a, b):
- c = 'a'
- return int(a) + int(b) + int(c)
- """)
- value_error_msg = ["Traceback",
- "call",
- "ValueError",
- "invalid literal",
- "base"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(4, len(err))
- for msg in value_error_msg:
- self.assertIn(msg, result.get("error"))
-
- def test_file_based_assert(self):
- # Given
- self.test_case_data = [{"test_case": "assert(ans()=='2')", "weight": 0.0}]
- self.file_paths = [('/tmp/test.txt', False)]
- user_answer = dedent("""
- def ans():
- with open("test.txt") as f:
- return f.read()[0]
- """)
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertIn("Correct answer", result.get('error'))
- self.assertTrue(result.get('success'))
-
- def test_single_testcase_error(self):
- # Given
- """ Tests the user answer with just an incorrect test case """
-
- user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
- test_case_data = [{"test_case": 's="abbb"\nasert palindrome(s)==False',
- "weight": 0.0
- }
- ]
- syntax_error_msg = ["Traceback",
- "call",
- "File",
- "line",
- "<string>",
- "SyntaxError",
- "invalid syntax"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(5, len(err))
- for msg in syntax_error_msg:
- self.assertIn(msg, result.get("error"))
-
-
- def test_multiple_testcase_error(self):
- """ Tests the user answer with an correct test case
- first and then with an incorrect test case """
- # Given
- user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
- test_case_data = [{"test_case": 'assert(palindrome("abba")==True)',
- "weight": 0.0
- },
- {"test_case": 's="abbb"\nassert palindrome(S)==False',
- "weight": 0.0
- }
- ]
- name_error_msg = ["Traceback",
- "call",
- "File",
- "line",
- "<string>",
- "NameError",
- "name 'S' is not defined"
- ]
- kwargs = {'user_answer': user_answer,
- 'test_case_data': test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonAssertionEvaluator()
- result = evaluator.evaluate(**kwargs)
- err = result.get("error").splitlines()
-
- # Then
- self.assertFalse(result.get("success"))
- self.assertEqual(3, len(err))
- for msg in name_error_msg:
- self.assertIn(msg, result.get("error"))
-
-
-class PythonStdIOEvaluationTestCases(unittest.TestCase):
- def setUp(self):
- with open('/tmp/test.txt', 'wb') as f:
- f.write('2'.encode('ascii'))
- self.file_paths = None
-
- def test_correct_answer_integer(self):
- # Given
- self.test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3",
- "weight": 0.0
- }]
- user_answer = dedent("""
- a = int(input())
- b = int(input())
- print(a+b)
- """
- )
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
-
- def test_correct_answer_list(self):
- # Given
- self.test_case_data = [{"expected_input": "1,2,3\n5,6,7",
- "expected_output": "[1, 2, 3, 5, 6, 7]",
- "weight": 0.0
- }]
- user_answer = dedent("""
- from six.moves import input
- input_a = input()
- input_b = input()
- a = [int(i) for i in input_a.split(',')]
- b = [int(i) for i in input_b.split(',')]
- print(a+b)
- """
- )
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
-
- def test_correct_answer_string(self):
- # Given
- self.test_case_data = [{"expected_input": ("the quick brown fox jumps over the lazy dog\nthe"),
- "expected_output": "2",
- "weight": 0.0
- }]
- user_answer = dedent("""
- from six.moves import input
- a = str(input())
- b = str(input())
- print(a.count(b))
- """
- )
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertTrue(result.get('success'))
- self.assertIn("Correct answer", result.get('error'))
-
- def test_incorrect_answer_integer(self):
- # Given
- self.test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3",
- "weight": 0.0
- }]
- user_answer = dedent("""
- a = int(input())
- b = int(input())
- print(a-b)
- """
- )
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertFalse(result.get('success'))
- self.assertIn("Incorrect answer", result.get('error'))
-
- def test_file_based_answer(self):
- # Given
- self.test_case_data = [{"expected_input": "",
- "expected_output": "2",
- "weight": 0.0
- }]
- self.file_paths = [('/tmp/test.txt', False)]
-
- user_answer = dedent("""
- with open("test.txt") as f:
- a = f.read()
- print(a[0])
- """
- )
- kwargs = {'user_answer': user_answer,
- 'test_case_data': self.test_case_data,
- 'file_paths': self.file_paths,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertEqual(result.get('error'), "Correct answer\n")
- self.assertTrue(result.get('success'))
-
- def test_infinite_loop(self):
- # Given
- test_case_data = [{"expected_input": "1\n2",
- "expected_output": "3",
- "weight": 0.0
- }]
- timeout_msg = ("Code took more than {0} seconds to run. "
- "You probably have an infinite loop in"
- " your code.").format(SERVER_TIMEOUT)
- user_answer = "while True:\n\tpass"
- kwargs = {'user_answer': user_answer,
- 'test_case_data': test_case_data,
- 'partial_grading': False
- }
-
- # When
- evaluator = PythonStdioEvaluator()
- result = evaluator.evaluate(**kwargs)
-
- # Then
- self.assertEqual(result.get('error'), timeout_msg)
- self.assertFalse(result.get('success'))
-
-if __name__ == '__main__':
- unittest.main()
generated by cgit (git 2.34.1) at 2024-12-20 12:45:02 +0000