diff options
Diffstat (limited to 'yaksh')
| -rw-r--r-- | yaksh/evaluator_tests/test_simple_question_types.py | 164 | ||||
| -rw-r--r-- | yaksh/models.py | 16 | ||||
| -rw-r--r-- | yaksh/test_models.py | 79 |
3 files changed, 245 insertions, 14 deletions
diff --git a/yaksh/evaluator_tests/test_simple_question_types.py b/yaksh/evaluator_tests/test_simple_question_types.py index fb1c220..c0b81d6 100644 --- a/yaksh/evaluator_tests/test_simple_question_types.py +++ b/yaksh/evaluator_tests/test_simple_question_types.py @@ -68,6 +68,7 @@ class IntegerQuestionTestCases(unittest.TestCase): self.answerpaper = AnswerPaper.objects.get(question_paper\ =self.question_paper) self.answerpaper.attempt_number = 1 + self.answerpaper.questions.add(self.question1) self.answerpaper.save() # For question self.integer_based_testcase = IntegerTestCase(question=self.question1, @@ -80,7 +81,7 @@ class IntegerQuestionTestCases(unittest.TestCase): def tearDownClass(self): self.question1.delete() - def test_integer_correct_answer(self): + def test_validate_regrade_integer_correct_answer(self): # Given integer_answer = 25 self.answer = Answer(question=self.question1, @@ -88,7 +89,7 @@ class IntegerQuestionTestCases(unittest.TestCase): ) self.answer.save() self.answerpaper.answers.add(self.answer) - + self.answerpaper.save() # When json_data = None result = self.answerpaper.validate_answer(integer_answer, @@ -98,7 +99,26 @@ class IntegerQuestionTestCases(unittest.TestCase): # Then self.assertTrue(result['success']) - def test_integer_incorrect_answer(self): + # Regrade + # Given + self.answer.correct = True + self.answer.marks = 1 + + self.answer.answer = 200 + self.answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 0) + self.assertFalse(self.answer.correct) + + + def test_validate_regrade_integer_incorrect_answer(self): # Given integer_answer = 26 self.answer = Answer(question=self.question1, @@ -116,6 +136,24 @@ class IntegerQuestionTestCases(unittest.TestCase): # Then self.assertFalse(result['success']) + # Regrade + # Given + self.answer.correct = True + self.answer.marks = 1 + + self.answer.answer = 25 + self.answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 1) + self.assertTrue(self.answer.correct) + class StringQuestionTestCases(unittest.TestCase): @classmethod @@ -147,6 +185,7 @@ class StringQuestionTestCases(unittest.TestCase): self.answerpaper = AnswerPaper.objects.get(question_paper\ =self.question_paper) self.answerpaper.attempt_number = 1 + self.answerpaper.questions.add(*[self.question1, self.question2]) self.answerpaper.save() # For question @@ -169,7 +208,7 @@ class StringQuestionTestCases(unittest.TestCase): self.question1.delete() self.question2.delete() - def test_case_insensitive_string_correct_answer(self): + def test_validate_regrade_case_insensitive_string_correct_answer(self): # Given string_answer = "hello, earth!" answer = Answer(question=self.question1,answer=string_answer) @@ -184,7 +223,25 @@ class StringQuestionTestCases(unittest.TestCase): # Then self.assertTrue(result['success']) - def test_case_insensitive_string_incorrect_answer(self): + # Regrade + # Given + answer.correct = True + answer.marks = 1 + + answer.answer = "hello, mars!" + answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + answer = self.answerpaper.answers.filter(question=self.question1)\ + .last() + self.assertTrue(details[0]) + self.assertEqual(answer.marks, 0) + self.assertFalse(answer.correct) + + def test_validate_regrade_case_insensitive_string_incorrect_answer(self): # Given string_answer = "hello, mars!" answer = Answer(question=self.question1,answer=string_answer) @@ -200,7 +257,25 @@ class StringQuestionTestCases(unittest.TestCase): # Then self.assertFalse(result['success']) - def test_case_sensitive_string_correct_answer(self): + # Regrade + # Given + answer.correct = True + answer.marks = 1 + + answer.answer = "hello, earth!" + answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + answer = self.answerpaper.answers.filter(question=self.question1)\ + .last() + self.assertTrue(details[0]) + self.assertEqual(answer.marks, 1) + self.assertTrue(answer.correct) + + def test_validate_regrade_case_sensitive_string_correct_answer(self): # Given string_answer = "Hello, EARTH!" answer = Answer(question=self.question2,answer=string_answer) @@ -215,6 +290,24 @@ class StringQuestionTestCases(unittest.TestCase): # Then self.assertTrue(result['success']) + # Regrade + # Given + answer.correct = True + answer.marks = 1 + + answer.answer = "hello, earth!" + answer.save() + + # When + details = self.answerpaper.regrade(self.question2.id) + + # Then + answer = self.answerpaper.answers.filter(question=self.question2)\ + .last() + self.assertTrue(details[0]) + self.assertEqual(answer.marks, 0) + self.assertFalse(answer.correct) + def test_case_sensitive_string_incorrect_answer(self): # Given string_answer = "hello, earth!" @@ -231,6 +324,24 @@ class StringQuestionTestCases(unittest.TestCase): # Then self.assertFalse(result['success']) + # Regrade + # Given + answer.correct = True + answer.marks = 1 + + answer.answer = "Hello, EARTH!" + answer.save() + + # When + details = self.answerpaper.regrade(self.question2.id) + + # Then + answer = self.answerpaper.answers.filter(question=self.question2)\ + .last() + self.assertTrue(details[0]) + self.assertEqual(answer.marks, 1) + self.assertTrue(answer.correct) + class FloatQuestionTestCases(unittest.TestCase): @classmethod @@ -254,6 +365,7 @@ class FloatQuestionTestCases(unittest.TestCase): self.answerpaper = AnswerPaper.objects.get(question_paper\ =self.question_paper) self.answerpaper.attempt_number = 1 + self.answerpaper.questions.add(self.question1) self.answerpaper.save() # For question self.float_based_testcase = FloatTestCase(question=self.question1, @@ -267,7 +379,7 @@ class FloatQuestionTestCases(unittest.TestCase): def tearDownClass(self): self.question1.delete() - def test_float_correct_answer(self): + def test_validate_regrade_float_correct_answer(self): # Given float_answer = 99.9 self.answer = Answer(question=self.question1, @@ -285,7 +397,25 @@ class FloatQuestionTestCases(unittest.TestCase): # Then self.assertTrue(result['success']) - def test_integer_incorrect_answer(self): + # Regrade + # Given + self.answer.correct = True + self.answer.marks = 1 + + self.answer.answer = 0.0 + self.answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 0) + self.assertFalse(self.answer.correct) + + def test_float_incorrect_answer(self): # Given float_answer = 99.8 self.answer = Answer(question=self.question1, @@ -302,3 +432,21 @@ class FloatQuestionTestCases(unittest.TestCase): # Then self.assertFalse(result['success']) + + # Regrade + # Given + self.answer.correct = True + self.answer.marks = 1 + + self.answer.answer = 99.9 + self.answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 1) + self.assertTrue(self.answer.correct) diff --git a/yaksh/models.py b/yaksh/models.py index 8e7089f..7e97e6c 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -29,9 +29,10 @@ import tempfile from textwrap import dedent from ast import literal_eval from .file_utils import extract_files, delete_files -from yaksh.code_server import(submit, SERVER_POOL_PORT, +from yaksh.code_server import(submit, get_result as get_result_from_code_server ) +from yaksh.settings import SERVER_POOL_PORT from django.conf import settings from django.forms.models import model_to_dict @@ -1385,7 +1386,8 @@ class AnswerPaper(models.Model): def get_previous_answers(self, question): return self.answers.filter(question=question).order_by('-id') - def validate_answer(self, user_answer, question, json_data=None, uid=None): + def validate_answer(self, user_answer, question, json_data=None, uid=None, + server_port=SERVER_POOL_PORT): """ Checks whether the answer submitted by the user is right or wrong. If right then returns correct = True, success and @@ -1447,12 +1449,12 @@ class AnswerPaper(models.Model): elif question.type == 'code' or question.type == "upload": user_dir = self.user.profile.get_user_dir() - url = 'http://localhost:%s' % SERVER_POOL_PORT + url = 'http://localhost:%s' % server_port submit(url, uid, json_data, user_dir) result = {'uid': uid, 'status': 'running'} return result - def regrade(self, question_id): + def regrade(self, question_id, server_port=SERVER_POOL_PORT): try: question = self.questions.get(id=question_id) msg = 'User: {0}; Quiz: {1}; Question: {2}.\n'.format( @@ -1478,9 +1480,11 @@ class AnswerPaper(models.Model): json_data = question.consolidate_answer_data(answer) \ if question.type == 'code' else None result = self.validate_answer(answer, question, - json_data, user_answer.id) + json_data, user_answer.id, + server_port=server_port + ) if question.type == "code": - url = 'http://localhost:%s' % SERVER_POOL_PORT + url = 'http://localhost:%s' % server_port check_result = get_result_from_code_server(url, result['uid'], block=True ) diff --git a/yaksh/test_models.py b/yaksh/test_models.py index ddacb2a..2eea631 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -2,6 +2,9 @@ import unittest from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ StdIOBasedTestCase, FileUpload, McqTestCase, AssignmentUpload +from yaksh.code_server import(ServerPool, + get_result as get_result_from_code_server + ) import json import ruamel.yaml as yaml from datetime import datetime, timedelta @@ -10,10 +13,13 @@ import pytz from django.contrib.auth.models import Group from django.core.files import File from django.forms.models import model_to_dict +from textwrap import dedent import zipfile import os import shutil import tempfile +from threading import Thread +from yaksh import settings def setUpModule(): # create user profile @@ -735,6 +741,20 @@ class AnswerPaperTestCases(unittest.TestCase): self.user2_answerpaper2 = self.question_paper.make_answerpaper( self.user2, self.ip, 1 ) + settings.code_evaluators['python']['standardtestcase'] = \ + "yaksh.python_assertion_evaluator.PythonAssertionEvaluator" + self.SERVER_POOL_PORT = 4000 + server_pool = ServerPool(n=1, pool_port=self.SERVER_POOL_PORT) + self.server_pool = server_pool + self.server_thread = t = Thread(target=server_pool.run) + t.start() + + @classmethod + def tearDownClass(self): + self.server_pool.stop() + self.server_thread.join() + settings.code_evaluators['python']['standardtestcase'] = \ + "python_assertion_evaluator.PythonAssertionEvaluator" def test_get_per_question_score(self): # Given @@ -839,6 +859,65 @@ class AnswerPaperTestCases(unittest.TestCase): self.assertEqual(self.answer.marks, 0) self.assertFalse(self.answer.correct) + def test_validate_and_regrade_code_correct_answer(self): + # Given + # Start code server + + user_answer = dedent("""\ + def add(a,b): + return a+b + """) + self.answer = Answer(question=self.question1, + answer=user_answer, + ) + self.answer.save() + self.answerpaper.answers.add(self.answer) + user = self.answerpaper.user + + # When + json_data = self.question1.consolidate_answer_data(user_answer, + user + ) + get_result = self.answerpaper.validate_answer(user_answer, + self.question1, + json_data, + self.answer.id, + self.SERVER_POOL_PORT + ) + url = 'http://localhost:%s' % self.SERVER_POOL_PORT + check_result = get_result_from_code_server(url,get_result['uid'], + block=True + ) + result = json.loads(check_result.get('result')) + + # Then + self.assertTrue(result['success']) + self.answer.correct = True + self.answer.marks = 1 + + # Regrade + # Given + self.answer.correct = True + self.answer.marks = 1 + + self.answer.answer = dedent(""" + def add(a,b): + return a-b + """) + self.answer.save() + + # When + details = self.answerpaper.regrade(self.question1.id, + self.SERVER_POOL_PORT + ) + + # Then + self.answer = self.answerpaper.answers.filter(question=self.question1 + ).last() + self.assertTrue(details[0]) + self.assertEqual(self.answer.marks, 0) + self.assertFalse(self.answer.correct) + def test_validate_and_regrade_mcq_correct_answer(self): # Given mcq_answer = str(self.mcq_based_testcase.id) |