From 5ec456d5208df78ef87d0660e4bef430d553d65d Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 11 Mar 2016 12:36:49 +0530 Subject: Revert testcase formsets --- yaksh/forms.py | 4 --- yaksh/templates/yaksh/add_question.html | 3 --- yaksh/views.py | 47 ++++++++++++++------------------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/yaksh/forms.py b/yaksh/forms.py index c5bec4c..0eed8eb 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -180,10 +180,6 @@ class QuestionFilterForm(forms.Form): (choices=question_types)) -TestCaseFormSet = inlineformset_factory(Question, TestCase, fields='__all__', - can_order=False, can_delete=False, extra=1) - - class CourseForm(forms.ModelForm): class Meta: model = Course diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index 61b146c..1b1d28d 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -40,9 +40,6 @@ {% endif %} -
- -

diff --git a/yaksh/views.py b/yaksh/views.py index a986d4c..f540351 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -146,14 +146,14 @@ def add_question(request, question_id=None): """To add a new question in the database. Create a new question and store it.""" - def add_or_delete_test_form(post_request, instance): - request_copy = post_request.copy() - if 'add_test' in post_request: - request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) + 1 - elif 'delete_test' in post_request: - request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) - 1 - test_case_formset = TestCaseFormSet(request_copy, prefix='test', instance=instance) - return test_case_formset + # def add_or_delete_test_form(post_request, instance): + # request_copy = post_request.copy() + # if 'add_test' in post_request: + # request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) + 1 + # elif 'delete_test' in post_request: + # request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) - 1 + # test_case_formset = TestCaseFormSet(request_copy, prefix='test', instance=instance) + # return test_case_formset user = request.user ci = RequestContext(request) @@ -163,7 +163,7 @@ def add_question(request, question_id=None): form = QuestionForm(request.POST) if form.is_valid(): if question_id is None: - test_case_formset = add_or_delete_test_form(request.POST, form.save(commit=False)) + # test_case_formset = add_or_delete_test_form(request.POST, form.save(commit=False)) if 'save_question' in request.POST: qtn = form.save(commit=False) qtn.user = user @@ -178,52 +178,45 @@ def add_question(request, question_id=None): {'form': form, 'formset': test_case_formset}, context_instance=ci) - return my_redirect("/exam/manage/questions") return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, + {'form': form}, + # 'formset': test_case_formset}, context_instance=ci) else: d = Question.objects.get(id=question_id) - form = QuestionForm(request.POST, instance=d) - test_case_formset = add_or_delete_test_form(request.POST, d) if 'save_question' in request.POST: qtn = form.save(commit=False) test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=qtn) form.save() question = Question.objects.get(id=question_id) - if test_case_formset.is_valid(): - test_case_formset.save() return my_redirect("/exam/manage/questions") return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, + {'form': form}, + # 'formset': test_case_formset}, context_instance=ci) else: - test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, + {'form': form}, + # 'formset': test_case_formset}, context_instance=ci) else: if question_id is None: form = QuestionForm() - test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) + # test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, + {'form': form}, + # 'formset': test_case_formset}, context_instance=ci) else: d = Question.objects.get(id=question_id) form = QuestionForm(instance=d) - test_case_formset = TestCaseFormSet(prefix='test', instance=d) return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, + {'form': form}, + # 'formset': test_case_formset}, context_instance=ci) -- cgit From 3b56d8259d6aa8311f86e0561e40d0b17bc62534 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 4 Mar 2016 14:55:09 +0530 Subject: Language registry altered, Name cleanup: - set_registry will only act when registry is None - _setup, _teardown and _check_code are setup,teardown,check_code - Corresponding name changes in evaluator modules --- yaksh/code_evaluator.py | 16 ++++++++-------- yaksh/code_server.py | 9 +++++---- yaksh/cpp_code_evaluator.py | 13 ++++++------- yaksh/java_code_evaluator.py | 13 ++++++------- yaksh/language_registry.py | 11 +++++++++-- yaksh/python_code_evaluator.py | 4 ++-- yaksh/scilab_code_evaluator.py | 13 ++++++------- 7 files changed, 42 insertions(+), 37 deletions(-) diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 381b2e8..f877952 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -98,18 +98,18 @@ class CodeEvaluator(object): A tuple: (success, error message). """ - self._setup() - success, err = self._evaluate(self.test_case_args) - self._teardown() + self.setup() + success, err = self.safe_evaluate(self.test_case_args) + self.teardown() result = {'success': success, 'error': err} return result # Private Protocol ########## - def _setup(self): + def setup(self): self._change_dir(self.in_dir) - def _evaluate(self, args): + def safe_evaluate(self, args): # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False @@ -117,7 +117,7 @@ class CodeEvaluator(object): # Do whatever testing needed. try: - success, err = self._check_code(*args) + success, err = self.check_code(*args) except TimeoutException: err = self.timeout_msg @@ -130,11 +130,11 @@ class CodeEvaluator(object): return success, err - def _teardown(self): + def teardown(self): # Cancel the signal delete_signal_handler() - def _check_code(self): + def check_code(self): raise NotImplementedError("check_code method not implemented") def create_submit_code_file(self, file_name): diff --git a/yaksh/code_server.py b/yaksh/code_server.py index faf9c0d..2762f12 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -31,7 +31,7 @@ import re import json # Local imports. from settings import SERVER_PORTS, SERVER_POOL_PORT -from language_registry import set_registry, get_registry +from language_registry import get_registry, registry MY_DIR = abspath(dirname(__file__)) @@ -82,9 +82,10 @@ class CodeServer(object): # Private Protocol ########## def _create_evaluator_instance(self, language, json_data, in_dir): """Create instance of relevant EvaluateCode class based on language""" - set_registry() - registry = get_registry() - cls = registry.get_class(language) + # set_registry() + registry1 = get_registry() + print registry + cls = registry1.get_class(language) instance = cls.from_json(language, json_data, in_dir) return instance diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 7242884..129a1c8 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -17,11 +17,10 @@ class CppCodeEvaluator(CodeEvaluator): super(CppCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, ref_code_path, in_dir) - self.test_case_args = self._setup() + self.test_case_args = self.setup() - # Private Protocol ########## - def _setup(self): - super(CppCodeEvaluator, self)._setup() + def setup(self): + super(CppCodeEvaluator, self).setup() get_ref_path = self.ref_code_path ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) @@ -44,12 +43,12 @@ class CppCodeEvaluator(CodeEvaluator): return (ref_path, self.submit_path, compile_command, compile_main, run_command_args, remove_user_output, remove_ref_output) - def _teardown(self): + def teardown(self): # Delete the created file. - super(CppCodeEvaluator, self)._teardown() + super(CppCodeEvaluator, self).teardown() os.remove(self.submit_path) - def _check_code(self, ref_code_path, submit_code_path, compile_command, + def check_code(self, ref_code_path, submit_code_path, compile_command, compile_main, run_command_args, remove_user_output, remove_ref_output): """ Function validates student code using instructor code as diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index 4367259..6f39d71 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -17,11 +17,10 @@ class JavaCodeEvaluator(CodeEvaluator): super(JavaCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, ref_code_path, in_dir) - self.test_case_args = self._setup() + self.test_case_args = self.setup() - # Private Protocol ########## - def _setup(self): - super(JavaCodeEvaluator, self)._setup() + def setup(self): + super(JavaCodeEvaluator, self).setup() ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) self.submit_path = self.create_submit_code_file('Test.java') @@ -46,12 +45,12 @@ class JavaCodeEvaluator(CodeEvaluator): return (ref_path, self.submit_path, compile_command, compile_main, run_command_args, remove_user_output, remove_ref_output) - def _teardown(self): + def teardown(self): # Delete the created file. - super(JavaCodeEvaluator, self)._teardown() + super(JavaCodeEvaluator, self).teardown() os.remove(self.submit_path) - def _check_code(self, ref_code_path, submit_code_path, compile_command, + def check_code(self, ref_code_path, submit_code_path, compile_command, compile_main, run_command_args, remove_user_output, remove_ref_output): """ Function validates student code using instructor code as diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 76a23d7..ee311ec 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -3,11 +3,18 @@ import importlib registry = None -def set_registry(): +# def set_registry(): +# global registry +# registry = _LanguageRegistry() + +def _set_registry(): global registry - registry = _LanguageRegistry() + if registry is None: + registry = _LanguageRegistry() + return registry def get_registry(): + registry = _set_registry() return registry class _LanguageRegistry(object): diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py index c87c420..3835b44 100644 --- a/yaksh/python_code_evaluator.py +++ b/yaksh/python_code_evaluator.py @@ -11,8 +11,8 @@ from code_evaluator import CodeEvaluator, TimeoutException class PythonCodeEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - # Private Protocol ########## - def _check_code(self): + + def check_code(self): success = False try: diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 392cd45..a8bd4cd 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -20,23 +20,22 @@ class ScilabCodeEvaluator(CodeEvaluator): # Removes all the commands that terminates scilab self.user_answer, self.terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - self.test_case_args = self._setup() + self.test_case_args = self.setup() - # Private Protocol ########## - def _setup(self): - super(ScilabCodeEvaluator, self)._setup() + def setup(self): + super(ScilabCodeEvaluator, self).setup() ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) self.submit_path = self.create_submit_code_file('function.sci') return ref_path, # Return as a tuple - def _teardown(self): + def teardown(self): # Delete the created file. - super(ScilabCodeEvaluator, self)._teardown() + super(ScilabCodeEvaluator, self).teardown() os.remove(self.submit_path) - def _check_code(self, ref_path): + def check_code(self, ref_path): success = False # Throw message if there are commmands that terminates scilab -- cgit From ceb4f2cbc1a03835a3c7e34d806ec21e47e3f059 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 4 Mar 2016 21:54:59 +0530 Subject: add test case selection --- yaksh/forms.py | 6 ++++++ yaksh/models.py | 39 +++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/yaksh/forms.py b/yaksh/forms.py index 0eed8eb..5c8dafa 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -28,6 +28,12 @@ question_types = ( ("upload", "Assignment Upload"), ) +test_case_types = ( + ("assert_based", "Assertion Based Testcase"), + # ("argument_based", "Multiple Correct Choices"), + # ("stdout_based", "Code"), + ) + UNAME_CHARS = letters + "._" + digits PWD_CHARS = letters + punctuation + digits diff --git a/yaksh/models.py b/yaksh/models.py index 6e59d7a..c4f3561 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -8,21 +8,6 @@ from django.contrib.auth.models import User from taggit.managers import TaggableManager -############################################################################### -class ConcurrentUser(models.Model): - concurrent_user = models.OneToOneField(User, null=False) - session_key = models.CharField(null=False, max_length=40) - - -############################################################################### -class Profile(models.Model): - """Profile for a user to store roll number and other details.""" - user = models.OneToOneField(User) - roll_number = models.CharField(max_length=20) - institute = models.CharField(max_length=128) - department = models.CharField(max_length=64) - position = models.CharField(max_length=64) - languages = ( ("python", "Python"), ("bash", "Bash"), @@ -45,6 +30,12 @@ enrollment_methods = ( ("open", "Open Course"), ) +test_case_types = ( + ("assert_based", "Assertion Based Testcase"), + # ("argument_based", "Multiple Correct Choices"), + # ("stdout_based", "Code"), + ) + attempts = [(i, i) for i in range(1, 6)] attempts.append((-1, 'Infinite')) days_between_attempts = ((j, j) for j in range(401)) @@ -54,7 +45,6 @@ test_status = ( ('completed', 'Completed'), ) - def get_assignment_dir(instance, filename): return '%s/%s' % (instance.user.roll_number, instance.assignmentQuestion.id) @@ -132,6 +122,21 @@ class Course(models.Model): def __unicode__(self): return self.name +############################################################################### +class ConcurrentUser(models.Model): + concurrent_user = models.OneToOneField(User, null=False) + session_key = models.CharField(null=False, max_length=40) + + +############################################################################### +class Profile(models.Model): + """Profile for a user to store roll number and other details.""" + user = models.OneToOneField(User) + roll_number = models.CharField(max_length=20) + institute = models.CharField(max_length=128) + department = models.CharField(max_length=64) + position = models.CharField(max_length=64) + ############################################################################### class Question(models.Model): @@ -163,6 +168,8 @@ class Question(models.Model): # The type of question. type = models.CharField(max_length=24, choices=question_types) + # The type of evaluator + test_case_type = models.CharField(max_length=24, choices=test_case_types) # Is this question active or not. If it is inactive it will not be used # when creating a QuestionPaper. active = models.BooleanField(default=True) -- cgit From 1e993bee18028c59d809f49d853b60e41326991c Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 11 Mar 2016 12:11:49 +0530 Subject: Add a python standard out evaluator --- yaksh/code_evaluator.py | 63 ++++++++++------- yaksh/code_server.py | 26 +++---- yaksh/forms.py | 2 +- yaksh/language_registry.py | 27 ++++--- yaksh/models.py | 44 ++++++------ yaksh/python_code_evaluator.py | 74 ++++++++++++------- yaksh/python_stdout_evaluator.py | 47 +++++++++++++ yaksh/settings.py | 5 +- yaksh/templates/yaksh/add_question.html | 1 + yaksh/tester/python/verifier.py | 121 ++++++++++++++++++++++++++++++++ yaksh/views.py | 2 +- yaksh/xmlrpc_clients.py | 4 +- 12 files changed, 317 insertions(+), 99 deletions(-) create mode 100644 yaksh/python_stdout_evaluator.py create mode 100644 yaksh/tester/python/verifier.py diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index f877952..7e2a729 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -8,7 +8,7 @@ import signal from multiprocessing import Process, Queue import subprocess import re -import json +# import json # Local imports. from settings import SERVER_TIMEOUT @@ -50,33 +50,39 @@ def delete_signal_handler(): class CodeEvaluator(object): """Tests the code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + def __init__(self, in_dir, **kwargs): + + msg = 'Code took more than %s seconds to run. You probably '\ 'have an infinite loop in your code.' % SERVER_TIMEOUT self.timeout_msg = msg - self.test_case_data = test_case_data - self.language = language.lower() - self.user_answer = user_answer - self.ref_code_path = ref_code_path - self.test = test - self.in_dir = in_dir - self.test_case_args = None + # self.test_case_data = test_case_data + # self.language = language.lower() #@@@remove + # self.user_answer = user_answer #@@@specific to check-code + # self.ref_code_path = ref_code_path #@@@specific to check-code + # self.test = test #@@@specific to check-code + self.in_dir = in_dir #@@@Common for all, no change + self.test_case_args = None #@@@no change # Public Protocol ########## - @classmethod - def from_json(cls, language, json_data, in_dir): - json_data = json.loads(json_data) - test_case_data = json_data.get("test_case_data") - user_answer = json_data.get("user_answer") - ref_code_path = json_data.get("ref_code_path") - test = json_data.get("test") - - instance = cls(test_case_data, test, language, user_answer, ref_code_path, - in_dir) - return instance - - def evaluate(self): + # @classmethod + # def from_json(cls, language, json_data, in_dir): + # json_data = json.loads(json_data) + # # test_case_data = json_data.get("test_case_data") + # user_answer = json_data.get("user_answer") + # ref_code_path = json_data.get("ref_code_path") + # test = json_data.get("test") + + # # instance = cls(test_case_data, test, language, user_answer, ref_code_path, + # # in_dir) + # instance = cls(test, language, user_answer, ref_code_path, + # in_dir) + # return instance + + # def evaluate(self): + def evaluate(self, **kwargs): """Evaluates given code with the test cases based on given arguments in test_case_data. @@ -99,7 +105,8 @@ class CodeEvaluator(object): """ self.setup() - success, err = self.safe_evaluate(self.test_case_args) + # success, err = self.safe_evaluate(self.test_case_args) + success, err = self.safe_evaluate(**kwargs) self.teardown() result = {'success': success, 'error': err} @@ -109,15 +116,17 @@ class CodeEvaluator(object): def setup(self): self._change_dir(self.in_dir) - def safe_evaluate(self, args): + # def safe_evaluate(self, args): + def safe_evaluate(self, **kwargs): # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False - args = args or [] + # args = args or [] # Do whatever testing needed. try: - success, err = self.check_code(*args) + # success, err = self.check_code(*args) + success, err = self.check_code(**kwargs) except TimeoutException: err = self.timeout_msg diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 2762f12..7951ac8 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -31,7 +31,7 @@ import re import json # Local imports. from settings import SERVER_PORTS, SERVER_POOL_PORT -from language_registry import get_registry, registry +from language_registry import get_registry, create_evaluator_instance, unpack_json MY_DIR = abspath(dirname(__file__)) @@ -58,13 +58,14 @@ class CodeServer(object): self.queue = queue # Public Protocol ########## - def check_code(self, language, json_data, in_dir=None): + def check_code(self, language, test_case_type, json_data, in_dir=None): """Calls relevant EvaluateCode class based on language to check the answer code """ - code_evaluator = self._create_evaluator_instance(language, json_data, + code_evaluator = create_evaluator_instance(language, test_case_type, json_data, in_dir) - result = code_evaluator.evaluate() + data = unpack_json(json_data) #@@@ def should be here + result = code_evaluator.evaluate(**data) # Put us back into the server pool queue since we are free now. self.queue.put(self.port) @@ -79,15 +80,14 @@ class CodeServer(object): self.queue.put(self.port) server.serve_forever() - # Private Protocol ########## - def _create_evaluator_instance(self, language, json_data, in_dir): - """Create instance of relevant EvaluateCode class based on language""" - # set_registry() - registry1 = get_registry() - print registry - cls = registry1.get_class(language) - instance = cls.from_json(language, json_data, in_dir) - return instance + # # Private Protocol ########## + # def _create_evaluator_instance(self, language, json_data, in_dir): + # """Create instance of relevant EvaluateCode class based on language""" + # # set_registry() + # registry = get_registry() + # cls = registry.get_class(language) + # instance = cls.from_json(language, json_data, in_dir) + # return instance ############################################################################### diff --git a/yaksh/forms.py b/yaksh/forms.py index 5c8dafa..5959dc4 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -31,7 +31,7 @@ question_types = ( test_case_types = ( ("assert_based", "Assertion Based Testcase"), # ("argument_based", "Multiple Correct Choices"), - # ("stdout_based", "Code"), + ("stdout_based", "Stdout Based Testcase"), ) UNAME_CHARS = letters + "._" + digits diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index ee311ec..512e2f5 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -1,21 +1,31 @@ from settings import code_evaluators import importlib +import json registry = None # def set_registry(): # global registry # registry = _LanguageRegistry() - -def _set_registry(): + +def get_registry(): #@@@get_evaluator_registry global registry if registry is None: registry = _LanguageRegistry() return registry - -def get_registry(): - registry = _set_registry() - return registry + +def unpack_json(json_data): + data = json.loads(json_data) + return data + +def create_evaluator_instance(language, test_case_type, json_data, in_dir): + """Create instance of relevant EvaluateCode class based on language""" + # set_registry() + registry = get_registry() + cls = registry.get_class(language, test_case_type) #@@@get_evaluator_for_language + instance = cls(in_dir) + # instance = cls.from_json(language, json_data, in_dir) + return instance class _LanguageRegistry(object): def __init__(self): @@ -24,12 +34,13 @@ class _LanguageRegistry(object): self._register[language] = None # Public Protocol ########## - def get_class(self, language): + def get_class(self, language, test_case_type): """ Get the code evaluator class for the given language """ if not self._register.get(language): self._register[language] = code_evaluators.get(language) - cls = self._register[language] + test_case_register = self._register[language] + cls = test_case_register.get(test_case_type) module_name, class_name = cls.rsplit(".", 1) # load the module, will raise ImportError if module cannot be loaded get_module = importlib.import_module(module_name) diff --git a/yaksh/models.py b/yaksh/models.py index c4f3561..6fa96bf 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -33,7 +33,7 @@ enrollment_methods = ( test_case_types = ( ("assert_based", "Assertion Based Testcase"), # ("argument_based", "Multiple Correct Choices"), - # ("stdout_based", "Code"), + ("stdout_based", "Stdout Based Testcase"), ) attempts = [(i, i) for i in range(1, 6)] @@ -170,6 +170,7 @@ class Question(models.Model): # The type of evaluator test_case_type = models.CharField(max_length=24, choices=test_case_types) + # Is this question active or not. If it is inactive it will not be used # when creating a QuestionPaper. active = models.BooleanField(default=True) @@ -183,38 +184,39 @@ class Question(models.Model): # user for particular question user = models.ForeignKey(User, related_name="user") - def consolidate_answer_data(self, test_cases, user_answer): + def consolidate_answer_data(self, user_answer): test_case_data_dict = [] question_info_dict = {} - for test_case in test_cases: - kw_args_dict = {} - pos_args_list = [] + # for test_case in test_cases: + # kw_args_dict = {} + # pos_args_list = [] - test_case_data = {} - test_case_data['test_id'] = test_case.id - test_case_data['func_name'] = test_case.func_name - test_case_data['expected_answer'] = test_case.expected_answer + # test_case_data = {} + # test_case_data['test_id'] = test_case.id + # test_case_data['func_name'] = test_case.func_name + # test_case_data['expected_answer'] = test_case.expected_answer - if test_case.kw_args: - for args in test_case.kw_args.split(","): - arg_name, arg_value = args.split("=") - kw_args_dict[arg_name.strip()] = arg_value.strip() + # if test_case.kw_args: + # for args in test_case.kw_args.split(","): + # arg_name, arg_value = args.split("=") + # kw_args_dict[arg_name.strip()] = arg_value.strip() - if test_case.pos_args: - for args in test_case.pos_args.split(","): - pos_args_list.append(args.strip()) + # if test_case.pos_args: + # for args in test_case.pos_args.split(","): + # pos_args_list.append(args.strip()) - test_case_data['kw_args'] = kw_args_dict - test_case_data['pos_args'] = pos_args_list - test_case_data_dict.append(test_case_data) + # test_case_data['kw_args'] = kw_args_dict + # test_case_data['pos_args'] = pos_args_list + # test_case_data_dict.append(test_case_data) # question_info_dict['language'] = self.language - question_info_dict['id'] = self.id + # question_info_dict['id'] = self.id question_info_dict['user_answer'] = user_answer - question_info_dict['test_parameter'] = test_case_data_dict + # question_info_dict['test_parameter'] = test_case_data_dict question_info_dict['ref_code_path'] = self.ref_code_path question_info_dict['test'] = self.test + # question_info_dict['test_case_type'] = self.test_case_type return json.dumps(question_info_dict) diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py index 3835b44..5722b2d 100644 --- a/yaksh/python_code_evaluator.py +++ b/yaksh/python_code_evaluator.py @@ -12,13 +12,13 @@ from code_evaluator import CodeEvaluator, TimeoutException class PythonCodeEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self): + def check_code(self, test, user_answer, ref_code_path): success = False try: tb = None - test_code = self._create_test_case() - submitted = compile(self.user_answer, '', mode='exec') + test_code = test + submitted = compile(user_answer, '', mode='exec') g = {} exec submitted in g _tests = compile(test_code, '', mode='exec') @@ -40,26 +40,50 @@ class PythonCodeEvaluator(CodeEvaluator): del tb return success, err - def _create_test_case(self): - """ - Create assert based test cases in python - """ - test_code = "" - if self.test: - return self.test - elif self.test_case_data: - for test_case in self.test_case_data: - pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \ - if test_case.get('pos_args') else "" - kw_args = ", ".join(str(k+"="+a) for k, a - in test_case.get('kw_args').iteritems()) \ - if test_case.get('kw_args') else "" - args = pos_args + ", " + kw_args if pos_args and kw_args \ - else pos_args or kw_args - function_name = test_case.get('func_name') - expected_answer = test_case.get('expected_answer') + # def check_code(self): + # success = False - tcode = "assert {0}({1}) == {2}".format(function_name, args, - expected_answer) - test_code += tcode + "\n" - return test_code + # try: + # tb = None + # test_code = self._create_test_case() + # submitted = compile(self.user_answer, '', mode='exec') + # g = {} + # exec submitted in g + # _tests = compile(test_code, '', mode='exec') + # exec _tests in g + # except AssertionError: + # type, value, tb = sys.exc_info() + # info = traceback.extract_tb(tb) + # fname, lineno, func, text = info[-1] + # text = str(test_code).splitlines()[lineno-1] + # err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + # else: + # success = True + # err = 'Correct answer' + + # del tb + # return success, err + + # def _create_test_case(self): + # """ + # Create assert based test cases in python + # """ + # test_code = "" + # if self.test: + # return self.test + # elif self.test_case_data: + # for test_case in self.test_case_data: + # pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \ + # if test_case.get('pos_args') else "" + # kw_args = ", ".join(str(k+"="+a) for k, a + # in test_case.get('kw_args').iteritems()) \ + # if test_case.get('kw_args') else "" + # args = pos_args + ", " + kw_args if pos_args and kw_args \ + # else pos_args or kw_args + # function_name = test_case.get('func_name') + # expected_answer = test_case.get('expected_answer') + + # tcode = "assert {0}({1}) == {2}".format(function_name, args, + # expected_answer) + # test_code += tcode + "\n" + # return test_code diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py new file mode 100644 index 0000000..89d3424 --- /dev/null +++ b/yaksh/python_stdout_evaluator.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib +from contextlib import contextmanager + +# local imports +from code_evaluator import CodeEvaluator + + +@contextmanager +def redirect_stdout(): + from StringIO import StringIO + new_target = StringIO() + + old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout + try: + yield new_target # run some code with the replaced stdout + finally: + sys.stdout = old_target # restore to the previous value + + +class PythonStdoutEvaluator(CodeEvaluator): + """Tests the Python code obtained from Code Server""" + + def check_code(self, test, user_answer, ref_code_path): + success = False + + try: + tb = None + test_code = test + submitted = compile(user_answer, '', mode='exec') + with redirect_stdout() as output_buffer: + g = {} + exec submitted in g + raw_output_value = output_buffer.getvalue() + output_value = raw_output_value.encode('string_escape').strip() + if output_value == str(test_code): + success = True + err = 'Correct answer' + else: + raise ValueError("Incorrect Answer") + + del tb + return success, err \ No newline at end of file diff --git a/yaksh/settings.py b/yaksh/settings.py index 63bd875..f8b240d 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,7 +20,10 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": "python_code_evaluator.PythonCodeEvaluator", + "python": {"assert_based": "python_code_evaluator.PythonCodeEvaluator", + "argument_based": "python_argument_based_evaluator.PythonCodeEvaluator", + "stdout_based": "python_stdout_evaluator.PythonStdoutEvaluator" + }, "c": "cpp_code_evaluator.CppCodeEvaluator", "cpp": "cpp_code_evaluator.CppCodeEvaluator", "java": "java_code_evaluator.JavaCodeEvaluator", diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index 1b1d28d..88d8f03 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -30,6 +30,7 @@ Options: {{ form.options }} {{form.options.errors}} Test: {{ form.test }} {{form.test.errors}} Reference Code Path: {{ form.ref_code_path }} {{form.ref_code_path.errors}} + test_case_type: {{ form.test_case_type }}{{ form.test_case_type.errors }}
{% if formset%} diff --git a/yaksh/tester/python/verifier.py b/yaksh/tester/python/verifier.py new file mode 100644 index 0000000..102dcb9 --- /dev/null +++ b/yaksh/tester/python/verifier.py @@ -0,0 +1,121 @@ +import sys +from .utils import import_by_path +from contextlib import contextmanager + + +@contextmanager +def redirect_stdout(): + from StringIO import StringIO + new_target = StringIO() + + old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout + try: + yield new_target # run some code with the replaced stdout + finally: + sys.stdout = old_target # restore to the previous value + +# def redirect_stdout(): +# # import sys +# from StringIO import StringIO +# oldout,olderr = sys.stdout, sys.stderr +# try: +# out = StringIO() +# err = StringIO() +# # sys.stdout,sys.stderr = out, err +# yield out, err +# finally: +# sys.stdout,sys.stderr = oldout, olderr +# out = out.getvalue() +# err = err.getvalue() + +TESTER_BACKEND = { + "python": "PythonPrintTesterBackend" #@@@rename to test-case-creator, this file should be backend.py +} + +class TesterException(Exception): + """ Parental class for all tester exceptions """ + pass + +class UnknownBackendException(TesterException): + """ Exception thrown if tester backend is not recognized. """ + pass + + +def detect_backend(language): + """ + Detect the right backend for a test case. + """ + backend_name = TESTER_BACKEND.get(language) + # backend = import_by_path(backend_name) + backend = PythonTesterBackend() #@@@ + return backend + +class PythonPrintTesterBackend(object): + def test_code(self, submitted, reference_output): + """ + create a test command + """ + with redirect_stdout() as output_buffer: + g = {} + exec submitted in g + + # return_buffer = out.encode('string_escape') + raw_output_value = output_buffer.getvalue() + output_value = raw_output_value.encode('string_escape').strip() + if output_value == str(reference_output): + return True + else: + raise ValueError("Incorrect Answer", output_value, reference_output) + + +class PythonTesterBackend(object): + # def __init__(self, test_case): + # self._test_case = test_case + def create(self): #@@@ test() + """ + create a test command + """ + test_code = "assert {0}({1}) == {2}".format(self.test_case_parameters['function_name'], self.test_case_parameters['args'], + self.test_case_parameters['expected_answer']) + return test_code + + def pack(self, test_case): + kw_args_dict = {} + pos_args_list = [] + test_case_data = {} + test_case_data['test_id'] = test_case.id + test_case_data['func_name'] = test_case.func_name + test_case_data['expected_answer'] = test_case.expected_answer + + if test_case.kw_args: + for args in test_case.kw_args.split(","): + arg_name, arg_value = args.split("=") + kw_args_dict[arg_name.strip()] = arg_value.strip() + + if test_case.pos_args: + for args in test_case.pos_args.split(","): + pos_args_list.append(args.strip()) + + test_case_data['kw_args'] = kw_args_dict + test_case_data['pos_args'] = pos_args_list + + return test_case_data + + def unpack(self, test_case_data): + pos_args = ", ".join(str(i) for i in test_case_data.get('pos_args')) \ + if test_case_data.get('pos_args') else "" + kw_args = ", ".join(str(k+"="+a) for k, a + in test_case_data.get('kw_args').iteritems()) \ + if test_case_data.get('kw_args') else "" + args = pos_args + ", " + kw_args if pos_args and kw_args \ + else pos_args or kw_args + function_name = test_case_data.get('func_name') + expected_answer = test_case_data.get('expected_answer') + + self.test_case_parameters = { + 'args': args, + 'function_name': function_name, + 'expected_answer': expected_answer + } + + return self.test_case_parameters \ No newline at end of file diff --git a/yaksh/views.py b/yaksh/views.py index f540351..520f396 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -513,7 +513,7 @@ def validate_answer(user, user_answer, question, json_data=None): message = 'Correct answer' elif question.type == 'code': user_dir = get_user_dir(user) - json_result = code_server.run_code(question.language, json_data, user_dir) + json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir) result = json.loads(json_result) if result.get('success'): correct = True diff --git a/yaksh/xmlrpc_clients.py b/yaksh/xmlrpc_clients.py index 3a3c0c6..7124550 100644 --- a/yaksh/xmlrpc_clients.py +++ b/yaksh/xmlrpc_clients.py @@ -23,7 +23,7 @@ class CodeServerProxy(object): pool_url = 'http://localhost:%d' % (SERVER_POOL_PORT) self.pool_server = ServerProxy(pool_url) - def run_code(self, language, json_data, user_dir): + def run_code(self, language, test_case_type, json_data, user_dir): """Tests given code (`answer`) with the `test_code` supplied. If the optional `in_dir` keyword argument is supplied it changes the directory to that directory (it does not change it back to the original when @@ -51,7 +51,7 @@ class CodeServerProxy(object): try: server = self._get_server() - result = server.check_code(language, json_data, user_dir) + result = server.check_code(language, test_case_type, json_data, user_dir) except ConnectionError: result = json.dumps({'success': False, 'error': 'Unable to connect to any code servers!'}) return result -- cgit From 195aead9b0fab0d8cdb86a9fc884ac3edca5db84 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 16 Mar 2016 10:57:07 +0530 Subject: - Connect test case type models to backend code server - Support for Stdout test case and Standard assertion test case - Add MCQ Test case and support for validations - Remove tester dir --- yaksh/admin.py | 5 +- yaksh/code_server.py | 2 +- yaksh/forms.py | 4 +- yaksh/models.py | 115 ++++++++++++++++++------------ yaksh/python_code_evaluator.py | 17 +++-- yaksh/python_stdout_evaluator.py | 52 ++++++++------ yaksh/settings.py | 9 ++- yaksh/templates/yaksh/add_question.html | 2 +- yaksh/tester/python/verifier.py | 121 -------------------------------- yaksh/views.py | 8 +-- 10 files changed, 127 insertions(+), 208 deletions(-) delete mode 100644 yaksh/tester/python/verifier.py diff --git a/yaksh/admin.py b/yaksh/admin.py index 71dfb3b..d223cd4 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -1,6 +1,9 @@ -from yaksh.models import Question, Quiz, TestCase +from yaksh.models import Question, Quiz, TestCase,\ + StandardTestCase, StdoutBasedTestCase from django.contrib import admin admin.site.register(Question) admin.site.register(TestCase) +admin.site.register(StandardTestCase) +admin.site.register(StdoutBasedTestCase) admin.site.register(Quiz) diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 7951ac8..66c4271 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -64,7 +64,7 @@ class CodeServer(object): """ code_evaluator = create_evaluator_instance(language, test_case_type, json_data, in_dir) - data = unpack_json(json_data) #@@@ def should be here + data = unpack_json(json_data) result = code_evaluator.evaluate(**data) # Put us back into the server pool queue since we are free now. diff --git a/yaksh/forms.py b/yaksh/forms.py index 5959dc4..94498a1 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -29,9 +29,9 @@ question_types = ( ) test_case_types = ( - ("assert_based", "Assertion Based Testcase"), + ("standardtestcase", "Standard Testcase"), # ("argument_based", "Multiple Correct Choices"), - ("stdout_based", "Stdout Based Testcase"), + ("stdoutbasedtestcase", "Stdout Based Testcase"), ) UNAME_CHARS = letters + "._" + digits diff --git a/yaksh/models.py b/yaksh/models.py index 6fa96bf..331446f 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -5,6 +5,7 @@ from itertools import islice, cycle from collections import Counter from django.db import models from django.contrib.auth.models import User +from django.forms.models import model_to_dict from taggit.managers import TaggableManager @@ -31,9 +32,9 @@ enrollment_methods = ( ) test_case_types = ( - ("assert_based", "Assertion Based Testcase"), - # ("argument_based", "Multiple Correct Choices"), - ("stdout_based", "Stdout Based Testcase"), + ("standardtestcase", "Standard Testcase"), + ("stdoutbasedtestcase", "Stdout Based Testcase"), + # ("mcqtestcase", "MCQ Testcase"), ) attempts = [(i, i) for i in range(1, 6)] @@ -152,14 +153,14 @@ class Question(models.Model): points = models.FloatField(default=1.0) # Answer for MCQs. - test = models.TextField(blank=True) + # test = models.TextField(blank=True) # Test cases file paths (comma seperated for reference code path and test case code path) # Applicable for CPP, C, Java and Scilab - ref_code_path = models.TextField(blank=True) + # ref_code_path = models.TextField(blank=True) - # Any multiple choice options. Place one option per line. - options = models.TextField(blank=True) + # # Any multiple choice options. Place one option per line. + # options = models.TextField(blank=True) # The language for question. language = models.CharField(max_length=24, @@ -176,7 +177,7 @@ class Question(models.Model): active = models.BooleanField(default=True) # Snippet of code provided to the user. - snippet = models.CharField(max_length=256, blank=True) + # snippet = models.CharField(max_length=256) # Tags for the Question. tags = TaggableManager(blank=True) @@ -184,41 +185,57 @@ class Question(models.Model): # user for particular question user = models.ForeignKey(User, related_name="user") - def consolidate_answer_data(self, user_answer): - test_case_data_dict = [] - question_info_dict = {} + # def consolidate_answer_data(self, test_cases, user_answer): + # def consolidate_answer_data(self, user_answer): + # test_case_data_dict = [] + # question_info_dict = {} + + # for test_case in test_cases: + # kw_args_dict = {} + # pos_args_list = [] + + # test_case_data = {} + # test_case_data['test_id'] = test_case.id + # test_case_data['func_name'] = test_case.func_name + # test_case_data['expected_answer'] = test_case.expected_answer + + # if test_case.kw_args: + # for args in test_case.kw_args.split(","): + # arg_name, arg_value = args.split("=") + # kw_args_dict[arg_name.strip()] = arg_value.strip() + + # if test_case.pos_args: + # for args in test_case.pos_args.split(","): + # pos_args_list.append(args.strip()) - # for test_case in test_cases: - # kw_args_dict = {} - # pos_args_list = [] + # test_case_data['kw_args'] = kw_args_dict + # test_case_data['pos_args'] = pos_args_list + # test_case_data_dict.append(test_case_data) - # test_case_data = {} - # test_case_data['test_id'] = test_case.id - # test_case_data['func_name'] = test_case.func_name - # test_case_data['expected_answer'] = test_case.expected_answer + # question_info_dict['language'] = self.language + # question_info_dict['id'] = self.id + # question_info_dict['user_answer'] = user_answer + # question_info_dict['test_parameter'] = test_case_data_dict + # question_info_dict['ref_code_path'] = self.ref_code_path + # question_info_dict['test'] = self.test + # question_info_dict['test_case_type'] = self.test_case_type - # if test_case.kw_args: - # for args in test_case.kw_args.split(","): - # arg_name, arg_value = args.split("=") - # kw_args_dict[arg_name.strip()] = arg_value.strip() + # return json.dumps(question_info_dict) - # if test_case.pos_args: - # for args in test_case.pos_args.split(","): - # pos_args_list.append(args.strip()) + def consolidate_answer_data(self, user_answer): + question_data = {} + test_case_data = [] - # test_case_data['kw_args'] = kw_args_dict - # test_case_data['pos_args'] = pos_args_list - # test_case_data_dict.append(test_case_data) + test_cases = self.testcase_set.all() + for test in test_cases: + test_instance = test.get_child_instance(self.test_case_type) + test_case_field_value = test_instance.get_field_value() + test_case_data.append(test_case_field_value) - # question_info_dict['language'] = self.language - # question_info_dict['id'] = self.id - question_info_dict['user_answer'] = user_answer - # question_info_dict['test_parameter'] = test_case_data_dict - question_info_dict['ref_code_path'] = self.ref_code_path - question_info_dict['test'] = self.test - # question_info_dict['test_case_type'] = self.test_case_type + question_data['test_case_data'] = test_case_data + question_data['user_answer'] = user_answer - return json.dumps(question_info_dict) + return json.dumps(question_data) def dump_into_json(self, question_ids, user): questions = Question.objects.filter(id__in = question_ids, user_id = user.id) @@ -755,14 +772,24 @@ class AssignmentUpload(models.Model): class TestCase(models.Model): question = models.ForeignKey(Question, blank=True, null = True) - # Test case function name - func_name = models.CharField(blank=True, null = True, max_length=200) + def get_child_instance(self, type): + return getattr(self, type) + +class StandardTestCase(TestCase): + test_case = models.TextField(blank=True) + + def get_field_value(self): + return self.test_case - # Test case Keyword arguments in dict form - kw_args = models.TextField(blank=True, null = True) +class StdoutBasedTestCase(TestCase): + output = models.TextField(blank=True) - # Test case Positional arguments in list form - pos_args = models.TextField(blank=True, null = True) + def get_field_value(self): + return self.output + +class McqTestCase(TestCase): + options = models.TextField() + correct = models.BooleanField(default=False) - # Test case Expected answer in list form - expected_answer = models.TextField(blank=True, null = True) + def validate(self, user_answer): + pass diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py index 5722b2d..a131a0e 100644 --- a/yaksh/python_code_evaluator.py +++ b/yaksh/python_code_evaluator.py @@ -12,17 +12,18 @@ from code_evaluator import CodeEvaluator, TimeoutException class PythonCodeEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self, test, user_answer, ref_code_path): + # def check_code(self, test, user_answer, ref_code_path): + def check_code(self, user_answer, test_cases): success = False try: tb = None - test_code = test submitted = compile(user_answer, '', mode='exec') g = {} exec submitted in g - _tests = compile(test_code, '', mode='exec') - exec _tests in g + for test_code in test_cases: + _tests = compile(test_code, '', mode='exec') + exec _tests in g except AssertionError: type, value, tb = sys.exc_info() info = traceback.extract_tb(tb) @@ -40,6 +41,14 @@ class PythonCodeEvaluator(CodeEvaluator): del tb return success, err + # def unpack_test_case_data(self, test_case_data): + # test_cases = [] + # for t in test_case_data: + # test_case = t.get('test_case') + # test_cases.append(test_case) + + # return test_cases + # def check_code(self): # success = False diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 89d3424..28c3372 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -12,36 +12,42 @@ from code_evaluator import CodeEvaluator @contextmanager def redirect_stdout(): - from StringIO import StringIO - new_target = StringIO() + from StringIO import StringIO + new_target = StringIO() - old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout - try: - yield new_target # run some code with the replaced stdout - finally: - sys.stdout = old_target # restore to the previous value + old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout + try: + yield new_target # run some code with the replaced stdout + finally: + sys.stdout = old_target # restore to the previous value class PythonStdoutEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self, test, user_answer, ref_code_path): + def check_code(self, user_answer, test_cases): success = False - try: - tb = None - test_code = test - submitted = compile(user_answer, '', mode='exec') - with redirect_stdout() as output_buffer: - g = {} - exec submitted in g - raw_output_value = output_buffer.getvalue() - output_value = raw_output_value.encode('string_escape').strip() - if output_value == str(test_code): - success = True - err = 'Correct answer' - else: - raise ValueError("Incorrect Answer") + tb = None + expected_output = test_cases[0] + submitted = compile(user_answer, '', mode='exec') + with redirect_stdout() as output_buffer: + g = {} + exec submitted in g + raw_output_value = output_buffer.getvalue() + output_value = raw_output_value.encode('string_escape').strip() + if output_value == str(test_code): + success = True + err = 'Correct answer' + else: + success = False + err = "Incorrect Answer" del tb - return success, err \ No newline at end of file + return success, err + + # def unpack_test_case_data(self, test_case_data): + # for t in test_case_data: + # test_case = t.get('output') + + # return test_case diff --git a/yaksh/settings.py b/yaksh/settings.py index f8b240d..a4e78db 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,13 +20,12 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"assert_based": "python_code_evaluator.PythonCodeEvaluator", - "argument_based": "python_argument_based_evaluator.PythonCodeEvaluator", - "stdout_based": "python_stdout_evaluator.PythonStdoutEvaluator" - }, + "python": {"standardtestcase": "python_code_evaluator.PythonCodeEvaluator", + "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" + }, "c": "cpp_code_evaluator.CppCodeEvaluator", "cpp": "cpp_code_evaluator.CppCodeEvaluator", "java": "java_code_evaluator.JavaCodeEvaluator", "bash": "bash_code_evaluator.BashCodeEvaluator", "scilab": "scilab_code_evaluator.ScilabCodeEvaluator", - } + } diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index 88d8f03..0d74663 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -28,7 +28,7 @@ Snippet: {{ form.snippet }}{{ form.snippet.errors }} Tags: {{ form.tags }} Options: {{ form.options }} {{form.options.errors}} - Test: {{ form.test }} {{form.test.errors}} + Reference Code Path: {{ form.ref_code_path }} {{form.ref_code_path.errors}} test_case_type: {{ form.test_case_type }}{{ form.test_case_type.errors }} diff --git a/yaksh/tester/python/verifier.py b/yaksh/tester/python/verifier.py deleted file mode 100644 index 102dcb9..0000000 --- a/yaksh/tester/python/verifier.py +++ /dev/null @@ -1,121 +0,0 @@ -import sys -from .utils import import_by_path -from contextlib import contextmanager - - -@contextmanager -def redirect_stdout(): - from StringIO import StringIO - new_target = StringIO() - - old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout - try: - yield new_target # run some code with the replaced stdout - finally: - sys.stdout = old_target # restore to the previous value - -# def redirect_stdout(): -# # import sys -# from StringIO import StringIO -# oldout,olderr = sys.stdout, sys.stderr -# try: -# out = StringIO() -# err = StringIO() -# # sys.stdout,sys.stderr = out, err -# yield out, err -# finally: -# sys.stdout,sys.stderr = oldout, olderr -# out = out.getvalue() -# err = err.getvalue() - -TESTER_BACKEND = { - "python": "PythonPrintTesterBackend" #@@@rename to test-case-creator, this file should be backend.py -} - -class TesterException(Exception): - """ Parental class for all tester exceptions """ - pass - -class UnknownBackendException(TesterException): - """ Exception thrown if tester backend is not recognized. """ - pass - - -def detect_backend(language): - """ - Detect the right backend for a test case. - """ - backend_name = TESTER_BACKEND.get(language) - # backend = import_by_path(backend_name) - backend = PythonTesterBackend() #@@@ - return backend - -class PythonPrintTesterBackend(object): - def test_code(self, submitted, reference_output): - """ - create a test command - """ - with redirect_stdout() as output_buffer: - g = {} - exec submitted in g - - # return_buffer = out.encode('string_escape') - raw_output_value = output_buffer.getvalue() - output_value = raw_output_value.encode('string_escape').strip() - if output_value == str(reference_output): - return True - else: - raise ValueError("Incorrect Answer", output_value, reference_output) - - -class PythonTesterBackend(object): - # def __init__(self, test_case): - # self._test_case = test_case - def create(self): #@@@ test() - """ - create a test command - """ - test_code = "assert {0}({1}) == {2}".format(self.test_case_parameters['function_name'], self.test_case_parameters['args'], - self.test_case_parameters['expected_answer']) - return test_code - - def pack(self, test_case): - kw_args_dict = {} - pos_args_list = [] - test_case_data = {} - test_case_data['test_id'] = test_case.id - test_case_data['func_name'] = test_case.func_name - test_case_data['expected_answer'] = test_case.expected_answer - - if test_case.kw_args: - for args in test_case.kw_args.split(","): - arg_name, arg_value = args.split("=") - kw_args_dict[arg_name.strip()] = arg_value.strip() - - if test_case.pos_args: - for args in test_case.pos_args.split(","): - pos_args_list.append(args.strip()) - - test_case_data['kw_args'] = kw_args_dict - test_case_data['pos_args'] = pos_args_list - - return test_case_data - - def unpack(self, test_case_data): - pos_args = ", ".join(str(i) for i in test_case_data.get('pos_args')) \ - if test_case_data.get('pos_args') else "" - kw_args = ", ".join(str(k+"="+a) for k, a - in test_case_data.get('kw_args').iteritems()) \ - if test_case_data.get('kw_args') else "" - args = pos_args + ", " + kw_args if pos_args and kw_args \ - else pos_args or kw_args - function_name = test_case_data.get('func_name') - expected_answer = test_case_data.get('expected_answer') - - self.test_case_parameters = { - 'args': args, - 'function_name': function_name, - 'expected_answer': expected_answer - } - - return self.test_case_parameters \ No newline at end of file diff --git a/yaksh/views.py b/yaksh/views.py index 520f396..a8fb66a 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -188,7 +188,6 @@ def add_question(request, question_id=None): d = Question.objects.get(id=question_id) if 'save_question' in request.POST: qtn = form.save(commit=False) - test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=qtn) form.save() question = Question.objects.get(id=question_id) return my_redirect("/exam/manage/questions") @@ -213,7 +212,6 @@ def add_question(request, question_id=None): else: d = Question.objects.get(id=question_id) form = QuestionForm(instance=d) - return my_render_to_response('yaksh/add_question.html', {'form': form}, # 'formset': test_case_formset}, @@ -455,8 +453,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): # If we were not skipped, we were asked to check. For any non-mcq # questions, we obtain the results via XML-RPC with the code executed # safely in a separate process (the code_server.py) running as nobody. - test_cases = TestCase.objects.filter(question=question) - json_data = question.consolidate_answer_data(test_cases, user_answer) \ + json_data = question.consolidate_answer_data(user_answer) \ if question.type == 'code' else None correct, result = validate_answer(user, user_answer, question, json_data) if correct: @@ -505,12 +502,10 @@ def validate_answer(user, user_answer, question, json_data=None): if question.type == 'mcq': if user_answer.strip() == question.test.strip(): correct = True - message = 'Correct answer' elif question.type == 'mcc': answers = set(question.test.splitlines()) if set(user_answer) == answers: correct = True - message = 'Correct answer' elif question.type == 'code': user_dir = get_user_dir(user) json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir) @@ -848,6 +843,7 @@ def show_all_questions(request): return my_render_to_response('yaksh/showquestions.html', context, context_instance=ci) + @login_required def user_data(request, user_id, questionpaper_id=None): """Render user data.""" -- cgit From 0520bf284f9b34782fa90b433d714c887049f339 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 17 Mar 2016 16:42:33 +0530 Subject: - Add test cases for multiple python evaluators - Change name from python_standard_evaluator to python_assertion_evaluator --- yaksh/code_evaluator.py | 6 +- yaksh/evaluator_tests/test_python_evaluation.py | 159 +++++++++++++++--------- yaksh/python_assertion_evaluator.py | 41 ++++++ yaksh/python_code_evaluator.py | 98 --------------- yaksh/python_stdout_evaluator.py | 15 +-- yaksh/settings.py | 2 +- 6 files changed, 149 insertions(+), 172 deletions(-) create mode 100644 yaksh/python_assertion_evaluator.py delete mode 100644 yaksh/python_code_evaluator.py diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 7e2a729..535daa3 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -52,9 +52,7 @@ class CodeEvaluator(object): """Tests the code obtained from Code Server""" # def __init__(self, test_case_data, test, language, user_answer, # ref_code_path=None, in_dir=None): - def __init__(self, in_dir, **kwargs): - - + def __init__(self, in_dir=None): msg = 'Code took more than %s seconds to run. You probably '\ 'have an infinite loop in your code.' % SERVER_TIMEOUT self.timeout_msg = msg @@ -64,7 +62,7 @@ class CodeEvaluator(object): # self.ref_code_path = ref_code_path #@@@specific to check-code # self.test = test #@@@specific to check-code self.in_dir = in_dir #@@@Common for all, no change - self.test_case_args = None #@@@no change + # self.test_case_args = None #@@@no change # Public Protocol ########## # @classmethod diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index c55f04f..e157cde 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -1,56 +1,47 @@ import unittest import os -from yaksh.python_code_evaluator import PythonCodeEvaluator +from yaksh.python_assertion_evaluator import PythonAssertionEvaluator +from yaksh.python_stdout_evaluator import PythonStdoutEvaluator from yaksh.settings import SERVER_TIMEOUT from textwrap import dedent -class PythonEvaluationTestCases(unittest.TestCase): +class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "Python" - self.test = None - self.test_case_data = [{"func_name": "add", - "expected_answer": "5", - "test_id": u'null', - "pos_args": ["3", "2"], - "kw_args": {} - }] + self.test_case_data = ['assert(add(1,2)==3)'] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = dedent(""" - def add(a, b): - return a + b - """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + # {u'user_answer': u'def adder(a,b):\r\n return a', u'test_case_data': [u'assert(adder(1,2)==3)']} + user_answer = "def add(a,b):\n\treturn a + b" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEqual(result.get('error'), "Correct answer") def test_incorrect_answer(self): - user_answer = dedent(""" - def add(a, b): - return a - b - """) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertFalse(result.get("success")) - self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5") + user_answer = "def add(a,b):\n\treturn a - b" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)") def test_infinite_loop(self): - user_answer = dedent(""" - def add(a, b): - while True: - pass - """) - timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() - self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), timeout_msg) + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), self.timeout_msg) def test_syntax_error(self): user_answer = dedent(""" @@ -59,9 +50,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) syntax_error_msg = ["Traceback", "call", "File", "line", "", "SyntaxError", "invalid syntax"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) @@ -75,9 +68,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) indent_error_msg = ["Traceback", "call", "File", "line", "", "IndentationError", "indented block"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) @@ -87,9 +82,11 @@ class PythonEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -103,9 +100,11 @@ class PythonEvaluationTestCases(unittest.TestCase): """) recursion_error_msg = ["Traceback", "call", "RuntimeError", "maximum recursion depth exceeded"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -118,9 +117,11 @@ class PythonEvaluationTestCases(unittest.TestCase): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) @@ -134,14 +135,54 @@ class PythonEvaluationTestCases(unittest.TestCase): return int(a) + int(b) + int(c) """) value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] - get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer) - result = get_evaluator.evaluate() + get_class = PythonAssertionEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error")) +class PythonStdoutEvaluationTestCases(unittest.TestCase): + def setUp(self): + self.output = ['Hello World\\n'] + self.timeout_msg = ("Code took more than {0} seconds to run. " + "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + + def test_correct_answer(self): + user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertEqual(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) + + def test_incorrect_answer(self): + user_answer = "a = 'Goodbye'\nb = 'Name'\nprint '{0} {1}'.format(a, b)" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), "Incorrect Answer") + + def test_infinite_loop(self): + user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), 'Incorrect Answer') + +>>>>>>> - Add test cases for multiple python evaluators + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py new file mode 100644 index 0000000..0615d84 --- /dev/null +++ b/yaksh/python_assertion_evaluator.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +import sys +import traceback +import os +from os.path import join +import importlib + +# local imports +from code_evaluator import CodeEvaluator, TimeoutException + + +class PythonAssertionEvaluator(CodeEvaluator): + """Tests the Python code obtained from Code Server""" + + # def check_code(self, test, user_answer, ref_code_path): + def check_code(self, user_answer, test_case_data): + success = False + try: + tb = None + submitted = compile(user_answer, '', mode='exec') + g = {} + exec submitted in g + for test_code in test_case_data: + _tests = compile(test_code, '', mode='exec') + exec _tests in g + except AssertionError: + type, value, tb = sys.exc_info() + info = traceback.extract_tb(tb) + fname, lineno, func, text = info[-1] + text = str(test_code).splitlines()[lineno-1] + err = "{0} {1} in: {2}".format(type.__name__, str(value), text) + except TimeoutException: + raise + except Exception: + err = traceback.format_exc(limit=0) + else: + success = True + err = 'Correct answer' + + del tb + return success, err diff --git a/yaksh/python_code_evaluator.py b/yaksh/python_code_evaluator.py deleted file mode 100644 index a131a0e..0000000 --- a/yaksh/python_code_evaluator.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -import sys -import traceback -import os -from os.path import join -import importlib - -# local imports -from code_evaluator import CodeEvaluator, TimeoutException - - -class PythonCodeEvaluator(CodeEvaluator): - """Tests the Python code obtained from Code Server""" - - # def check_code(self, test, user_answer, ref_code_path): - def check_code(self, user_answer, test_cases): - success = False - - try: - tb = None - submitted = compile(user_answer, '', mode='exec') - g = {} - exec submitted in g - for test_code in test_cases: - _tests = compile(test_code, '', mode='exec') - exec _tests in g - except AssertionError: - type, value, tb = sys.exc_info() - info = traceback.extract_tb(tb) - fname, lineno, func, text = info[-1] - text = str(test_code).splitlines()[lineno-1] - err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - except TimeoutException: - raise - except Exception: - err = traceback.format_exc(limit=0) - else: - success = True - err = 'Correct answer' - - del tb - return success, err - - # def unpack_test_case_data(self, test_case_data): - # test_cases = [] - # for t in test_case_data: - # test_case = t.get('test_case') - # test_cases.append(test_case) - - # return test_cases - - # def check_code(self): - # success = False - - # try: - # tb = None - # test_code = self._create_test_case() - # submitted = compile(self.user_answer, '', mode='exec') - # g = {} - # exec submitted in g - # _tests = compile(test_code, '', mode='exec') - # exec _tests in g - # except AssertionError: - # type, value, tb = sys.exc_info() - # info = traceback.extract_tb(tb) - # fname, lineno, func, text = info[-1] - # text = str(test_code).splitlines()[lineno-1] - # err = "{0} {1} in: {2}".format(type.__name__, str(value), text) - # else: - # success = True - # err = 'Correct answer' - - # del tb - # return success, err - - # def _create_test_case(self): - # """ - # Create assert based test cases in python - # """ - # test_code = "" - # if self.test: - # return self.test - # elif self.test_case_data: - # for test_case in self.test_case_data: - # pos_args = ", ".join(str(i) for i in test_case.get('pos_args')) \ - # if test_case.get('pos_args') else "" - # kw_args = ", ".join(str(k+"="+a) for k, a - # in test_case.get('kw_args').iteritems()) \ - # if test_case.get('kw_args') else "" - # args = pos_args + ", " + kw_args if pos_args and kw_args \ - # else pos_args or kw_args - # function_name = test_case.get('func_name') - # expected_answer = test_case.get('expected_answer') - - # tcode = "assert {0}({1}) == {2}".format(function_name, args, - # expected_answer) - # test_code += tcode + "\n" - # return test_code diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 28c3372..8f3eb65 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -25,29 +25,24 @@ def redirect_stdout(): class PythonStdoutEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self, user_answer, test_cases): + def check_code(self, user_answer, test_case_data): success = False tb = None - expected_output = test_cases[0] + expected_output = test_case_data[0] submitted = compile(user_answer, '', mode='exec') with redirect_stdout() as output_buffer: g = {} exec submitted in g raw_output_value = output_buffer.getvalue() output_value = raw_output_value.encode('string_escape').strip() - if output_value == str(test_code): + if output_value == expected_output: success = True - err = 'Correct answer' + err = "Correct answer" + else: success = False err = "Incorrect Answer" del tb return success, err - - # def unpack_test_case_data(self, test_case_data): - # for t in test_case_data: - # test_case = t.get('output') - - # return test_case diff --git a/yaksh/settings.py b/yaksh/settings.py index a4e78db..30fab0a 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,7 +20,7 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"standardtestcase": "python_code_evaluator.PythonCodeEvaluator", + "python": {"standardtestcase": "python_assertion_evaluator.PythonStandardEvaluator", "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" }, "c": "cpp_code_evaluator.CppCodeEvaluator", -- cgit From 597c23866be3ecfdf94c40693b060fe9ebbf6446 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 22 Mar 2016 12:02:47 +0530 Subject: Refactor C/CPP code evaluator and add test cases --- yaksh/code_evaluator.py | 28 ++- yaksh/cpp_code_evaluator.py | 290 +++++++++++++++++-------- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 69 ++---- 3 files changed, 234 insertions(+), 153 deletions(-) diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 535daa3..b777b67 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -3,7 +3,7 @@ from SimpleXMLRPCServer import SimpleXMLRPCServer import pwd import os import stat -from os.path import isdir, dirname, abspath, join, isfile +from os.path import isdir, dirname, abspath, join, isfile, exists import signal from multiprocessing import Process, Queue import subprocess @@ -144,15 +144,31 @@ class CodeEvaluator(object): def check_code(self): raise NotImplementedError("check_code method not implemented") + # def create_submit_code_file(self, file_name): + # """ Write the code (`answer`) to a file and set the file path""" + # submit_f = open(file_name, 'w') + # submit_f.write(self.user_answer.lstrip()) + # submit_f.close() + # submit_path = abspath(submit_f.name) + + # return submit_path + def create_submit_code_file(self, file_name): - """ Write the code (`answer`) to a file and set the file path""" - submit_f = open(file_name, 'w') - submit_f.write(self.user_answer.lstrip()) - submit_f.close() - submit_path = abspath(submit_f.name) + """ Set the file path for code (`answer`)""" + submit_path = abspath(file_name) + if not exists(submit_path): + submit_f = open(submit_path, 'w') + submit_f.close() return submit_path + + def write_to_submit_code_file(self, file_path, user_answer): + """ Write the code (`answer`) to a file""" + submit_f = open(file_path, 'w') + submit_f.write(user_answer.lstrip()) + submit_f.close() + def _set_file_as_executable(self, fname): os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 129a1c8..6d18982 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -12,113 +12,211 @@ from code_evaluator import CodeEvaluator class CppCodeEvaluator(CodeEvaluator): """Tests the C code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): - super(CppCodeEvaluator, self).__init__(test_case_data, test, language, - user_answer, ref_code_path, - in_dir) - self.test_case_args = self.setup() - def setup(self): super(CppCodeEvaluator, self).setup() - - get_ref_path = self.ref_code_path - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) - self.submit_path = self.create_submit_code_file('submit.c') - - # Set file paths - c_user_output_path = os.getcwd() + '/output' - c_ref_output_path = os.getcwd() + '/executable' - - # Set command variables - compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, - c_user_output_path) - compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, - c_user_output_path, - c_ref_output_path) - run_command_args = [c_ref_output_path] - remove_user_output = c_user_output_path - remove_ref_output = c_ref_output_path - - return (ref_path, self.submit_path, compile_command, compile_main, - run_command_args, remove_user_output, remove_ref_output) + self.submit_code_path = self.create_submit_code_file('submit.c') def teardown(self): - # Delete the created file. super(CppCodeEvaluator, self).teardown() - os.remove(self.submit_path) - - def check_code(self, ref_code_path, submit_code_path, compile_command, - compile_main, run_command_args, remove_user_output, - remove_ref_output): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - if not isfile(ref_code_path): - return False, "No file at %s or Incorrect path" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s or Incorrect path' % submit_code_path - - success = False - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret - stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - ret = self._compile_command(compile_main) - proc, main_err = ret - main_err = self._remove_null_substitute_char(main_err) - - if main_err == '': - ret = self._run_command(run_command_args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" + # Delete the created file. + os.remove(self.submit_code_path) + + def set_file_paths(self): + user_output_path = os.getcwd() + '/output' + ref_output_path = os.getcwd() + '/executable' + + return user_output_path, ref_output_path + + def get_commands(self, clean_ref_code_path, user_output_path, + ref_output_path): + compile_command = 'g++ {0} -c -o {1}'.format(self.submit_code_path, + user_output_path) + compile_main = 'g++ {0} {1} -o {2}'.format(clean_ref_code_path, + user_output_path, + ref_output_path) + return compile_command, compile_main + + def check_code(self, user_answer, test_case_data): + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + + """ + ref_code_path = test_case_data[0] + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, 'No file at %s or Incorrect path' % self.submit_code_path + + success = False + self.write_to_submit_code_file(self.submit_code_path, user_answer) + user_output_path, ref_output_path = self.set_file_paths() + compile_command, compile_main = self.get_commands(clean_ref_code_path, user_output_path, ref_output_path) + ret = self._compile_command(compile_command) + proc, stdnt_stderr = ret + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + ret = self._compile_command(compile_main) + proc, main_err = ret + main_err = self._remove_null_substitute_char(main_err) + + if main_err == '': + ret = self._run_command([ref_output_path], stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" + else: + err = stdout + "\n" + stderr + os.remove(ref_output_path) else: - err = stdout + "\n" + stderr - os.remove(remove_ref_output) + err = "Error:" + try: + error_lines = main_err.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + main_err + os.remove(user_output_path) else: - err = "Error:" + err = "Compilation Error:" try: - error_lines = main_err.splitlines() + error_lines = stdnt_stderr.splitlines() for e in error_lines: if ':' in e: err = err + "\n" + e.split(":", 1)[1] else: err = err + "\n" + e except: - err = err + "\n" + main_err - os.remove(remove_user_output) - else: - err = "Compilation Error:" - try: - error_lines = stdnt_stderr.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + stdnt_stderr - - return success, err + err = err + "\n" + stdnt_stderr + + return success, err + + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + # super(CppCodeEvaluator, self).__init__(test_case_data, test, language, + # user_answer, ref_code_path, + # in_dir) + # self.test_case_args = self.setup() + + # def setup(self): + # super(CppCodeEvaluator, self).setup() + + # get_ref_path = self.ref_code_path + # ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) + # self.submit_path = self.create_submit_code_file('submit.c') + + # # Set file paths #@@@ in different func get_output_file_paths + # c_user_output_path = os.getcwd() + '/output' + # c_ref_output_path = os.getcwd() + '/executable' + + # # Set command variables #@@@ This section in different func get_commands + # compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, + # c_user_output_path) + # compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, + # c_user_output_path, + # c_ref_output_path) + # run_command_args = [c_ref_output_path] + # remove_user_output = c_user_output_path #@@@ not required + # remove_ref_output = c_ref_output_path #@@@ not required + + # return (ref_path, self.submit_path, compile_command, compile_main, + # run_command_args, remove_user_output, remove_ref_output) + + # def check_code(self, ref_code_path, submit_code_path, compile_command, + # compile_main, run_command_args, remove_user_output, + # remove_ref_output): + # """ Function validates student code using instructor code as + # reference.The first argument ref_code_path, is the path to + # instructor code, it is assumed to have executable permission. + # The second argument submit_code_path, is the path to the student + # code, it is assumed to have executable permission. + + # Returns + # -------- + + # returns (True, "Correct answer") : If the student function returns + # expected output when called by reference code. + + # returns (False, error_msg): If the student function fails to return + # expected output when called by reference code. + + # Returns (False, error_msg): If mandatory arguments are not files or + # if the required permissions are not given to the file(s). + + # """ + # if not isfile(ref_code_path): + # return False, "No file at %s or Incorrect path" % ref_code_path + # if not isfile(submit_code_path): + # return False, 'No file at %s or Incorrect path' % submit_code_path + + # success = False + # ret = self._compile_command(compile_command) + # proc, stdnt_stderr = ret + # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # # Only if compilation is successful, the program is executed + # # And tested with testcases + # if stdnt_stderr == '': + # ret = self._compile_command(compile_main) + # proc, main_err = ret + # main_err = self._remove_null_substitute_char(main_err) + + # if main_err == '': + # ret = self._run_command(run_command_args, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdout, stderr = ret + # if proc.returncode == 0: + # success, err = True, "Correct answer" + # else: + # err = stdout + "\n" + stderr + # os.remove(remove_ref_output) + # else: + # err = "Error:" + # try: + # error_lines = main_err.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + main_err + # os.remove(remove_user_output) + # else: + # err = "Compilation Error:" + # try: + # error_lines = stdnt_stderr.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + stdnt_stderr + + # return success, err diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index d01cc9c..c5f8775 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -5,71 +5,38 @@ from yaksh.settings import SERVER_TIMEOUT class CEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "C" - self.ref_code_path = "c_cpp_files/main.cpp" + self.test_case_data = ["c_cpp_files/main.cpp"] self.in_dir = "/tmp" - self.test_case_data = [] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEqual(result.get('error'), "Correct answer") def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertFalse(result.get("success")) - self.assertTrue("Compilation Error" in result.get("error")) - - def test_infinite_loop(self): - user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertFalse(result.get("success")) - self.assertEquals(result.get("error"), self.timeout_msg) - - -############################################################################### -class CppEvaluationTestCases(unittest.TestCase): - def setUp(self): - self.language = "CPP" - self.ref_code_path = "c_cpp_files/main.cpp" - self.in_dir = "/tmp" - self.test_case_data = [] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None - - def test_correct_answer(self): - user_answer = "int add(int a, int b)\n{return a+b;}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") - - def test_compilation_error(self): - user_answer = "int add(int a, int b)\n{return a+b}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Compilation Error" in result.get("error")) def test_infinite_loop(self): user_answer = "int add(int a, int b)\n{while(1>0){}}" - get_class = CppCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) -- cgit From 1f4542cdbea66899899b9c7de3ea38e39d794112 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 22 Mar 2016 16:11:28 +0530 Subject: Add anti-cheating for python stdout evaluator and modify test --- yaksh/evaluator_tests/test_python_evaluation.py | 18 +++++++++++++++--- yaksh/python_stdout_evaluator.py | 9 +++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index e157cde..b23942f 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -148,12 +148,13 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): - self.output = ['Hello World\\n'] + self.output = ['0 1 1 2 3'] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)" + # user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)" + user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.output @@ -163,7 +164,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.assertTrue(result.get('success')) def test_incorrect_answer(self): - user_answer = "a = 'Goodbye'\nb = 'Name'\nprint '{0} {1}'.format(a, b)" + user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.output @@ -172,6 +173,17 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), "Incorrect Answer") + def test_direct_printed_answer(self): + user_answer = "print '0 1 1 2 3'" + error_msg = "Incorrect Answer: Please avoid printing the expected output directly" + get_class = PythonStdoutEvaluator() + kwargs = {'user_answer': user_answer, + 'test_case_data': self.output + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertEqual(result.get('error'), error_msg) + def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonStdoutEvaluator() diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 8f3eb65..815752f 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -35,8 +35,12 @@ class PythonStdoutEvaluator(CodeEvaluator): g = {} exec submitted in g raw_output_value = output_buffer.getvalue() - output_value = raw_output_value.encode('string_escape').strip() - if output_value == expected_output: + # output_value = raw_output_value.encode('string_escape').strip() + output_value = raw_output_value.strip() + if expected_output in user_answer: + success = False + err = "Incorrect Answer: Please avoid printing the expected output directly" + elif output_value == expected_output: success = True err = "Correct answer" @@ -46,3 +50,4 @@ class PythonStdoutEvaluator(CodeEvaluator): del tb return success, err + -- cgit From e8e6b4bc35c7d012d9c41fb37d91687eb605b56c Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 29 Mar 2016 19:00:12 +0530 Subject: - Refactor Scilab, Bash, Java and C - Add Test Cases to reflect refactoring changes - Fix language registry - Add test case for language registry - Fix register in settings - Keep snippet field for now --- yaksh/bash_code_evaluator.py | 176 ++++++++++++++++------ yaksh/code_server.py | 2 +- yaksh/evaluator_tests/test_bash_evaluation.py | 74 ++++++++-- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 13 +- yaksh/evaluator_tests/test_code_evaluation.py | 38 ++++- yaksh/evaluator_tests/test_java_evaluation.py | 96 +++++++++--- yaksh/evaluator_tests/test_scilab_evaluation.py | 70 ++++++--- yaksh/java_code_evaluator.py | 188 ++++++++++++++++++------ yaksh/language_registry.py | 4 +- yaksh/scilab_code_evaluator.py | 72 ++++++--- yaksh/settings.py | 16 +- 11 files changed, 572 insertions(+), 177 deletions(-) diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index a468fd7..fbfa232 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -3,6 +3,7 @@ import traceback import pwd import os from os.path import join, isfile +import sys import subprocess import importlib @@ -11,34 +12,32 @@ from code_evaluator import CodeEvaluator class BashCodeEvaluator(CodeEvaluator): - """Tests the Bash code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): - super(BashCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, - ref_code_path, in_dir) - self.test_case_args = self._setup() + # """Tests the Bash code obtained from Code Server""" + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + # super(BashCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, + # ref_code_path, in_dir) + # self.test_case_args = self._setup() # Private Protocol ########## - def _setup(self): - super(BashCodeEvaluator, self)._setup() - - self.submit_path = self.create_submit_code_file('submit.sh') - self._set_file_as_executable(self.submit_path) - get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') - get_ref_path = get_ref_path.strip() - get_test_case_path = get_test_case_path.strip() - ref_path, test_case_path = self._set_test_code_file_path(get_ref_path, - get_test_case_path) - - return ref_path, self.submit_path, test_case_path - - def _teardown(self): + def setup(self): + super(BashCodeEvaluator, self).setup() + self.submit_code_path = self.create_submit_code_file('submit.sh') + self._set_file_as_executable(self.submit_code_path) + # get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') + # get_ref_path = get_ref_path.strip() + # get_test_case_path = get_test_case_path.strip() + # ref_path, test_case_path = self._set_test_code_file_path(get_ref_path, + # get_test_case_path) + + # return ref_path, self.submit_code_path, test_case_path + + def teardown(self): # Delete the created file. - super(BashCodeEvaluator, self)._teardown() - os.remove(self.submit_path) + super(BashCodeEvaluator, self).teardown() + os.remove(self.submit_code_path) - def _check_code(self, ref_path, submit_path, - test_case_path=None): + def check_code(self, user_answer, test_case_data): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -61,23 +60,31 @@ class BashCodeEvaluator(CodeEvaluator): the required permissions are not given to the file(s). """ - if not isfile(ref_path): - return False, "No file at %s or Incorrect path" % ref_path - if not isfile(submit_path): - return False, "No file at %s or Incorrect path" % submit_path - if not os.access(ref_path, os.X_OK): - return False, "Script %s is not executable" % ref_path - if not os.access(submit_path, os.X_OK): - return False, "Script %s is not executable" % submit_path + ref_code_path = test_case_data[0] + get_ref_path, get_test_case_path = ref_code_path.strip().split(',') + get_ref_path = get_ref_path.strip() + get_test_case_path = get_test_case_path.strip() + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(get_ref_path, + get_test_case_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, "No file at %s or Incorrect path" % self.submit_code_path + if not os.access(clean_ref_code_path, os.X_OK): + return False, "Script %s is not executable" % clean_ref_code_path + if not os.access(self.submit_code_path, os.X_OK): + return False, "Script %s is not executable" % self.submit_code_path success = False + self.write_to_submit_code_file(self.submit_code_path, user_answer) - if test_case_path is None or "": - ret = self._run_command(ref_path, stdin=None, + if clean_test_case_path is None or "": + ret = self._run_command(clean_ref_code_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, inst_stdout, inst_stderr = ret - ret = self._run_command(submit_path, stdin=None, + ret = self._run_command(self.submit_code_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, stdnt_stdout, stdnt_stderr = ret @@ -88,26 +95,26 @@ class BashCodeEvaluator(CodeEvaluator): stdnt_stderr) return False, err else: - if not isfile(test_case_path): - return False, "No test case at %s" % test_case_path - if not os.access(ref_path, os.R_OK): - return False, "Test script %s, not readable" % test_case_path + if not isfile(clean_test_case_path): + return False, "No test case at %s" % clean_test_case_path + if not os.access(clean_ref_code_path, os.R_OK): + return False, "Test script %s, not readable" % clean_test_case_path # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. # Useful for caching things like empty test files,etc. loop_count = 0 - test_cases = open(test_case_path).readlines() + test_cases = open(clean_test_case_path).readlines() num_lines = len(test_cases) for test_case in test_cases: loop_count += 1 if valid_answer: - args = [ref_path] + [x for x in test_case.split()] + args = [clean_ref_code_path] + [x for x in test_case.split()] ret = self._run_command(args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, inst_stdout, inst_stderr = ret - args = [submit_path]+[x for x in test_case.split()] + args = [self.submit_code_path]+[x for x in test_case.split()] ret = self._run_command(args, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -120,3 +127,88 @@ class BashCodeEvaluator(CodeEvaluator): stdnt_stdout+stdnt_stderr) return False, err + + + # def _check_code(self, ref_path, submit_path, + # test_case_path=None): + # """ Function validates student script using instructor script as + # reference. Test cases can optionally be provided. The first argument + # ref_path, is the path to instructor script, it is assumed to + # have executable permission. The second argument submit_path, is + # the path to the student script, it is assumed to have executable + # permission. The Third optional argument is the path to test the + # scripts. Each line in this file is a test case and each test case is + # passed to the script as standard arguments. + + # Returns + # -------- + + # returns (True, "Correct answer") : If the student script passes all + # test cases/have same output, when compared to the instructor script + + # returns (False, error_msg): If the student script fails a single + # test/have dissimilar output, when compared to the instructor script. + + # Returns (False, error_msg): If mandatory arguments are not files or if + # the required permissions are not given to the file(s). + + # """ + # if not isfile(ref_path): + # return False, "No file at %s or Incorrect path" % ref_path + # if not isfile(submit_path): + # return False, "No file at %s or Incorrect path" % submit_path + # if not os.access(ref_path, os.X_OK): + # return False, "Script %s is not executable" % ref_path + # if not os.access(submit_path, os.X_OK): + # return False, "Script %s is not executable" % submit_path + + # success = False + + # if test_case_path is None or "": + # ret = self._run_command(ref_path, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, inst_stdout, inst_stderr = ret + # ret = self._run_command(submit_path, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdnt_stdout, stdnt_stderr = ret + # if inst_stdout == stdnt_stdout: + # return True, "Correct answer" + # else: + # err = "Error: expected %s, got %s" % (inst_stderr, + # stdnt_stderr) + # return False, err + # else: + # if not isfile(test_case_path): + # return False, "No test case at %s" % test_case_path + # if not os.access(ref_path, os.R_OK): + # return False, "Test script %s, not readable" % test_case_path + # # valid_answer is True, so that we can stop once a test case fails + # valid_answer = True + # # loop_count has to be greater than or equal to one. + # # Useful for caching things like empty test files,etc. + # loop_count = 0 + # test_cases = open(test_case_path).readlines() + # num_lines = len(test_cases) + # for test_case in test_cases: + # loop_count += 1 + # if valid_answer: + # args = [ref_path] + [x for x in test_case.split()] + # ret = self._run_command(args, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, inst_stdout, inst_stderr = ret + # args = [submit_path]+[x for x in test_case.split()] + # ret = self._run_command(args, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdnt_stdout, stdnt_stderr = ret + # valid_answer = inst_stdout == stdnt_stdout + # if valid_answer and (num_lines == loop_count): + # return True, "Correct answer" + # else: + # err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, + # stdnt_stdout+stdnt_stderr) + # return False, err + diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 66c4271..48b97a7 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -31,7 +31,7 @@ import re import json # Local imports. from settings import SERVER_PORTS, SERVER_POOL_PORT -from language_registry import get_registry, create_evaluator_instance, unpack_json +from language_registry import create_evaluator_instance, unpack_json MY_DIR = abspath(dirname(__file__)) diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index c6a5408..5fa3d2d 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -5,37 +5,81 @@ from yaksh.settings import SERVER_TIMEOUT class BashEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "bash" - self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args" + self.test_case_data = ["bash_files/sample.sh,bash_files/sample.args"] self.in_dir = "/tmp" - self.test_case_data = [] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None + + # def setUp(self): + # self.language = "bash" + # self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args" + # self.in_dir = "/tmp" + # self.test_case_data = [] + # self.timeout_msg = ("Code took more than {0} seconds to run. " + # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + # self.test = None def test_correct_answer(self): user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" - get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + get_class = BashCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertTrue(result.get('success')) + self.assertEquals(result.get('error'), "Correct answer") def test_error(self): user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" - get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = BashCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" - get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = BashCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + # def test_infinite_loop(self): + # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertFalse(result.get("success")) + # self.assertEquals(result.get("error"), self.timeout_msg) + + # def test_correct_answer(self): + # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertTrue(result.get("success")) + # self.assertEqual(result.get("error"), "Correct answer") + + # def test_error(self): + # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" + # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertFalse(result.get("success")) + # self.assertTrue("Error" in result.get("error")) + + # def test_infinite_loop(self): + # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertFalse(result.get("success")) + # self.assertEquals(result.get("error"), self.timeout_msg) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index c5f8775..9d37fb4 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -18,7 +18,18 @@ class CEvaluationTestCases(unittest.TestCase): } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) - self.assertEqual(result.get('error'), "Correct answer") + self.assertEquals(result.get('error'), "Correct answer") + + def test_incorrect_answer(self): + user_answer = "int add(int a, int b)\n{return a-b;}" + get_class = CppCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect:", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) def test_compilation_error(self): user_answer = "int add(int a, int b)\n{return a+b}" diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index 84701fb..e4f129c 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -1,20 +1,21 @@ import unittest import os -from yaksh import python_code_evaluator -from yaksh.language_registry import _LanguageRegistry, set_registry, get_registry +from yaksh import python_assertion_evaluator +from yaksh.language_registry import _LanguageRegistry, get_registry from yaksh.settings import SERVER_TIMEOUT class RegistryTestCase(unittest.TestCase): def setUp(self): - set_registry() self.registry_object = get_registry() self.language_registry = _LanguageRegistry() def test_set_register(self): - class_name = getattr(python_code_evaluator, 'PythonCodeEvaluator') - self.registry_object.register("python", "yaksh.python_code_evaluator.PythonCodeEvaluator") - self.assertEquals(self.registry_object.get_class("python"), class_name) + class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator') + self.registry_object.register("python", {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", + "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" + }) + self.assertEquals(self.registry_object.get_class("python", "standardtestcase"), class_name) def tearDown(self): self.registry_object = None @@ -22,3 +23,28 @@ class RegistryTestCase(unittest.TestCase): if __name__ == '__main__': unittest.main() + + +# import unittest +# import os +# from yaksh import cpp_code_evaluator +# from yaksh.language_registry import _LanguageRegistry, get_registry +# from yaksh.settings import SERVER_TIMEOUT + + +# class RegistryTestCase(unittest.TestCase): +# def setUp(self): +# self.registry_object = get_registry() +# self.language_registry = _LanguageRegistry() + +# def test_set_register(self): +# class_name = getattr(cpp_code_evaluator, 'CppCodeEvaluator') +# self.registry_object.register("c", {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}) +# self.assertEquals(self.registry_object.get_class("c", "standardtestcase"), class_name) + +# def tearDown(self): +# self.registry_object = None + + +# if __name__ == '__main__': +# unittest.main() diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index dfa1066..a5852a3 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -2,52 +2,104 @@ import unittest import os from yaksh import code_evaluator as evaluator from yaksh.java_code_evaluator import JavaCodeEvaluator +from yaksh.settings import SERVER_TIMEOUT class JavaEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "java" - self.ref_code_path = "java_files/main_square.java" + self.test_case_data = ["java_files/main_square.java"] self.in_dir = "/tmp" - self.test_case_data = [] evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in " - "your code.").format(evaluator.SERVER_TIMEOUT) - self.test = None + "You probably have an infinite loop in your code.").format(evaluator.SERVER_TIMEOUT) def tearDown(self): evaluator.SERVER_TIMEOUT = 2 def test_correct_answer(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" - get_class = JavaCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() + get_class = JavaCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + def test_incorrect_answer(self): + user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}" + get_class = JavaCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Incorrect:", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) def test_error(self): user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" - get_class = JavaCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = JavaCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" - get_class = JavaCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = JavaCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) + # def setUp(self): + # self.language = "java" + # self.ref_code_path = "java_files/main_square.java" + # self.in_dir = "/tmp" + # self.test_case_data = [] + # evaluator.SERVER_TIMEOUT = 9 + # self.timeout_msg = ("Code took more than {0} seconds to run. " + # "You probably have an infinite loop in " + # "your code.").format(evaluator.SERVER_TIMEOUT) + # self.test = None + + # def tearDown(self): + # evaluator.SERVER_TIMEOUT = 2 + + # def test_correct_answer(self): + # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" + # get_class = JavaCodeEvaluator(self.test_case_data, self.test, + # self.language, user_answer, + # self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertTrue(result.get("success")) + # self.assertEqual(result.get("error"), "Correct answer") + + # def test_error(self): + # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" + # get_class = JavaCodeEvaluator(self.test_case_data, self.test, + # self.language, user_answer, + # self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertFalse(result.get("success")) + # self.assertTrue("Error" in result.get("error")) + + # def test_infinite_loop(self): + # user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" + # get_class = JavaCodeEvaluator(self.test_case_data, self.test, + # self.language, user_answer, + # self.ref_code_path, self.in_dir) + # result = get_class.evaluate() + + # self.assertFalse(result.get("success")) + # self.assertEquals(result.get("error"), self.timeout_msg) + if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 30af041..2b459fc 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -1,45 +1,75 @@ import unittest import os +from yaksh import code_evaluator as evaluator from yaksh.scilab_code_evaluator import ScilabCodeEvaluator from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): - self.language = "scilab" - self.ref_code_path = "scilab_files/test_add.sce" + self.test_case_data = ["scilab_files/test_add.sce"] self.in_dir = "/tmp" - self.test_case_data = [] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - self.test = None + + # def setUp(self): + # self.language = "scilab" + # self.ref_code_path = "scilab_files/test_add.sce" + # self.in_dir = "/tmp" + # self.test_case_data = [] + # self.timeout_msg = ("Code took more than {0} seconds to run. " + # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + # self.test = None + def test_correct_answer(self): user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" - get_class = ScilabCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() + get_class = ScilabCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertEquals(result.get('error'), "Correct answer") + self.assertTrue(result.get('success')) + + # def test_correct_answer(self): + # user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" + # get_class = ScilabCodeEvaluator(self.test_case_data, self.test, + # self.language, user_answer, + # self.ref_code_path, self.in_dir) + # result = get_class.evaluate() - self.assertTrue(result.get("success")) - self.assertEqual(result.get("error"), "Correct answer") + # self.assertTrue(result.get("success")) + # self.assertEqual(result.get("error"), "Correct answer") def test_error(self): user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction" - get_class = ScilabCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = ScilabCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertTrue("error" in result.get("error")) + + def test_incorrect_answer(self): + user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction" + get_class = ScilabCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) + self.assertFalse(result.get('success')) + self.assertIn("Message", result.get('error')) + self.assertTrue(result.get('error').splitlines > 1) + def test_infinite_loop(self): user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a;\nwhile(1==1)\nend\nendfunction" - get_class = ScilabCodeEvaluator(self.test_case_data, self.test, - self.language, user_answer, - self.ref_code_path, self.in_dir) - result = get_class.evaluate() - + get_class = ScilabCodeEvaluator(self.in_dir) + kwargs = {'user_answer': user_answer, + 'test_case_data': self.test_case_data + } + result = get_class.evaluate(**kwargs) self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index 6f39d71..e99448c 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -12,47 +12,24 @@ from code_evaluator import CodeEvaluator class JavaCodeEvaluator(CodeEvaluator): """Tests the Java code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): - super(JavaCodeEvaluator, self).__init__(test_case_data, test, - language, user_answer, - ref_code_path, in_dir) - self.test_case_args = self.setup() - def setup(self): super(JavaCodeEvaluator, self).setup() - - ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) - self.submit_path = self.create_submit_code_file('Test.java') - - # Set file paths - java_student_directory = os.getcwd() + '/' - java_ref_file_name = (ref_path.split('/')[-1]).split('.')[0] - - # Set command variables - compile_command = 'javac {0}'.format(self.submit_path), - compile_main = ('javac {0} -classpath ' - '{1} -d {2}').format(ref_path, - java_student_directory, - java_student_directory) - run_command_args = "java -cp {0} {1}".format(java_student_directory, - java_ref_file_name) - remove_user_output = "{0}{1}.class".format(java_student_directory, - 'Test') - remove_ref_output = "{0}{1}.class".format(java_student_directory, - java_ref_file_name) - - return (ref_path, self.submit_path, compile_command, compile_main, - run_command_args, remove_user_output, remove_ref_output) + self.submit_code_path = self.create_submit_code_file('Test.java') def teardown(self): - # Delete the created file. super(JavaCodeEvaluator, self).teardown() - os.remove(self.submit_path) + # Delete the created file. + os.remove(self.submit_code_path) + + def get_commands(self, clean_ref_code_path, user_code_directory): + compile_command = 'javac {0}'.format(self.submit_code_path), + compile_main = ('javac {0} -classpath ' + '{1} -d {2}').format(clean_ref_code_path, + user_code_directory, + user_code_directory) + return compile_command, compile_main - def check_code(self, ref_code_path, submit_code_path, compile_command, - compile_main, run_command_args, remove_user_output, - remove_ref_output): + def check_code(self, user_answer, test_case_data): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -72,12 +49,26 @@ class JavaCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - if not isfile(ref_code_path): - return False, "No file at %s or Incorrect path" % ref_code_path - if not isfile(submit_code_path): - return False, 'No file at %s or Incorrect path' % submit_code_path + ref_code_path = test_case_data[0] + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, 'No file at %s or Incorrect path' % self.submit_code_path success = False + user_code_directory = os.getcwd() + '/' + self.write_to_submit_code_file(self.submit_code_path, user_answer) + ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] + user_output_path = "{0}{1}.class".format(user_code_directory, + 'Test') + ref_output_path = "{0}{1}.class".format(user_code_directory, + ref_file_name) + # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) + compile_command, compile_main = self.get_commands(clean_ref_code_path, user_code_directory) + run_command_args = "java -cp {0} {1}".format(user_code_directory, + ref_file_name) ret = self._compile_command(compile_command) proc, stdnt_stderr = ret stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -99,7 +90,7 @@ class JavaCodeEvaluator(CodeEvaluator): success, err = True, "Correct answer" else: err = stdout + "\n" + stderr - os.remove(remove_ref_output) + os.remove(ref_output_path) else: err = "Error:" try: @@ -111,7 +102,7 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + main_err - os.remove(remove_user_output) + os.remove(user_output_path) else: err = "Compilation Error:" try: @@ -125,3 +116,118 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + stdnt_stderr return success, err + + + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + # super(JavaCodeEvaluator, self).__init__(test_case_data, test, + # language, user_answer, + # ref_code_path, in_dir) + # self.test_case_args = self.setup() + + # def setup(self): + # super(JavaCodeEvaluator, self).setup() + + # ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) + # self.submit_path = self.create_submit_code_file('Test.java') + + # # Set file paths + # java_student_directory = os.getcwd() + '/' + # java_ref_file_name = (ref_path.split('/')[-1]).split('.')[0] + + # # Set command variables + # compile_command = 'javac {0}'.format(self.submit_path), + # compile_main = ('javac {0} -classpath ' + # '{1} -d {2}').format(ref_path, + # java_student_directory, + # java_student_directory) + # run_command_args = "java -cp {0} {1}".format(java_student_directory, + # java_ref_file_name) + # remove_user_output = "{0}{1}.class".format(java_student_directory, + # 'Test') + # remove_ref_output = "{0}{1}.class".format(java_student_directory, + # java_ref_file_name) + + # return (ref_path, self.submit_path, compile_command, compile_main, + # run_command_args, remove_user_output, remove_ref_output) + + # def teardown(self): + # # Delete the created file. + # super(JavaCodeEvaluator, self).teardown() + # os.remove(self.submit_path) + + # def check_code(self, ref_code_path, submit_code_path, compile_command, + # compile_main, run_command_args, remove_user_output, + # remove_ref_output): + # """ Function validates student code using instructor code as + # reference.The first argument ref_code_path, is the path to + # instructor code, it is assumed to have executable permission. + # The second argument submit_code_path, is the path to the student + # code, it is assumed to have executable permission. + + # Returns + # -------- + + # returns (True, "Correct answer") : If the student function returns + # expected output when called by reference code. + + # returns (False, error_msg): If the student function fails to return + # expected output when called by reference code. + + # Returns (False, error_msg): If mandatory arguments are not files or + # if the required permissions are not given to the file(s). + + # """ + # if not isfile(ref_code_path): + # return False, "No file at %s or Incorrect path" % ref_code_path + # if not isfile(submit_code_path): + # return False, 'No file at %s or Incorrect path' % submit_code_path + + # success = False + # ret = self._compile_command(compile_command) + # proc, stdnt_stderr = ret + # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # # Only if compilation is successful, the program is executed + # # And tested with testcases + # if stdnt_stderr == '': + # ret = self._compile_command(compile_main) + # proc, main_err = ret + # main_err = self._remove_null_substitute_char(main_err) + + # if main_err == '': + # ret = self._run_command(run_command_args, shell=True, + # stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdout, stderr = ret + # if proc.returncode == 0: + # success, err = True, "Correct answer" + # else: + # err = stdout + "\n" + stderr + # os.remove(remove_ref_output) + # else: + # err = "Error:" + # try: + # error_lines = main_err.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + main_err + # os.remove(remove_user_output) + # else: + # err = "Compilation Error:" + # try: + # error_lines = stdnt_stderr.splitlines() + # for e in error_lines: + # if ':' in e: + # err = err + "\n" + e.split(":", 1)[1] + # else: + # err = err + "\n" + e + # except: + # err = err + "\n" + stdnt_stderr + + # return success, err diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 512e2f5..3205279 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -48,7 +48,7 @@ class _LanguageRegistry(object): get_class = getattr(get_module, class_name) return get_class - def register(self, language, class_name): + def register(self, language, class_names): """ Register a new code evaluator class for language""" - self._register[language] = class_name + self._register[language] = class_names diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index a8bd4cd..6ddfa5a 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -12,40 +12,42 @@ from code_evaluator import CodeEvaluator class ScilabCodeEvaluator(CodeEvaluator): """Tests the Scilab code obtained from Code Server""" - def __init__(self, test_case_data, test, language, user_answer, - ref_code_path=None, in_dir=None): - super(ScilabCodeEvaluator, self).__init__(test_case_data, test, - language, user_answer, - ref_code_path, in_dir) + # def __init__(self, test_case_data, test, language, user_answer, + # ref_code_path=None, in_dir=None): + # super(ScilabCodeEvaluator, self).__init__(test_case_data, test, + # language, user_answer, + # ref_code_path, in_dir) - # Removes all the commands that terminates scilab - self.user_answer, self.terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - self.test_case_args = self.setup() + # # Removes all the commands that terminates scilab + # self.user_answer, self.terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + # self.test_case_args = self.setup() def setup(self): super(ScilabCodeEvaluator, self).setup() - - ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) - self.submit_path = self.create_submit_code_file('function.sci') - - return ref_path, # Return as a tuple + # ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) + self.submit_code_path = self.create_submit_code_file('function.sci') + # return ref_path, # Return as a tuple def teardown(self): - # Delete the created file. super(ScilabCodeEvaluator, self).teardown() - os.remove(self.submit_path) + # Delete the created file. + os.remove(self.submit_code_path) - def check_code(self, ref_path): - success = False + def check_code(self, user_answer, test_case_data): + ref_code_path = test_case_data[0] + clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + success = False + self.write_to_submit_code_file(self.submit_code_path, user_answer) # Throw message if there are commmands that terminates scilab add_err="" - if self.terminate_commands: + if terminate_commands: add_err = "Please do not use exit, quit and abort commands in your\ code.\n Otherwise your code will not be evaluated\ correctly.\n" - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) cmd += ' | timeout 8 scilab-cli -nb' ret = self._run_command(cmd, shell=True, @@ -67,6 +69,38 @@ class ScilabCodeEvaluator(CodeEvaluator): return success, err + # def check_code(self, ref_path): + # success = False + + # # Throw message if there are commmands that terminates scilab + # add_err="" + # if self.terminate_commands: + # add_err = "Please do not use exit, quit and abort commands in your\ + # code.\n Otherwise your code will not be evaluated\ + # correctly.\n" + + # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) + # cmd += ' | timeout 8 scilab-cli -nb' + # ret = self._run_command(cmd, + # shell=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # proc, stdout, stderr = ret + + # # Get only the error. + # stderr = self._get_error(stdout) + # if stderr is None: + # # Clean output + # stdout = self._strip_output(stdout) + # if proc.returncode == 5: + # success, err = True, "Correct answer" + # else: + # err = add_err + stdout + # else: + # err = add_err + stderr + + # return success, err + def _remove_scilab_exit(self, string): """ Removes exit, quit and abort from the scilab code diff --git a/yaksh/settings.py b/yaksh/settings.py index 30fab0a..aaf6226 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,12 +20,12 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"standardtestcase": "python_assertion_evaluator.PythonStandardEvaluator", - "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" - }, - "c": "cpp_code_evaluator.CppCodeEvaluator", - "cpp": "cpp_code_evaluator.CppCodeEvaluator", - "java": "java_code_evaluator.JavaCodeEvaluator", - "bash": "bash_code_evaluator.BashCodeEvaluator", - "scilab": "scilab_code_evaluator.ScilabCodeEvaluator", + "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", + "stdoutbasedtestcase": "yaksh.python_stdout_evaluator.PythonStdoutEvaluator" + }, + "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator"}, + "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator"}, + "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator"}, + "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator"}, + "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, } -- cgit From 0b5a48f0ba4c4510c0026101ab84e19099691635 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 31 Mar 2016 15:59:06 +0530 Subject: - Fix django model test cases for Questions and TestCases - Remove unrequired fields from Question in its models, views and forms --- yaksh/tests.py | 96 +++++++++++++++++++++++++++++++++------------------------- yaksh/views.py | 20 ++++++++++++ 2 files changed, 75 insertions(+), 41 deletions(-) diff --git a/yaksh/tests.py b/yaksh/tests.py index 58b8518..900a261 100644 --- a/yaksh/tests.py +++ b/yaksh/tests.py @@ -1,6 +1,7 @@ import unittest from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\ - QuestionSet, AnswerPaper, Answer, TestCase, Course + QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\ + StdoutBasedTestCase import json from datetime import datetime, timedelta from django.contrib.auth.models import Group @@ -76,32 +77,23 @@ class QuestionTestCases(unittest.TestCase): self.user2 = User.objects.get(pk=2) self.question1 = Question(summary='Demo question', language='Python', type='Code', active=True, + test_case_type='standardtestcase', description='Write a function', points=1.0, - snippet='def myfunc()', user=self.user1) + user=self.user1) self.question1.save() self.question2 = Question(summary='Demo Json', language='python', type='code', active=True, description='factorial of a no', points=2.0, - snippet='def fact()', user=self.user2) + user=self.user2) self.question2.save() self.question1.tags.add('python', 'function') - self.testcase = TestCase(question=self.question1, - func_name='def myfunc', kw_args='a=10,b=11', - pos_args='12,13', expected_answer='15') - answer_data = { "test": "", - "user_answer": "demo_answer", - "test_parameter": [{"func_name": "def myfunc", - "expected_answer": "15", - "test_id": self.testcase.id, - "pos_args": ["12", "13"], - "kw_args": {"a": "10", - "b": "11"} - }], - "id": self.question1.id, - "ref_code_path": "", - } + self.assertion_testcase = StandardTestCase(question=self.question1, + test_case='assert myfunc(12, 13) == 15') + answer_data = {"user_answer": "demo_answer", + "test_case_data": ["assert myfunc(12, 13) == 15"], + } self.answer_data_json = json.dumps(answer_data) self.user_answer = "demo_answer" questions_data = [{"snippet": "def fact()", "active": True, "points": 1.0, @@ -115,11 +107,11 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(self.question1.summary, 'Demo question') self.assertEqual(self.question1.language, 'Python') self.assertEqual(self.question1.type, 'Code') - self.assertFalse(self.question1.options) + # self.assertFalse(self.question.options) self.assertEqual(self.question1.description, 'Write a function') self.assertEqual(self.question1.points, 1.0) self.assertTrue(self.question1.active) - self.assertEqual(self.question1.snippet, 'def myfunc()') + # self.assertEqual(self.question.snippet, 'def myfunc()') tag_list = [] for tag in self.question1.tags.all(): tag_list.append(tag.name) @@ -127,8 +119,7 @@ class QuestionTestCases(unittest.TestCase): def test_consolidate_answer_data(self): """ Test consolidate_answer_data function """ - result = self.question1.consolidate_answer_data([self.testcase], - self.user_answer) + result = self.question1.consolidate_answer_data(self.user_answer) self.assertEqual(result, self.answer_data_json) def test_dump_questions_into_json(self): @@ -159,25 +150,24 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(question_data.snippet, 'def fact()') ############################################################################### -class TestCaseTestCases(unittest.TestCase): - def setUp(self): - self.user = User.objects.get(pk=1) - self.question = Question(summary='Demo question', language='Python', - type='Code', active=True, - description='Write a function', points=1.0, - snippet='def myfunc()', user=self.user) - self.question.save() - self.testcase = TestCase(question=self.question, - func_name='def myfunc', kw_args='a=10,b=11', - pos_args='12,13', expected_answer='15') - - def test_testcase(self): - """ Test question """ - self.assertEqual(self.testcase.question, self.question) - self.assertEqual(self.testcase.func_name, 'def myfunc') - self.assertEqual(self.testcase.kw_args, 'a=10,b=11') - self.assertEqual(self.testcase.pos_args, '12,13') - self.assertEqual(self.testcase.expected_answer, '15') +# class TestCaseTestCases(unittest.TestCase): +# def setUp(self): +# self.question = Question(summary='Demo question', language='Python', +# type='Code', active=True, +# description='Write a function', points=1.0, +# snippet='def myfunc()') +# self.question.save() +# self.testcase = TestCase(question=self.question, +# func_name='def myfunc', kw_args='a=10,b=11', +# pos_args='12,13', expected_answer='15') + +# def test_testcase(self): +# """ Test question """ +# self.assertEqual(self.testcase.question, self.question) +# self.assertEqual(self.testcase.func_name, 'def myfunc') +# self.assertEqual(self.testcase.kw_args, 'a=10,b=11') +# self.assertEqual(self.testcase.pos_args, '12,13') +# self.assertEqual(self.testcase.expected_answer, '15') ############################################################################### @@ -507,3 +497,27 @@ class CourseTestCases(unittest.TestCase): self.course.add_teachers(self.student2) result = self.course.is_teacher(self.student2) self.assertTrue(result) + + +############################################################################### +class TestCaseTestCases(unittest.TestCase): + def setUp(self): + self.question = Question(summary='Demo question', language='Python', + type='Code', active=True, + description='Write a function', points=1.0, + ) + self.question.save() + self.assertion_testcase = StandardTestCase(question=self.question, + test_case='assert myfunc(12, 13) == 15') + self.stdout_based_testcase = StdoutBasedTestCase(question=self.question, + output='Hello World') + + def test_assertion_testcase(self): + """ Test question """ + self.assertEqual(self.assertion_testcase.question, self.question) + self.assertEqual(self.assertion_testcase.test_case, 'assert myfunc(12, 13) == 15') + + def test_stdout_based_testcase(self): + """ Test question """ + self.assertEqual(self.stdout_based_testcase.question, self.question) + self.assertEqual(self.stdout_based_testcase.output, 'Hello World') diff --git a/yaksh/views.py b/yaksh/views.py index a8fb66a..2faf127 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -217,6 +217,26 @@ def add_question(request, question_id=None): # 'formset': test_case_formset}, context_instance=ci) +# @login_required +# def add_testcase(request, question_id=None): +# """To add new test case for a question""" + +# ci = RequestContext(request) +# if not question_id: +# raise Http404('No Question Found') +# question = Question.objects.get(id=question_id) +# initial = {'question': question} + +# # if question.test + +# if request.method == "POST": +# pass +# else: +# form = TestCaseForm(user=user) +# return my_render_to_response('yaksh/add_testcase.html', +# {'form': form}, +# context_instance=ci) + @login_required def add_quiz(request, quiz_id=None): -- cgit From f120f5763904589d3c18b6cc0f4e227bcaef9a0a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 1 Apr 2016 11:18:04 +0530 Subject: Add testcase addition templates, views and forms --- yaksh/forms.py | 11 +++++++++ yaksh/templates/yaksh/add_testcase.html | 21 ++++++++++++++++ yaksh/views.py | 44 +++++++++++++++++++-------------- 3 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 yaksh/templates/yaksh/add_testcase.html diff --git a/yaksh/forms.py b/yaksh/forms.py index 94498a1..1375d10 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -214,3 +214,14 @@ class ProfileForm(forms.ModelForm): class UploadFileForm(forms.Form): file = forms.FileField() + +class StandardTestCaseForm(forms.ModelForm): + class Meta: + model = StandardTestCase + fields = ['test_case'] + + +class StdoutBasedTestCaseForm(forms.ModelForm): + class Meta: + model = StdoutBasedTestCase + fields = ['output'] diff --git a/yaksh/templates/yaksh/add_testcase.html b/yaksh/templates/yaksh/add_testcase.html new file mode 100644 index 0000000..298bd50 --- /dev/null +++ b/yaksh/templates/yaksh/add_testcase.html @@ -0,0 +1,21 @@ +{% extends "manage.html" %} + + +{% block subtitle %}Add Question{% endblock %} + +{% block css %} + + +{% endblock %} + +{% block script %} + + +{% endblock %} + +{% block onload %} onload='javascript:textareaformat();' {% endblock %} + +{% block manage %} + +Test Case: {{ form.test_case }}{{ form.test_case.errors }} + \ No newline at end of file diff --git a/yaksh/views.py b/yaksh/views.py index 2faf127..c7fdc7f 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -217,26 +217,32 @@ def add_question(request, question_id=None): # 'formset': test_case_formset}, context_instance=ci) -# @login_required -# def add_testcase(request, question_id=None): -# """To add new test case for a question""" - -# ci = RequestContext(request) -# if not question_id: -# raise Http404('No Question Found') -# question = Question.objects.get(id=question_id) -# initial = {'question': question} - -# # if question.test - -# if request.method == "POST": -# pass -# else: -# form = TestCaseForm(user=user) -# return my_render_to_response('yaksh/add_testcase.html', -# {'form': form}, -# context_instance=ci) +@login_required +def add_testcase(request, question_id=None): + """To add new test case for a question""" + + ci = RequestContext(request) + if not question_id: + raise Http404('No Question Found') + question = Question.objects.get(id=question_id) + initial = {'question': question} + + test_case_type = question.test_case_type + + if test_case_type == "standardtestcase": + from yaksh.forms import StandardTestCaseForm + if request.method == "POST": + form = StandardTestCaseForm(request.POST) + initial = {'question': question} + form = StandardTestCaseForm(initial) + if request.method == "POST": + if form.is_valid(): + form.save() + else: + return my_render_to_response('yaksh/add_testcase.html', + {'form': form}, + context_instance=ci) @login_required def add_quiz(request, quiz_id=None): -- cgit From d3241512c71d61b355358a691d18e4ff8a8df34c Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 6 Apr 2016 11:26:52 +0530 Subject: Multiple test cases passed as dicts, check_code() is iterated based on no. of test cases --- yaksh/bash_code_evaluator.py | 5 +++-- yaksh/code_evaluator.py | 9 +++++++-- yaksh/cpp_code_evaluator.py | 4 ++-- yaksh/evaluator_tests/test_bash_evaluation.py | 2 +- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 4 ++-- yaksh/evaluator_tests/test_java_evaluation.py | 2 +- yaksh/evaluator_tests/test_python_evaluation.py | 15 +++++++++------ yaksh/evaluator_tests/test_scilab_evaluation.py | 2 +- yaksh/java_code_evaluator.py | 4 ++-- yaksh/models.py | 15 ++++++++------- yaksh/python_assertion_evaluator.py | 10 +++++----- yaksh/python_stdout_evaluator.py | 4 ++-- yaksh/scilab_code_evaluator.py | 4 ++-- 13 files changed, 45 insertions(+), 35 deletions(-) diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index fbfa232..b649af7 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -37,7 +37,7 @@ class BashCodeEvaluator(CodeEvaluator): super(BashCodeEvaluator, self).teardown() os.remove(self.submit_code_path) - def check_code(self, user_answer, test_case_data): + def check_code(self, user_answer, test_case): """ Function validates student script using instructor script as reference. Test cases can optionally be provided. The first argument ref_path, is the path to instructor script, it is assumed to @@ -60,7 +60,8 @@ class BashCodeEvaluator(CodeEvaluator): the required permissions are not given to the file(s). """ - ref_code_path = test_case_data[0] + # ref_code_path = test_case_data[0] + ref_code_path = test_case get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() get_test_case_path = get_test_case_path.strip() diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index b777b67..39f0d48 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -115,7 +115,8 @@ class CodeEvaluator(object): self._change_dir(self.in_dir) # def safe_evaluate(self, args): - def safe_evaluate(self, **kwargs): + # def safe_evaluate(self, **kwargs): #@@@v2 + def safe_evaluate(self, user_answer, test_case_data): # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False @@ -124,7 +125,11 @@ class CodeEvaluator(object): # Do whatever testing needed. try: # success, err = self.check_code(*args) - success, err = self.check_code(**kwargs) + # success, err = self.check_code(**kwargs) #@@@v2 + for test_case in test_case_data: + success, err = self.check_code(user_answer, **test_case) + if not success: + break except TimeoutException: err = self.timeout_msg diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 6d18982..8cdc27c 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -36,7 +36,7 @@ class CppCodeEvaluator(CodeEvaluator): ref_output_path) return compile_command, compile_main - def check_code(self, user_answer, test_case_data): + def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -56,7 +56,7 @@ class CppCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - ref_code_path = test_case_data[0] + ref_code_path = test_case clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) if not isfile(clean_ref_code_path): diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 5fa3d2d..39247f7 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -5,7 +5,7 @@ from yaksh.settings import SERVER_TIMEOUT class BashEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = ["bash_files/sample.sh,bash_files/sample.args"] + self.test_case_data = [{"test_case": "bash_files/sample.sh,bash_files/sample.args"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index 9d37fb4..a07150d 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -5,7 +5,7 @@ from yaksh.settings import SERVER_TIMEOUT class CEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = ["c_cpp_files/main.cpp"] + self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) @@ -23,7 +23,7 @@ class CEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): user_answer = "int add(int a, int b)\n{return a-b;}" get_class = CppCodeEvaluator(self.in_dir) - kwargs = {'user_answer': user_answer, + kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index a5852a3..eacd62e 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -7,7 +7,7 @@ from yaksh.settings import SERVER_TIMEOUT class JavaEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = ["java_files/main_square.java"] + self.test_case_data = [{"test_case": "java_files/main_square.java"}] self.in_dir = "/tmp" evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index b23942f..39d4723 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -8,7 +8,10 @@ from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = ['assert(add(1,2)==3)'] + self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, + {"test_case": 'assert(add(-1,2)==1)'}, + {"test_case": 'assert(add(-1,-2)==-3)'}, + ] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) @@ -148,7 +151,7 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): - self.output = ['0 1 1 2 3'] + self.test_case_data = [{"expected_output": "0 1 1 2 3"}] self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) @@ -157,7 +160,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.output + 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") @@ -167,7 +170,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.output + 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -178,7 +181,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): error_msg = "Incorrect Answer: Please avoid printing the expected output directly" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.output + 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) @@ -188,7 +191,7 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.output + 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 2b459fc..928cd7e 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -6,7 +6,7 @@ from yaksh.settings import SERVER_TIMEOUT class ScilabEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = ["scilab_files/test_add.sce"] + self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index e99448c..b325208 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -29,7 +29,7 @@ class JavaCodeEvaluator(CodeEvaluator): user_code_directory) return compile_command, compile_main - def check_code(self, user_answer, test_case_data): + def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to instructor code, it is assumed to have executable permission. @@ -49,7 +49,7 @@ class JavaCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - ref_code_path = test_case_data[0] + ref_code_path = test_case clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) if not isfile(clean_ref_code_path): diff --git a/yaksh/models.py b/yaksh/models.py index 331446f..fd60036 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -228,8 +228,8 @@ class Question(models.Model): test_cases = self.testcase_set.all() for test in test_cases: - test_instance = test.get_child_instance(self.test_case_type) - test_case_field_value = test_instance.get_field_value() + test_case_child_instance = test.get_child_instance(self.test_case_type) + test_case_instance_dict = test_case_child_instance.get_instance_as_dict() test_case_data.append(test_case_field_value) question_data['test_case_data'] = test_case_data @@ -779,17 +779,18 @@ class StandardTestCase(TestCase): test_case = models.TextField(blank=True) def get_field_value(self): - return self.test_case + return {"test_case": self.test_case} + class StdoutBasedTestCase(TestCase): - output = models.TextField(blank=True) + expected_output = models.TextField(blank=True) def get_field_value(self): - return self.output + return {"expected_output": self.expected_output} class McqTestCase(TestCase): options = models.TextField() correct = models.BooleanField(default=False) - def validate(self, user_answer): - pass + def get_field_value(self): + return {"options": self.options, "correct": self.correct} diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 0615d84..15ff8fd 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -13,21 +13,21 @@ class PythonAssertionEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" # def check_code(self, test, user_answer, ref_code_path): - def check_code(self, user_answer, test_case_data): + # def check_code(self, user_answer, test_case_data): #@@@v2 + def check_code(self, user_answer, test_case): success = False try: tb = None submitted = compile(user_answer, '', mode='exec') g = {} exec submitted in g - for test_code in test_case_data: - _tests = compile(test_code, '', mode='exec') - exec _tests in g + _tests = compile(test_case, '', mode='exec') + exec _tests in g except AssertionError: type, value, tb = sys.exc_info() info = traceback.extract_tb(tb) fname, lineno, func, text = info[-1] - text = str(test_code).splitlines()[lineno-1] + text = str(test_case).splitlines()[lineno-1] err = "{0} {1} in: {2}".format(type.__name__, str(value), text) except TimeoutException: raise diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 815752f..9443e37 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -25,11 +25,11 @@ def redirect_stdout(): class PythonStdoutEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" - def check_code(self, user_answer, test_case_data): + def check_code(self, user_answer, expected_output): success = False tb = None - expected_output = test_case_data[0] + # expected_output = test_case_data[0] submitted = compile(user_answer, '', mode='exec') with redirect_stdout() as output_buffer: g = {} diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 6ddfa5a..61642fd 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -33,8 +33,8 @@ class ScilabCodeEvaluator(CodeEvaluator): # Delete the created file. os.remove(self.submit_code_path) - def check_code(self, user_answer, test_case_data): - ref_code_path = test_case_data[0] + def check_code(self, user_answer, test_case): + ref_code_path = test_case clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) -- cgit From 2c7f278382f4fe8071508b0a880aae34f8edfd5e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 18 Apr 2016 15:22:25 +0530 Subject: add compile_code function to compile before checking --- yaksh/code_evaluator.py | 11 ++--- yaksh/cpp_code_evaluator.py | 57 +++++++++++++++++------- yaksh/java_code_evaluator.py | 88 ++++++++++++++++++++++++++----------- yaksh/python_assertion_evaluator.py | 17 +++++-- yaksh/python_stdout_evaluator.py | 27 ++++++++---- yaksh/scilab_code_evaluator.py | 67 +++++++++++++++++++--------- 6 files changed, 183 insertions(+), 84 deletions(-) diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 39f0d48..180d719 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -127,6 +127,7 @@ class CodeEvaluator(object): # success, err = self.check_code(*args) # success, err = self.check_code(**kwargs) #@@@v2 for test_case in test_case_data: + self.compile_user_answer(user_answer, **test_case) success, err = self.check_code(user_answer, **test_case) if not success: break @@ -149,14 +150,8 @@ class CodeEvaluator(object): def check_code(self): raise NotImplementedError("check_code method not implemented") - # def create_submit_code_file(self, file_name): - # """ Write the code (`answer`) to a file and set the file path""" - # submit_f = open(file_name, 'w') - # submit_f.write(self.user_answer.lstrip()) - # submit_f.close() - # submit_path = abspath(submit_f.name) - - # return submit_path + def compile_user_answer(self, user_answer, **kwargs): + pass def create_submit_code_file(self, file_name): """ Set the file path for code (`answer`)""" diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 8cdc27c..250de8e 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -19,7 +19,7 @@ class CppCodeEvaluator(CodeEvaluator): def teardown(self): super(CppCodeEvaluator, self).teardown() # Delete the created file. - os.remove(self.submit_code_path) + os.remove(self.submit_code_path) def set_file_paths(self): user_output_path = os.getcwd() + '/output' @@ -36,6 +36,24 @@ class CppCodeEvaluator(CodeEvaluator): ref_output_path) return compile_command, compile_main + def compile_code(self, user_answer, test_case): + if hasattr(self, 'compiled_output'): + return None + else: + ref_code_path = test_case + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, 'No file at %s or Incorrect path' % self.submit_code_path + + self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.user_output_path, self.ref_output_path = self.set_file_paths() + self.compile_command, self.compile_main = self.get_commands(clean_ref_code_path, self.user_output_path, self.ref_output_path) + self.compiled_output = self._compile_command(self.compile_command) + return self.compiled_output + def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to @@ -56,31 +74,36 @@ class CppCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - ref_code_path = test_case - clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - - if not isfile(clean_ref_code_path): - return False, "No file at %s or Incorrect path" % clean_ref_code_path - if not isfile(self.submit_code_path): - return False, 'No file at %s or Incorrect path' % self.submit_code_path + # ref_code_path = test_case + # clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + # if not isfile(clean_ref_code_path): + # return False, "No file at %s or Incorrect path" % clean_ref_code_path + # if not isfile(self.submit_code_path): + # return False, 'No file at %s or Incorrect path' % self.submit_code_path + + # success = False + # self.write_to_submit_code_file(self.submit_code_path, user_answer) + # user_output_path, ref_output_path = self.set_file_paths() + # compile_command, compile_main = self.get_commands(clean_ref_code_path, user_output_path, ref_output_path) + # ret = self._compile_command(compile_command) + # proc, stdnt_stderr = ret + # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) success = False - self.write_to_submit_code_file(self.submit_code_path, user_answer) - user_output_path, ref_output_path = self.set_file_paths() - compile_command, compile_main = self.get_commands(clean_ref_code_path, user_output_path, ref_output_path) - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret + proc, stdnt_stderr = self.compiled_output stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - ret = self._compile_command(compile_main) + ret = self._compile_command(self.compile_main) proc, main_err = ret main_err = self._remove_null_substitute_char(main_err) if main_err == '': - ret = self._run_command([ref_output_path], stdin=None, + ret = self._run_command([self.ref_output_path], stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc, stdout, stderr = ret @@ -88,7 +111,7 @@ class CppCodeEvaluator(CodeEvaluator): success, err = True, "Correct answer" else: err = stdout + "\n" + stderr - os.remove(ref_output_path) + os.remove(self.ref_output_path) else: err = "Error:" try: @@ -100,7 +123,7 @@ class CppCodeEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + main_err - os.remove(user_output_path) + os.remove(self.user_output_path) else: err = "Compilation Error:" try: diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index b325208..167981b 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -29,6 +29,39 @@ class JavaCodeEvaluator(CodeEvaluator): user_code_directory) return compile_command, compile_main + def set_file_paths(self, directory, file_name): + output_path = "{0}{1}.class".format(directory, file_name) + return output_path + + def compile_code(self, user_answer, test_case): + if hasattr(self, 'compiled_output'): + return None + else: + ref_code_path = test_case + clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + if not isfile(clean_ref_code_path): + return False, "No file at %s or Incorrect path" % clean_ref_code_path + if not isfile(self.submit_code_path): + return False, 'No file at %s or Incorrect path' % self.submit_code_path + + user_code_directory = os.getcwd() + '/' + self.write_to_submit_code_file(self.submit_code_path, user_answer) + ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] + # user_output_path = "{0}{1}.class".format(user_code_directory, + # 'Test') + # ref_output_path = "{0}{1}.class".format(user_code_directory, + # ref_file_name) + # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) + self.user_output_path = self.set_file_paths(user_code_directory, 'Test') + self.ref_output_path = self.set_file_paths(user_code_directory, ref_file_name) + + compile_command, self.compile_main = self.get_commands(clean_ref_code_path, user_code_directory) + self.run_command_args = "java -cp {0} {1}".format(user_code_directory, + ref_file_name) + self.compiled_output = self._compile_command(compile_command) + return self.compiled_output + def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as reference.The first argument ref_code_path, is the path to @@ -49,39 +82,44 @@ class JavaCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - ref_code_path = test_case - clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - - if not isfile(clean_ref_code_path): - return False, "No file at %s or Incorrect path" % clean_ref_code_path - if not isfile(self.submit_code_path): - return False, 'No file at %s or Incorrect path' % self.submit_code_path - + # ref_code_path = test_case + # clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + + # if not isfile(clean_ref_code_path): + # return False, "No file at %s or Incorrect path" % clean_ref_code_path + # if not isfile(self.submit_code_path): + # return False, 'No file at %s or Incorrect path' % self.submit_code_path + + # success = False + # user_code_directory = os.getcwd() + '/' + # self.write_to_submit_code_file(self.submit_code_path, user_answer) + # ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] + # # user_output_path = "{0}{1}.class".format(user_code_directory, + # # 'Test') + # # ref_output_path = "{0}{1}.class".format(user_code_directory, + # # ref_file_name) + # # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) + # user_output_path = self.set_file_paths(user_code_directory, 'Test') + # ref_output_path = self.set_file_paths(user_code_directory, ref_file_name) + + # compile_command, compile_main = self.get_commands(clean_ref_code_path, user_code_directory) + # run_command_args = "java -cp {0} {1}".format(user_code_directory, + # ref_file_name) + # ret = self._compile_command(compile_command) + # proc, stdnt_stderr = ret success = False - user_code_directory = os.getcwd() + '/' - self.write_to_submit_code_file(self.submit_code_path, user_answer) - ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] - user_output_path = "{0}{1}.class".format(user_code_directory, - 'Test') - ref_output_path = "{0}{1}.class".format(user_code_directory, - ref_file_name) - # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) - compile_command, compile_main = self.get_commands(clean_ref_code_path, user_code_directory) - run_command_args = "java -cp {0} {1}".format(user_code_directory, - ref_file_name) - ret = self._compile_command(compile_command) - proc, stdnt_stderr = ret + proc, stdnt_stderr = self.compiled_output stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - ret = self._compile_command(compile_main) + ret = self._compile_command(self.compile_main) proc, main_err = ret main_err = self._remove_null_substitute_char(main_err) if main_err == '': - ret = self._run_command(run_command_args, shell=True, + ret = self._run_command(self.run_command_args, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -90,7 +128,7 @@ class JavaCodeEvaluator(CodeEvaluator): success, err = True, "Correct answer" else: err = stdout + "\n" + stderr - os.remove(ref_output_path) + os.remove(self.ref_output_path) else: err = "Error:" try: @@ -102,7 +140,7 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + e except: err = err + "\n" + main_err - os.remove(user_output_path) + os.remove(self.user_output_path) else: err = "Compilation Error:" try: diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 15ff8fd..c6661d1 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -12,17 +12,26 @@ from code_evaluator import CodeEvaluator, TimeoutException class PythonAssertionEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" + def compile_code(self, user_answer, test_case): + if hasattr(self, 'g'): + return None + else: + submitted = compile(user_answer, '', mode='exec') + self.g = {} + exec submitted in self.g + return self.g + # def check_code(self, test, user_answer, ref_code_path): # def check_code(self, user_answer, test_case_data): #@@@v2 def check_code(self, user_answer, test_case): success = False try: tb = None - submitted = compile(user_answer, '', mode='exec') - g = {} - exec submitted in g + # submitted = compile(user_answer, '', mode='exec') + # g = {} + # exec submitted in g _tests = compile(test_case, '', mode='exec') - exec _tests in g + exec _tests in self.g except AssertionError: type, value, tb = sys.exc_info() info = traceback.extract_tb(tb) diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 9443e37..85efa51 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -25,22 +25,31 @@ def redirect_stdout(): class PythonStdoutEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" + def compile_code(self, user_answer, expected_output): + if hasattr(self, 'output_value'): + return None + else: + submitted = compile(user_answer, '', mode='exec') + with redirect_stdout() as output_buffer: + g = {} + exec submitted in g + self.output_value = output_buffer.getvalue() + return self.output_value + def check_code(self, user_answer, expected_output): success = False tb = None - # expected_output = test_case_data[0] - submitted = compile(user_answer, '', mode='exec') - with redirect_stdout() as output_buffer: - g = {} - exec submitted in g - raw_output_value = output_buffer.getvalue() - # output_value = raw_output_value.encode('string_escape').strip() - output_value = raw_output_value.strip() + # submitted = compile(user_answer, '', mode='exec') + # with redirect_stdout() as output_buffer: + # g = {} + # exec submitted in g + # raw_output_value = output_buffer.getvalue() + # output_value = raw_output_value.strip() if expected_output in user_answer: success = False err = "Incorrect Answer: Please avoid printing the expected output directly" - elif output_value == expected_output: + elif self.output_value == expected_output: success = True err = "Correct answer" diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 61642fd..87c0e1e 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -33,27 +33,52 @@ class ScilabCodeEvaluator(CodeEvaluator): # Delete the created file. os.remove(self.submit_code_path) - def check_code(self, user_answer, test_case): - ref_code_path = test_case - clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + def compile_code(self, user_answer, test_case): + if hasattr(self, 'compiled_output'): + return None + else: + ref_code_path = test_case + clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + + self.write_to_submit_code_file(self.submit_code_path, user_answer) + # Throw message if there are commmands that terminates scilab + self.add_err = "" + if terminate_commands: + self.add_err = "Please do not use exit, quit and abort commands in your\ + code.\n Otherwise your code will not be evaluated\ + correctly.\n" + + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) + cmd += ' | timeout 8 scilab-cli -nb' + self.compiled_output = self._run_command(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return self.compiled_output + def check_code(self, user_answer, test_case): + # ref_code_path = test_case + # clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + # user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + + # success = False + # self.write_to_submit_code_file(self.submit_code_path, user_answer) + # # Throw message if there are commmands that terminates scilab + # add_err = "" + # if terminate_commands: + # add_err = "Please do not use exit, quit and abort commands in your\ + # code.\n Otherwise your code will not be evaluated\ + # correctly.\n" + + # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) + # cmd += ' | timeout 8 scilab-cli -nb' + # ret = self._run_command(cmd, + # shell=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) success = False - self.write_to_submit_code_file(self.submit_code_path, user_answer) - # Throw message if there are commmands that terminates scilab - add_err="" - if terminate_commands: - add_err = "Please do not use exit, quit and abort commands in your\ - code.\n Otherwise your code will not be evaluated\ - correctly.\n" - - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) - cmd += ' | timeout 8 scilab-cli -nb' - ret = self._run_command(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret + proc, stdout, stderr = self.compiled_output # Get only the error. stderr = self._get_error(stdout) @@ -63,9 +88,9 @@ class ScilabCodeEvaluator(CodeEvaluator): if proc.returncode == 5: success, err = True, "Correct answer" else: - err = add_err + stdout + err = self.add_err + stdout else: - err = add_err + stderr + err = self.add_err + stderr return success, err -- cgit From 5684b1b19fcb383f494f0bfc04ad1bb760abce74 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 21 Apr 2016 17:31:09 +0530 Subject: Post review improvements: - compiled_output is set during setup - python exec context renamed - _compile_command deprecated --- yaksh/code_evaluator.py | 38 ++++++++-------- yaksh/cpp_code_evaluator.py | 31 +++++++++---- yaksh/java_code_evaluator.py | 29 +++++++++--- yaksh/python_assertion_evaluator.py | 14 ++++-- yaksh/python_stdout_evaluator.py | 4 +- yaksh/scilab_code_evaluator.py | 91 +++++++++++++++++++------------------ 6 files changed, 123 insertions(+), 84 deletions(-) diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 180d719..b7a523c 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -127,7 +127,7 @@ class CodeEvaluator(object): # success, err = self.check_code(*args) # success, err = self.check_code(**kwargs) #@@@v2 for test_case in test_case_data: - self.compile_user_answer(user_answer, **test_case) + self.compile_code(user_answer, **test_case) success, err = self.check_code(user_answer, **test_case) if not success: break @@ -150,7 +150,7 @@ class CodeEvaluator(object): def check_code(self): raise NotImplementedError("check_code method not implemented") - def compile_user_answer(self, user_answer, **kwargs): + def compile_code(self, user_answer, **kwargs): pass def create_submit_code_file(self, file_name): @@ -198,23 +198,23 @@ class CodeEvaluator(object): raise return proc, stdout, stderr - def _compile_command(self, cmd, *args, **kw): - """Compiles C/C++/java code and returns errors if any. - Run a command in a subprocess while blocking, the process is killed - if it takes more than 2 seconds to run. Return the Popen object, the - stderr. - """ - try: - proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc_compile.communicate() - except TimeoutException: - # Runaway code, so kill it. - proc_compile.kill() - # Re-raise exception. - raise - return proc_compile, err + # def _compile_command(self, cmd, *args, **kw): + # """Compiles C/C++/java code and returns errors if any. + # Run a command in a subprocess while blocking, the process is killed + # if it takes more than 2 seconds to run. Return the Popen object, the + # stderr. + # """ + # try: + # proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # out, err = proc_compile.communicate() + # except TimeoutException: + # # Runaway code, so kill it. + # proc_compile.kill() + # # Re-raise exception. + # raise + # return proc_compile, err def _change_dir(self, in_dir): if in_dir is not None and isdir(in_dir): diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 250de8e..312467d 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -15,6 +15,8 @@ class CppCodeEvaluator(CodeEvaluator): def setup(self): super(CppCodeEvaluator, self).setup() self.submit_code_path = self.create_submit_code_file('submit.c') + self.compiled_user_answer = None + self.compiled_test_code = None def teardown(self): super(CppCodeEvaluator, self).teardown() @@ -37,7 +39,7 @@ class CppCodeEvaluator(CodeEvaluator): return compile_command, compile_main def compile_code(self, user_answer, test_case): - if hasattr(self, 'compiled_output'): + if self.compiled_user_answer and self.compiled_test_code: return None else: ref_code_path = test_case @@ -51,8 +53,19 @@ class CppCodeEvaluator(CodeEvaluator): self.write_to_submit_code_file(self.submit_code_path, user_answer) self.user_output_path, self.ref_output_path = self.set_file_paths() self.compile_command, self.compile_main = self.get_commands(clean_ref_code_path, self.user_output_path, self.ref_output_path) - self.compiled_output = self._compile_command(self.compile_command) - return self.compiled_output + # self.compiled_output = self._compile_command(self.compile_command) + self.compiled_user_answer = self._run_command(self.compile_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + self.compiled_test_code = self._run_command(self.compile_main, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + + return self.compiled_user_answer, self.compiled_test_code def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as @@ -89,17 +102,19 @@ class CppCodeEvaluator(CodeEvaluator): # ret = self._compile_command(compile_command) # proc, stdnt_stderr = ret # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - success = False - proc, stdnt_stderr = self.compiled_output + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - ret = self._compile_command(self.compile_main) - proc, main_err = ret + # ret = self._compile_command(self.compile_main) + # ret = self._run_command(self.compile_main, + # shell=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + proc, main_out, main_err = self.compiled_test_code main_err = self._remove_null_substitute_char(main_err) if main_err == '': diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index 167981b..9ec4209 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -15,6 +15,8 @@ class JavaCodeEvaluator(CodeEvaluator): def setup(self): super(JavaCodeEvaluator, self).setup() self.submit_code_path = self.create_submit_code_file('Test.java') + self.compiled_user_answer = None + self.compiled_test_code = None def teardown(self): super(JavaCodeEvaluator, self).teardown() @@ -34,7 +36,7 @@ class JavaCodeEvaluator(CodeEvaluator): return output_path def compile_code(self, user_answer, test_case): - if hasattr(self, 'compiled_output'): + if self.compiled_user_answer and self.compiled_test_code: return None else: ref_code_path = test_case @@ -59,8 +61,18 @@ class JavaCodeEvaluator(CodeEvaluator): compile_command, self.compile_main = self.get_commands(clean_ref_code_path, user_code_directory) self.run_command_args = "java -cp {0} {1}".format(user_code_directory, ref_file_name) - self.compiled_output = self._compile_command(compile_command) - return self.compiled_output + # self.compiled_output = self._compile_command(compile_command) + self.compiled_user_answer = self._run_command(compile_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + self.compiled_test_code = self._run_command(self.compile_main, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + return self.compiled_user_answer, self.compiled_test_code def check_code(self, user_answer, test_case): """ Function validates student code using instructor code as @@ -108,14 +120,19 @@ class JavaCodeEvaluator(CodeEvaluator): # ret = self._compile_command(compile_command) # proc, stdnt_stderr = ret success = False - proc, stdnt_stderr = self.compiled_output + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - ret = self._compile_command(self.compile_main) - proc, main_err = ret + # ret = self._compile_command(self.compile_main) + # ret = self._run_command(self.compile_main, + # shell=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + + proc, main_out, main_err = self.compiled_test_code main_err = self._remove_null_substitute_char(main_err) if main_err == '': diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index c6661d1..82b5912 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -12,14 +12,18 @@ from code_evaluator import CodeEvaluator, TimeoutException class PythonAssertionEvaluator(CodeEvaluator): """Tests the Python code obtained from Code Server""" + def setup(self): + super(PythonAssertionEvaluator, self).setup() + self.exec_scope = None + def compile_code(self, user_answer, test_case): - if hasattr(self, 'g'): + if self.exec_scope: return None else: submitted = compile(user_answer, '', mode='exec') - self.g = {} - exec submitted in self.g - return self.g + self.exec_scope = {} + exec submitted in self.exec_scope + return self.exec_scope # def check_code(self, test, user_answer, ref_code_path): # def check_code(self, user_answer, test_case_data): #@@@v2 @@ -31,7 +35,7 @@ class PythonAssertionEvaluator(CodeEvaluator): # g = {} # exec submitted in g _tests = compile(test_case, '', mode='exec') - exec _tests in self.g + exec _tests in self.exec_scope except AssertionError: type, value, tb = sys.exc_info() info = traceback.extract_tb(tb) diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 85efa51..4d89e16 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -31,8 +31,8 @@ class PythonStdoutEvaluator(CodeEvaluator): else: submitted = compile(user_answer, '', mode='exec') with redirect_stdout() as output_buffer: - g = {} - exec submitted in g + exec_scope = {} + exec submitted in exec_scope self.output_value = output_buffer.getvalue() return self.output_value diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 87c0e1e..fedfab3 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -33,52 +33,53 @@ class ScilabCodeEvaluator(CodeEvaluator): # Delete the created file. os.remove(self.submit_code_path) - def compile_code(self, user_answer, test_case): - if hasattr(self, 'compiled_output'): - return None - else: - ref_code_path = test_case - clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - - self.write_to_submit_code_file(self.submit_code_path, user_answer) - # Throw message if there are commmands that terminates scilab - self.add_err = "" - if terminate_commands: - self.add_err = "Please do not use exit, quit and abort commands in your\ - code.\n Otherwise your code will not be evaluated\ - correctly.\n" - - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) - cmd += ' | timeout 8 scilab-cli -nb' - self.compiled_output = self._run_command(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - return self.compiled_output + # def compile_code(self, user_answer, test_case): + # if hasattr(self, 'compiled_output'): + # return None + # else: + # ref_code_path = test_case + # clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + # user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + + # self.write_to_submit_code_file(self.submit_code_path, user_answer) + # # Throw message if there are commmands that terminates scilab + # self.add_err = "" + # if terminate_commands: + # self.add_err = "Please do not use exit, quit and abort commands in your\ + # code.\n Otherwise your code will not be evaluated\ + # correctly.\n" + + # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) + # cmd += ' | timeout 8 scilab-cli -nb' + # self.compiled_output = self._run_command(cmd, + # shell=True, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE) + # return self.compiled_output def check_code(self, user_answer, test_case): - # ref_code_path = test_case - # clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - # user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + ref_code_path = test_case + clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - # success = False - # self.write_to_submit_code_file(self.submit_code_path, user_answer) - # # Throw message if there are commmands that terminates scilab - # add_err = "" - # if terminate_commands: - # add_err = "Please do not use exit, quit and abort commands in your\ - # code.\n Otherwise your code will not be evaluated\ - # correctly.\n" - - # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) - # cmd += ' | timeout 8 scilab-cli -nb' - # ret = self._run_command(cmd, - # shell=True, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) success = False - proc, stdout, stderr = self.compiled_output + self.write_to_submit_code_file(self.submit_code_path, user_answer) + # Throw message if there are commmands that terminates scilab + add_err = "" + if terminate_commands: + add_err = "Please do not use exit, quit and abort commands in your\ + code.\n Otherwise your code will not be evaluated\ + correctly.\n" + + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) + cmd += ' | timeout 8 scilab-cli -nb' + ret = self._run_command(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # success = False + # proc, stdout, stderr = self.compiled_output + proc, stdout, stderr = ret # Get only the error. stderr = self._get_error(stdout) @@ -88,9 +89,11 @@ class ScilabCodeEvaluator(CodeEvaluator): if proc.returncode == 5: success, err = True, "Correct answer" else: - err = self.add_err + stdout + # err = self.add_err + stdout + err = add_err + stdout else: - err = self.add_err + stderr + # err = self.add_err + stderr + err = add_err + stderr return success, err -- cgit From c557e19470a389aaac569516ed56e1c5b453fd88 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Apr 2016 19:37:42 +0530 Subject: Add views, forms and templates (with JS) for new test cases: - Add a view and template to list out test cases for particular question - Add a view and template to add/edit test cases --- yaksh/forms.py | 22 ++++++- yaksh/models.py | 44 ++++++++++--- yaksh/static/yaksh/js/show_testcase.js | 24 +++++++ yaksh/templates/yaksh/add_question.html | 15 +++-- yaksh/templates/yaksh/add_testcase.html | 32 +++++++--- yaksh/templates/yaksh/show_testcase.html | 31 +++++++++ yaksh/views.py | 104 ++++++++++++++++++++++++------- 7 files changed, 228 insertions(+), 44 deletions(-) create mode 100644 yaksh/static/yaksh/js/show_testcase.js create mode 100644 yaksh/templates/yaksh/show_testcase.html diff --git a/yaksh/forms.py b/yaksh/forms.py index 1375d10..9ffef5e 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,8 +1,10 @@ from django import forms -from yaksh.models import Profile, Quiz, Question, TestCase, Course +from yaksh.models import Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdoutBasedTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType + from taggit.managers import TaggableManager from taggit.forms import TagField from django.forms.models import inlineformset_factory @@ -30,8 +32,8 @@ question_types = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), - # ("argument_based", "Multiple Correct Choices"), ("stdoutbasedtestcase", "Stdout Based Testcase"), + ("mcqtestcase", "MCQ Testcase"), ) UNAME_CHARS = letters + "._" + digits @@ -41,6 +43,22 @@ attempts = [(i, i) for i in range(1, 6)] attempts.append((-1, 'Infinite')) days_between_attempts = ((j, j) for j in range(401)) +def get_object_form(model, exclude_fields=None): + ctype = ContentType.objects.get(app_label="yaksh", model=model) + # ctype = ContentType.objects.get(pk=type_id) + model_class = ctype.model_class() + class _ObjectForm(forms.ModelForm): + # def __init__(self, *args, **kwargs): + # if "question" in kwargs: + # question = kwargs.pop("question") + # else: + # question = None + # self.fields["question"] = question + class Meta: + model = model_class + exclude = exclude_fields + return _ObjectForm + class UserRegisterForm(forms.Form): """A Class to create new form for User's Registration. diff --git a/yaksh/models.py b/yaksh/models.py index fd60036..b1a53f0 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -6,6 +6,7 @@ from collections import Counter from django.db import models from django.contrib.auth.models import User from django.forms.models import model_to_dict +from django.contrib.contenttypes.models import ContentType from taggit.managers import TaggableManager @@ -34,7 +35,7 @@ enrollment_methods = ( test_case_types = ( ("standardtestcase", "Standard Testcase"), ("stdoutbasedtestcase", "Stdout Based Testcase"), - # ("mcqtestcase", "MCQ Testcase"), + ("mcqtestcase", "MCQ Testcase"), ) attempts = [(i, i) for i in range(1, 6)] @@ -226,11 +227,17 @@ class Question(models.Model): question_data = {} test_case_data = [] - test_cases = self.testcase_set.all() + test_cases = self.get_test_cases() + for test in test_cases: - test_case_child_instance = test.get_child_instance(self.test_case_type) - test_case_instance_dict = test_case_child_instance.get_instance_as_dict() - test_case_data.append(test_case_field_value) + test_case_as_dict = test.get_field_value() + test_case_data.append(test_case_as_dict) + + # test_cases = self.testcase_set.all() + # for test in test_cases: + # test_case_child_instance = test.get_child_instance(self.test_case_type) + # test_case_instance_dict = test_case_child_instance.get_instance_as_dict() + # test_case_data.append(test_case_field_value) question_data['test_case_data'] = test_case_data question_data['user_answer'] = user_answer @@ -257,6 +264,18 @@ class Question(models.Model): question['user'] = user Question.objects.get_or_create(**question) + def get_test_cases(self): + test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) + test_cases = test_case_ctype.get_all_objects_for_this_type(question=self) + + return test_cases + + def get_test_case(self, test_case_id): + test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) + test_case = test_case_ctype.get_object_for_this_type(question=self, id=test_case_id) + + return test_case + def __unicode__(self): return self.summary @@ -772,8 +791,9 @@ class AssignmentUpload(models.Model): class TestCase(models.Model): question = models.ForeignKey(Question, blank=True, null = True) - def get_child_instance(self, type): - return getattr(self, type) + # def get_child_instance(self, type): + # return getattr(self, type) + class StandardTestCase(TestCase): test_case = models.TextField(blank=True) @@ -781,6 +801,9 @@ class StandardTestCase(TestCase): def get_field_value(self): return {"test_case": self.test_case} + def __unicode__(self): + return u'Question: {0} | Test Case: {1}'.format(self.question, self.test_case) + class StdoutBasedTestCase(TestCase): expected_output = models.TextField(blank=True) @@ -788,9 +811,16 @@ class StdoutBasedTestCase(TestCase): def get_field_value(self): return {"expected_output": self.expected_output} + def __unicode__(self): + return u'Question: {0} | Exp. Output: {1}'.format(self.question, self.expected_output) + + class McqTestCase(TestCase): options = models.TextField() correct = models.BooleanField(default=False) def get_field_value(self): return {"options": self.options, "correct": self.correct} + + def __unicode__(self): + return u'Question: {0} | Correct: {1}'.format(self.question, self.correct) diff --git a/yaksh/static/yaksh/js/show_testcase.js b/yaksh/static/yaksh/js/show_testcase.js new file mode 100644 index 0000000..71be9dc --- /dev/null +++ b/yaksh/static/yaksh/js/show_testcase.js @@ -0,0 +1,24 @@ +function confirm_delete(frm) +{ + var n=0; + test_case = document.getElementsByName('test_case'); + for (var i =0;iPoints:{{ form.points }}{{ form.points.errors }} Rendered:

Description: {{ form.description}} {{form.description.errors}} - Snippet: {{ form.snippet }}{{ form.snippet.errors }} + Tags: {{ form.tags }} - Options: {{ form.options }} {{form.options.errors}} + - Reference Code Path: {{ form.ref_code_path }} {{form.ref_code_path.errors}} + test_case_type: {{ form.test_case_type }}{{ form.test_case_type.errors }}
@@ -41,7 +41,12 @@ {% endif %}
-
-
+
+ + + {% if question_id %} + + {% endif %} +
{% endblock %} diff --git a/yaksh/templates/yaksh/add_testcase.html b/yaksh/templates/yaksh/add_testcase.html index 298bd50..7e29765 100644 --- a/yaksh/templates/yaksh/add_testcase.html +++ b/yaksh/templates/yaksh/add_testcase.html @@ -1,21 +1,39 @@ {% extends "manage.html" %} - -{% block subtitle %}Add Question{% endblock %} +{% block subtitle %}Add Test Case{% endblock %} {% block css %} - {% endblock %} +{% block manage %} +
+ {% csrf_token %} +
+ + {{ form.as_table }} +
+
+ +
+
+
+{% endblock %} + + diff --git a/yaksh/templates/yaksh/show_testcase.html b/yaksh/templates/yaksh/show_testcase.html new file mode 100644 index 0000000..b9cb55f --- /dev/null +++ b/yaksh/templates/yaksh/show_testcase.html @@ -0,0 +1,31 @@ +{% extends "manage.html" %} + +{% block title %} Test Case List {% endblock title %} + +{% block script %} + +{% endblock %} + +{% block subtitle %} Test Case List {% endblock %} +{% block manage %} +{% if not test_cases %} +
No Test Cases Available for This Question.
+   +{% endif %} + + +{% if test_cases %} +
+{% csrf_token %} + +{% for test_case in test_cases %} +  {{ test_case }}
+{% endfor %} + +

+   + +
+{% endif %} + +{% endblock %} diff --git a/yaksh/views.py b/yaksh/views.py index c7fdc7f..5f07880 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -22,8 +22,9 @@ import json from yaksh.models import Quiz, Question, QuestionPaper, QuestionSet, Course from yaksh.models import Profile, Answer, AnswerPaper, User, TestCase from yaksh.forms import UserRegisterForm, UserLoginForm, QuizForm,\ - QuestionForm, RandomQuestionForm, TestCaseFormSet,\ - QuestionFilterForm, CourseForm, ProfileForm, UploadFileForm + QuestionForm, RandomQuestionForm,\ + QuestionFilterForm, CourseForm, ProfileForm, UploadFileForm,\ + get_object_form from yaksh.xmlrpc_clients import code_server from settings import URL_ROOT from yaksh.models import AssignmentUpload @@ -181,8 +182,9 @@ def add_question(request, question_id=None): return my_redirect("/exam/manage/questions") return my_render_to_response('yaksh/add_question.html', - {'form': form}, - # 'formset': test_case_formset}, + # {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) else: d = Question.objects.get(id=question_id) @@ -192,56 +194,112 @@ def add_question(request, question_id=None): question = Question.objects.get(id=question_id) return my_redirect("/exam/manage/questions") return my_render_to_response('yaksh/add_question.html', - {'form': form}, - # 'formset': test_case_formset}, + # {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) else: return my_render_to_response('yaksh/add_question.html', - {'form': form}, - # 'formset': test_case_formset}, + # {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) else: if question_id is None: form = QuestionForm() # test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) return my_render_to_response('yaksh/add_question.html', - {'form': form}, - # 'formset': test_case_formset}, + # {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) else: d = Question.objects.get(id=question_id) form = QuestionForm(instance=d) return my_render_to_response('yaksh/add_question.html', - {'form': form}, - # 'formset': test_case_formset}, + # {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) @login_required -def add_testcase(request, question_id=None): - """To add new test case for a question""" +def show_testcase(request, question_id=None): + """Show all test cases related to Questions""" + user = request.user ci = RequestContext(request) + if not user.is_authenticated() or not is_moderator(user): + raise Http404('You are not allowed to view this page!') if not question_id: raise Http404('No Question Found') + question = Question.objects.get(id=question_id) - initial = {'question': question} + test_cases = question.get_test_cases() + + if request.POST.get('delete') == 'delete': + data = request.POST.getlist('test_case') + for i in data: + for t in test_cases: + if int(i) == t.id: + test_case_deleted = t.delete() + test_cases = question.get_test_cases() + + return my_render_to_response('yaksh/show_testcase.html', + {'test_cases': test_cases, + 'question_id': question_id}, + context_instance=ci) + + +@login_required +def add_testcase(request, question_id=None, test_case_id=None): + """To add new test case for a question""" + user = request.user + ci = RequestContext(request) + if not user.is_authenticated() or not is_moderator(user): + raise Http404('You are not allowed to view this page!') + if not question_id: + raise Http404('No Question Found') + + question = Question.objects.get(id=question_id) test_case_type = question.test_case_type - if test_case_type == "standardtestcase": - from yaksh.forms import StandardTestCaseForm - if request.method == "POST": - form = StandardTestCaseForm(request.POST) - initial = {'question': question} - form = StandardTestCaseForm(initial) + if test_case_id: + instance = question.get_test_case(test_case_id) + else: + instance = None + + # test_cases = self.testcase_set.all() + # for test in test_cases: + # test_case_child_instance = test.get_child_instance(test_case_type) + + test_case_form_object = get_object_form(model=test_case_type, exclude_fields=['question']) + + # if test_case_type == "standardtestcase": + # from yaksh.forms import StandardTestCaseForm + # if request.method == "POST": + # form = StandardTestCaseForm(request.POST) + # form = StandardTestCaseForm(initial) if request.method == "POST": + form = test_case_form_object(request.POST, instance=instance) if form.is_valid(): - form.save() + form_data = form.save(commit=False) + form_data.question = question + form_data.save() + return my_redirect("/exam/manage/showtestcase/{0}".format(question_id)) + else: + return my_render_to_response('yaksh/add_testcase.html', + {'form': form, + 'question_id': question_id}, + context_instance=ci) + else: + form = test_case_form_object(initial={"question": question}, instance=instance) return my_render_to_response('yaksh/add_testcase.html', - {'form': form}, + {'form': form, + 'question_id': question_id}, context_instance=ci) @login_required -- cgit From 23216995e0fc0e94fd58c6186eed74c943ae5081 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 27 Apr 2016 18:44:18 +0530 Subject: Modify get_form_object method in forms.py --- yaksh/forms.py | 7 ++++--- yaksh/models.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/yaksh/forms.py b/yaksh/forms.py index 9ffef5e..ddb1819 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -1,5 +1,5 @@ from django import forms -from yaksh.models import Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdoutBasedTestCase +from yaksh.models import get_model_class, Profile, Quiz, Question, TestCase, Course, StandardTestCase, StdoutBasedTestCase from django.contrib.auth import authenticate from django.contrib.auth.models import User @@ -44,9 +44,10 @@ attempts.append((-1, 'Infinite')) days_between_attempts = ((j, j) for j in range(401)) def get_object_form(model, exclude_fields=None): - ctype = ContentType.objects.get(app_label="yaksh", model=model) + # ctype = ContentType.objects.get(app_label="yaksh", model=model) # ctype = ContentType.objects.get(pk=type_id) - model_class = ctype.model_class() + # model_class = ctype.model_class() + model_class = get_model_class(model) class _ObjectForm(forms.ModelForm): # def __init__(self, *args, **kwargs): # if "question" in kwargs: diff --git a/yaksh/models.py b/yaksh/models.py index b1a53f0..db4b7a8 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -50,6 +50,11 @@ test_status = ( def get_assignment_dir(instance, filename): return '%s/%s' % (instance.user.roll_number, instance.assignmentQuestion.id) +def get_model_class(model): + ctype = ContentType.objects.get(app_label="yaksh", model=model) + model_class = ctype.model_class() + + ############################################################################### class Course(models.Model): -- cgit From 5967e0e1cce19d356ba622719b332f776127b277 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 28 Apr 2016 18:19:09 +0530 Subject: - Add formsets to Add question templates - add/modify test cases in place when adding questions --- yaksh/models.py | 1 + yaksh/static/yaksh/js/add_question.js | 67 +----- yaksh/templates/yaksh/add_question.html | 27 ++- yaksh/templates/yaksh/add_testcase.html | 39 ---- yaksh/templates/yaksh/show_testcase.html | 31 --- yaksh/urls.py | 2 +- yaksh/views.py | 371 +++++++++++++++++++------------ 7 files changed, 250 insertions(+), 288 deletions(-) delete mode 100644 yaksh/templates/yaksh/add_testcase.html delete mode 100644 yaksh/templates/yaksh/show_testcase.html diff --git a/yaksh/models.py b/yaksh/models.py index db4b7a8..d15fdf2 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -54,6 +54,7 @@ def get_model_class(model): ctype = ContentType.objects.get(app_label="yaksh", model=model) model_class = ctype.model_class() + return model_class ############################################################################### diff --git a/yaksh/static/yaksh/js/add_question.js b/yaksh/static/yaksh/js/add_question.js index 946c139..e435dde 100644 --- a/yaksh/static/yaksh/js/add_question.js +++ b/yaksh/static/yaksh/js/add_question.js @@ -109,34 +109,6 @@ function textareaformat() document.getElementById('my').innerHTML = document.getElementById('id_description').value ; }); - $('#id_test').bind('focus', function( event ){ - document.getElementById("id_test").rows=5; - document.getElementById("id_test").cols=40; - }); - - $('#id_test').bind('blur', function( event ){ - document.getElementById("id_test").rows=1; - document.getElementById("id_test").cols=40; - }); - - $('#id_options').bind('focus', function( event ){ - document.getElementById("id_options").rows=5; - document.getElementById("id_options").cols=40; - }); - $('#id_options').bind('blur', function( event ){ - document.getElementById("id_options").rows=1; - document.getElementById("id_options").cols=40; - }); - - $('#id_snippet').bind('focus', function( event ){ - document.getElementById("id_snippet").rows=5; - document.getElementById("id_snippet").cols=40; - }); - $('#id_snippet').bind('blur', function( event ){ - document.getElementById("id_snippet").rows=1; - document.getElementById("id_snippet").cols=40; - }); - $('#id_type').bind('focus', function(event){ var type = document.getElementById('id_type'); @@ -147,32 +119,7 @@ function textareaformat() var language = document.getElementById('id_language'); language.style.border = '1px solid #ccc'; }); - - $('#id_type').bind('change',function(event){ - var value = document.getElementById('id_type').value; - if(value == 'mcq' || value == 'mcc') - { - document.getElementById('id_options').style.visibility='visible'; - document.getElementById('label_option').innerHTML="Options :"; - } - else - { - document.getElementById('id_options').style.visibility='hidden'; - document.getElementById('label_option').innerHTML = ""; - } - }); - document.getElementById('my').innerHTML = document.getElementById('id_description').value ; - var value = document.getElementById('id_type').value; - if(value == 'mcq' || value == 'mcc') - { - document.getElementById('id_options').style.visibility='visible'; - document.getElementById('label_option').innerHTML="Options :" - } - else - { - document.getElementById('id_options').style.visibility='hidden'; - document.getElementById('label_option').innerHTML = ""; - } + document.getElementById('my').innerHTML = document.getElementById('id_description').value ; } function autosubmit() @@ -190,16 +137,4 @@ function autosubmit() return false; } - - if (type.value == 'mcq' || type.value == 'mcc') - { - var value = document.getElementById('id_options').value; - if(value.split('\n').length < 4) - { - alert("Please Enter 4 options. One option per line."); - return false; - } - return true; - } - } diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index 44aca9d..bba8db2 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -1,6 +1,5 @@ {% extends "manage.html" %} - {% block subtitle %}Add Question{% endblock %} {% block css %} @@ -30,23 +29,23 @@ - test_case_type: {{ form.test_case_type }}{{ form.test_case_type.errors }} - -
- {% if formset%} - {{ formset.management_form }} - {% for form in formset %} - {{ form }} - {% endfor %} - {% endif %} -
+ Test Case Type: {{ form.test_case_type }}{{ form.test_case_type.errors }} + +
+ {{ test_case_formset.management_form }} + + {% for form in test_case_formset %} +
+ {{ form }} +
+ {% endfor %} + +
+
- {% if question_id %} - - {% endif %}
{% endblock %} diff --git a/yaksh/templates/yaksh/add_testcase.html b/yaksh/templates/yaksh/add_testcase.html deleted file mode 100644 index 7e29765..0000000 --- a/yaksh/templates/yaksh/add_testcase.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "manage.html" %} - -{% block subtitle %}Add Test Case{% endblock %} - -{% block css %} - -{% endblock %} - -{% block manage %} -
- {% csrf_token %} -
- - {{ form.as_table }} -
-
- -
-
-
-{% endblock %} - - diff --git a/yaksh/templates/yaksh/show_testcase.html b/yaksh/templates/yaksh/show_testcase.html deleted file mode 100644 index b9cb55f..0000000 --- a/yaksh/templates/yaksh/show_testcase.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "manage.html" %} - -{% block title %} Test Case List {% endblock title %} - -{% block script %} - -{% endblock %} - -{% block subtitle %} Test Case List {% endblock %} -{% block manage %} -{% if not test_cases %} -
No Test Cases Available for This Question.
-   -{% endif %} - - -{% if test_cases %} -
-{% csrf_token %} - -{% for test_case in test_cases %} -  {{ test_case }}
-{% endfor %} - -

-   - -
-{% endif %} - -{% endblock %} diff --git a/yaksh/urls.py b/yaksh/urls.py index 18a64c2..85d7fd1 100644 --- a/yaksh/urls.py +++ b/yaksh/urls.py @@ -43,7 +43,7 @@ urlpatterns += [ url(r'^manage/addquestion/$', views.add_question), url(r'^manage/addquestion/(?P\d+)/$', views.add_question), url(r'^manage/addquiz/$', views.add_quiz), - url(r'^manage/addquiz/(?P\d+)/$', views.add_quiz), + url(r'^manage/addquiz/(?P\d+)/$', views.edit_quiz), url(r'^manage/gradeuser/$', views.grade_user), url(r'^manage/gradeuser/(?P\d+)/$',views.grade_user), url(r'^manage/gradeuser/(?P\d+)/(?P\d+)/$',views.grade_user), diff --git a/yaksh/views.py b/yaksh/views.py index 5f07880..e478cfe 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -15,11 +15,12 @@ from django.db.models import Sum, Max, Q from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import login_required from django.contrib.auth.models import Group +from django.forms.models import inlineformset_factory from taggit.models import Tag from itertools import chain import json # Local imports. -from yaksh.models import Quiz, Question, QuestionPaper, QuestionSet, Course +from yaksh.models import get_model_class, Quiz, Question, QuestionPaper, QuestionSet, Course from yaksh.models import Profile, Answer, AnswerPaper, User, TestCase from yaksh.forms import UserRegisterForm, UserLoginForm, QuizForm,\ QuestionForm, RandomQuestionForm,\ @@ -143,165 +144,261 @@ def results_user(request): @login_required -def add_question(request, question_id=None): +def add_question(request): """To add a new question in the database. Create a new question and store it.""" - - # def add_or_delete_test_form(post_request, instance): - # request_copy = post_request.copy() - # if 'add_test' in post_request: - # request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) + 1 - # elif 'delete_test' in post_request: - # request_copy['test-TOTAL_FORMS'] = int(request_copy['test-TOTAL_FORMS']) - 1 - # test_case_formset = TestCaseFormSet(request_copy, prefix='test', instance=instance) - # return test_case_formset - user = request.user ci = RequestContext(request) - if not user.is_authenticated() or not is_moderator(user): - raise Http404('You are not allowed to view this page!') - if request.method == "POST": - form = QuestionForm(request.POST) - if form.is_valid(): - if question_id is None: - # test_case_formset = add_or_delete_test_form(request.POST, form.save(commit=False)) - if 'save_question' in request.POST: - qtn = form.save(commit=False) - qtn.user = user - qtn.save() - test_case_formset = TestCaseFormSet(request.POST, prefix='test', instance=qtn) - form.save() - question = Question.objects.order_by("-id")[0] - if test_case_formset.is_valid(): - test_case_formset.save() - else: - return my_render_to_response('yaksh/add_question.html', - {'form': form, - 'formset': test_case_formset}, - context_instance=ci) - return my_redirect("/exam/manage/questions") - - return my_render_to_response('yaksh/add_question.html', - # {'form': form}, - {'form': form, - 'question_id': question_id}, - context_instance=ci) - else: - d = Question.objects.get(id=question_id) - if 'save_question' in request.POST: - qtn = form.save(commit=False) - form.save() - question = Question.objects.get(id=question_id) - return my_redirect("/exam/manage/questions") - return my_render_to_response('yaksh/add_question.html', - # {'form': form}, - {'form': form, - 'question_id': question_id}, - context_instance=ci) + if request.method == "POST" and 'save_question' in request.POST: + question_form = QuestionForm(request.POST) + if question_form.is_valid(): + new_question = question_form.save() + tags = question_form['tags'].data.split(',') + for i in range(0, len(tags)): + tag = tags[i].strip() + new_question.tags.add(tag) + new_question.save() + return my_redirect("/exam/manage/questions") else: return my_render_to_response('yaksh/add_question.html', - # {'form': form}, - {'form': form, - 'question_id': question_id}, + {'form': question_form}, context_instance=ci) else: - if question_id is None: - form = QuestionForm() - # test_case_formset = TestCaseFormSet(prefix='test', instance=Question()) - return my_render_to_response('yaksh/add_question.html', - # {'form': form}, - {'form': form, - 'question_id': question_id}, - context_instance=ci) - else: - d = Question.objects.get(id=question_id) - form = QuestionForm(instance=d) - return my_render_to_response('yaksh/add_question.html', - # {'form': form}, - {'form': form, - 'question_id': question_id}, - context_instance=ci) - -@login_required -def show_testcase(request, question_id=None): - """Show all test cases related to Questions""" - - user = request.user - ci = RequestContext(request) - if not user.is_authenticated() or not is_moderator(user): - raise Http404('You are not allowed to view this page!') - if not question_id: - raise Http404('No Question Found') - - question = Question.objects.get(id=question_id) - test_cases = question.get_test_cases() - - if request.POST.get('delete') == 'delete': - data = request.POST.getlist('test_case') - for i in data: - for t in test_cases: - if int(i) == t.id: - test_case_deleted = t.delete() - test_cases = question.get_test_cases() - - return my_render_to_response('yaksh/show_testcase.html', - {'test_cases': test_cases, - 'question_id': question_id}, - context_instance=ci) - + question_form = QuestionForm() + return my_render_to_response('yaksh/add_question.html', + {'form': question_form}, + context_instance=ci) @login_required -def add_testcase(request, question_id=None, test_case_id=None): - """To add new test case for a question""" - +def edit_question(request, question_id=None): + """To add a new question in the database. + Create a new question and store it.""" user = request.user ci = RequestContext(request) - if not user.is_authenticated() or not is_moderator(user): - raise Http404('You are not allowed to view this page!') if not question_id: raise Http404('No Question Found') - question = Question.objects.get(id=question_id) - test_case_type = question.test_case_type - - if test_case_id: - instance = question.get_test_case(test_case_id) - else: - instance = None - - # test_cases = self.testcase_set.all() - # for test in test_cases: - # test_case_child_instance = test.get_child_instance(test_case_type) - - test_case_form_object = get_object_form(model=test_case_type, exclude_fields=['question']) - - # if test_case_type == "standardtestcase": - # from yaksh.forms import StandardTestCaseForm - # if request.method == "POST": - # form = StandardTestCaseForm(request.POST) - # form = StandardTestCaseForm(initial) - - if request.method == "POST": - form = test_case_form_object(request.POST, instance=instance) - if form.is_valid(): - form_data = form.save(commit=False) - form_data.question = question - form_data.save() - return my_redirect("/exam/manage/showtestcase/{0}".format(question_id)) + question_instance = Question.objects.get(id=question_id) + + if request.method == "POST" and 'save_question' in request.POST: + question_form = QuestionForm(request.POST, instance=question_instance) + if question_form.is_valid(): + new_question = question_form.save(commit=False) + tags = question_form['tags'].data.split(',') + for i in range(0, len(tags)): + tag = tags[i].strip() + new_question.tags.add(tag) + # new_question.save() + test_case_type = question_form.cleaned_data.get('test_case_type') + test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) + test_case_model_class = get_model_class(test_case_type) + TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class) + test_case_formset = TestCaseInlineFormSet(request.POST, request.FILES, instance=new_question) + if test_case_formset.is_valid(): + new_question.save() + test_case_formset.save() + return my_redirect("/exam/manage/questions") else: - return my_render_to_response('yaksh/add_testcase.html', - {'form': form, + return my_render_to_response('yaksh/add_question.html', + {'form': question_form, + 'test_case_formset': test_case_formset, 'question_id': question_id}, context_instance=ci) - else: - form = test_case_form_object(initial={"question": question}, instance=instance) - return my_render_to_response('yaksh/add_testcase.html', - {'form': form, + question_form = QuestionForm(instance=question_instance) + test_case_type = question_instance.test_case_type + test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) + test_case_model_class = get_model_class(test_case_type) + TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class) + test_case_formset = TestCaseInlineFormSet(instance=question_instance) + + return my_render_to_response('yaksh/add_question.html', + {'form': question_form, + 'test_case_formset': test_case_formset, 'question_id': question_id}, context_instance=ci) +# def add_question(request, question_id=None): +# def add_question(request, question_id=None): +# """To add a new question in the database. +# Create a new question and store it.""" +# user = request.user +# ci = RequestContext(request) +# if not user.is_authenticated() or not is_moderator(user): +# raise Http404('You are not allowed to view this page!') +# if request.method == "POST": +# form = QuestionForm(request.POST) +# if form.is_valid(): +# if question_id is None: +# if 'save_question' in request.POST: +# form.save() +# question = Question.objects.order_by("-id")[0] +# tags = form['tags'].data.split(',') +# for i in range(0, len(tags)-1): +# tag = tags[i].strip() +# question.tags.add(tag) + +# return my_redirect("/exam/manage/questions") + +# return my_render_to_response('yaksh/add_question.html', +# # {'form': form}, +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) + +# else: +# d = Question.objects.get(id=question_id) +# if 'save_question' in request.POST: +# d.summary = form['summary'].data +# d.description = form['description'].data +# d.points = form['points'].data +# # d.options = form['options'].data +# d.type = form['type'].data +# d.active = form['active'].data +# d.language = form['language'].data +# # d.snippet = form['snippet'].data +# # d.ref_code_path = form['ref_code_path'].data +# # d.test = form['test'].data +# d.save() +# question = Question.objects.get(id=question_id) +# for tag in question.tags.all(): +# question.tags.remove(tag) +# tags = form['tags'].data.split(',') +# for i in range(0, len(tags)-1): +# tag = tags[i].strip() +# question.tags.add(tag) + +# return my_redirect("/exam/manage/questions") + +# return my_render_to_response('yaksh/add_question.html', +# # {'form': form}, +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) + +# else: +# return my_render_to_response('yaksh/add_question.html', +# # {'form': form}, +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) +# else: +# form = QuestionForm() +# if question_id is None: +# form = QuestionForm() +# return my_render_to_response('yaksh/add_question.html', +# # {'form': form}, +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) +# else: +# d = Question.objects.get(id=question_id) +# form = QuestionForm() +# form.initial['summary'] = d.summary +# form.initial['description'] = d.description +# form.initial['points'] = d.points +# # form.initial['options'] = d.options +# form.initial['type'] = d.type +# form.initial['active'] = d.active +# form.initial['language'] = d.language +# # form.initial['snippet'] = d.snippet +# # form.initial['ref_code_path'] = d.ref_code_path +# # form.initial['test'] = d.test +# form_tags = d.tags.all() +# form_tags_split = form_tags.values('name') +# initial_tags = "" +# for tag in form_tags_split: +# initial_tags = initial_tags + str(tag['name']).strip() + "," +# if (initial_tags == ","): +# initial_tags = "" +# form.initial['tags'] = initial_tags + +# return my_render_to_response('yaksh/add_question.html', +# # {'form': form}, +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) + +# @login_required +# def show_testcase(request, question_id=None): +# """Show all test cases related to Questions""" + +# user = request.user +# ci = RequestContext(request) +# if not user.is_authenticated() or not is_moderator(user): +# raise Http404('You are not allowed to view this page!') +# if not question_id: +# raise Http404('No Question Found') + +# question = Question.objects.get(id=question_id) +# test_cases = question.get_test_cases() + +# if request.POST.get('delete') == 'delete': +# data = request.POST.getlist('test_case') +# for i in data: +# for t in test_cases: +# if int(i) == t.id: +# test_case_deleted = t.delete() +# test_cases = question.get_test_cases() + +# return my_render_to_response('yaksh/show_testcase.html', +# {'test_cases': test_cases, +# 'question_id': question_id}, +# context_instance=ci) + + +# @login_required +# def add_testcase(request, question_id=None, test_case_id=None): +# """To add new test case for a question""" + +# user = request.user +# ci = RequestContext(request) +# if not user.is_authenticated() or not is_moderator(user): +# raise Http404('You are not allowed to view this page!') +# if not question_id: +# raise Http404('No Question Found') + +# question = Question.objects.get(id=question_id) +# test_case_type = question.test_case_type + +# if test_case_id: +# instance = question.get_test_case(test_case_id) +# else: +# instance = None + +# # test_cases = self.testcase_set.all() +# # for test in test_cases: +# # test_case_child_instance = test.get_child_instance(test_case_type) + +# test_case_form_object = get_object_form(model=test_case_type, exclude_fields=['question']) + +# # if test_case_type == "standardtestcase": +# # from yaksh.forms import StandardTestCaseForm +# # if request.method == "POST": +# # form = StandardTestCaseForm(request.POST) +# # form = StandardTestCaseForm(initial) + +# if request.method == "POST": +# form = test_case_form_object(request.POST, instance=instance) +# if form.is_valid(): +# form_data = form.save(commit=False) +# form_data.question = question +# form_data.save() +# return my_redirect("/exam/manage/showtestcase/{0}".format(question_id)) +# else: +# return my_render_to_response('yaksh/add_testcase.html', +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) + +# else: +# form = test_case_form_object(initial={"question": question}, instance=instance) +# return my_render_to_response('yaksh/add_testcase.html', +# {'form': form, +# 'question_id': question_id}, +# context_instance=ci) + @login_required def add_quiz(request, quiz_id=None): """To add a new quiz in the database. -- cgit From 34fb6b71ce5fcbd9a5537b305504cadaeb86d9a3 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 29 Apr 2016 11:23:07 +0530 Subject: - Modify question - answer validation in views - Modify show_question to include test_cases in template context - Modify question display template - Minor changes in forms and urls for proper rebasing --- yaksh/models.py | 12 ++++++---- yaksh/settings.py | 14 ++++++------ yaksh/templates/yaksh/question.html | 18 +++++++-------- yaksh/urls.py | 2 +- yaksh/views.py | 44 +++++++++++++++++++++++++++++++++---- 5 files changed, 65 insertions(+), 25 deletions(-) diff --git a/yaksh/models.py b/yaksh/models.py index d15fdf2..fa4121d 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -250,6 +250,7 @@ class Question(models.Model): return json.dumps(question_data) +<<<<<<< HEAD def dump_into_json(self, question_ids, user): questions = Question.objects.filter(id__in = question_ids, user_id = user.id) questions_dict = [] @@ -270,15 +271,18 @@ class Question(models.Model): question['user'] = user Question.objects.get_or_create(**question) - def get_test_cases(self): + def get_test_cases(self, **kwargs): test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) - test_cases = test_case_ctype.get_all_objects_for_this_type(question=self) + test_cases = test_case_ctype.get_all_objects_for_this_type(question=self, **kwargs) return test_cases - def get_test_case(self, test_case_id): + # def get_test_case(self, test_case_id): + # test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) + # test_case = test_case_ctype.get_object_for_this_type(question=self, id=test_case_id) + def get_test_case(self, **kwargs): test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) - test_case = test_case_ctype.get_object_for_this_type(question=self, id=test_case_id) + test_case = test_case_ctype.get_object_for_this_type(question=self, **kwargs) return test_case diff --git a/yaksh/settings.py b/yaksh/settings.py index aaf6226..f8baa2c 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,12 +20,12 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdoutbasedtestcase": "yaksh.python_stdout_evaluator.PythonStdoutEvaluator" + "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", + "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" }, - "c": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator"}, - "cpp": {"standardtestcase": "yaksh.cpp_code_evaluator.CppCodeEvaluator"}, - "java": {"standardtestcase": "yaksh.java_code_evaluator.JavaCodeEvaluator"}, - "bash": {"standardtestcase": "yaksh.bash_code_evaluator.BashCodeEvaluator"}, - "scilab": {"standardtestcase": "yaksh.scilab_code_evaluator.ScilabCodeEvaluator"}, + "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, + "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, + "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"}, + "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, + "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, } diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html index e542fe9..8a67818 100644 --- a/yaksh/templates/yaksh/question.html +++ b/yaksh/templates/yaksh/question.html @@ -165,8 +165,14 @@ function call_skip(url) {% if question.type == "mcq" %} - {% for option in question.options.strip.splitlines %} - {{option}}
+ {% for test_case in test_cases %} + {{ test_case.options }}
+ {% endfor %} + {% endif %} + {% if question.type == "mcc" %} + {% for test_case in test_cases %} + {{ test_case.options }} +
{% endfor %} {% endif %} {% if question.type == "upload" %} @@ -174,12 +180,6 @@ function call_skip(url)
{% endif %} - {% if question.type == "mcc" %} - {% for option in question.options.strip.splitlines %} - {{ option }} -
- {% endfor %} - {% endif %} {% if question.type == "code" %}

Program:

@@ -204,7 +204,7 @@ function call_skip(url) {% endif %} - +
diff --git a/yaksh/urls.py b/yaksh/urls.py index 85d7fd1..6a33888 100644 --- a/yaksh/urls.py +++ b/yaksh/urls.py @@ -41,7 +41,7 @@ urlpatterns += [ url(r'^self_enroll/(?P\d+)/$', views.self_enroll), url(r'^manage/$', views.prof_manage), url(r'^manage/addquestion/$', views.add_question), - url(r'^manage/addquestion/(?P\d+)/$', views.add_question), + url(r'^manage/addquestion/(?P\d+)/$', views.edit_question), url(r'^manage/addquiz/$', views.add_quiz), url(r'^manage/addquiz/(?P\d+)/$', views.edit_quiz), url(r'^manage/gradeuser/$', views.grade_user), diff --git a/yaksh/views.py b/yaksh/views.py index e478cfe..9492166 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -565,7 +565,9 @@ def show_question(request, question, paper, error_message=None): if paper.time_left() <= 0: reason='Your time is up!' return complete(request, reason, paper.attempt_number, paper.question_paper.id) - context = {'question': question, 'paper': paper, 'error_message': error_message} + test_cases = question.get_test_cases() + context = {'question': question, 'paper': paper, 'error_message': error_message, + 'test_cases': test_cases} answers = paper.get_previous_answers(question) if answers: context['last_attempt'] = answers[0] @@ -666,6 +668,36 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): return show_question(request, question, paper) +# def validate_answer(user, user_answer, question, json_data=None): +# """ +# Checks whether the answer submitted by the user is right or wrong. +# If right then returns correct = True, success and +# message = Correct answer. +# success is True for MCQ's and multiple correct choices because +# only one attempt are allowed for them. +# For code questions success is True only if the answer is correct. +# """ + +# result = {'success': True, 'error': 'Incorrect answer'} +# correct = False + +# if user_answer is not None: +# if question.type == 'mcq': +# if user_answer.strip() == question.test.strip(): +# correct = True +# elif question.type == 'mcc': +# answers = set(question.test.splitlines()) +# if set(user_answer) == answers: +# correct = True +# elif question.type == 'code': +# user_dir = get_user_dir(user) +# json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir) +# result = json.loads(json_result) +# if result.get('success'): +# correct = True + +# return correct, result + def validate_answer(user, user_answer, question, json_data=None): """ Checks whether the answer submitted by the user is right or wrong. @@ -681,11 +713,15 @@ def validate_answer(user, user_answer, question, json_data=None): if user_answer is not None: if question.type == 'mcq': - if user_answer.strip() == question.test.strip(): + expected_answer = question.get_test_case(correct=True).options + if user_answer.strip() == expected_answer.strip(): correct = True elif question.type == 'mcc': - answers = set(question.test.splitlines()) - if set(user_answer) == answers: + expected_answers = [] + for opt in question.get_test_cases(correct=True): + expected_answers.append(opt.options) + # answers = set(question.test.splitlines()) + if set(user_answer) == set(expected_answers): correct = True elif question.type == 'code': user_dir = get_user_dir(user) -- cgit From b40b668bcecd01b3c7f9f16b87775770ba06a33a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 2 May 2016 11:14:26 +0530 Subject: - Fix redirections after saving Question - Rename Cancel button to 'Back to QUestions' in Add Questions page --- yaksh/templates/yaksh/add_question.html | 2 +- yaksh/views.py | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index bba8db2..bc700d1 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -45,7 +45,7 @@
- +
{% endblock %} diff --git a/yaksh/views.py b/yaksh/views.py index 9492166..59a51aa 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -154,12 +154,12 @@ def add_question(request): question_form = QuestionForm(request.POST) if question_form.is_valid(): new_question = question_form.save() - tags = question_form['tags'].data.split(',') - for i in range(0, len(tags)): - tag = tags[i].strip() - new_question.tags.add(tag) - new_question.save() - return my_redirect("/exam/manage/questions") + # tags = question_form['tags'].data.split(',') + # for i in range(0, len(tags)): + # tag = tags[i].strip() + # new_question.tags.add(tag) + # new_question.save() + return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: return my_render_to_response('yaksh/add_question.html', {'form': question_form}, @@ -185,20 +185,21 @@ def edit_question(request, question_id=None): question_form = QuestionForm(request.POST, instance=question_instance) if question_form.is_valid(): new_question = question_form.save(commit=False) - tags = question_form['tags'].data.split(',') - for i in range(0, len(tags)): - tag = tags[i].strip() - new_question.tags.add(tag) + # tags = question_form['tags'].data.split(',') + # for i in range(0, len(tags)): + # tag = tags[i].strip() + # new_question.tags.add(tag) # new_question.save() test_case_type = question_form.cleaned_data.get('test_case_type') test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) test_case_model_class = get_model_class(test_case_type) - TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class) + TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class, extra=1) test_case_formset = TestCaseInlineFormSet(request.POST, request.FILES, instance=new_question) if test_case_formset.is_valid(): new_question.save() test_case_formset.save() - return my_redirect("/exam/manage/questions") + # return my_redirect("/exam/manage/questions") + return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: return my_render_to_response('yaksh/add_question.html', {'form': question_form, @@ -210,7 +211,7 @@ def edit_question(request, question_id=None): test_case_type = question_instance.test_case_type test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) test_case_model_class = get_model_class(test_case_type) - TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class) + TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class, extra=1) test_case_formset = TestCaseInlineFormSet(instance=question_instance) return my_render_to_response('yaksh/add_question.html', -- cgit From 23b7abd3c1125e4c875e214e4f673c48c4bf4752 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 2 May 2016 11:52:58 +0530 Subject: Remove commented code and cleanup --- yaksh/bash_code_evaluator.py | 101 --------------- yaksh/code_server.py | 9 -- yaksh/cpp_code_evaluator.py | 127 ------------------ yaksh/forms.py | 9 -- yaksh/java_code_evaluator.py | 152 ---------------------- yaksh/language_registry.py | 10 +- yaksh/models.py | 64 --------- yaksh/python_assertion_evaluator.py | 5 - yaksh/python_stdout_evaluator.py | 6 - yaksh/scilab_code_evaluator.py | 68 ---------- yaksh/templates/yaksh/add_question.html | 4 - yaksh/tests.py | 23 ---- yaksh/views.py | 222 -------------------------------- 13 files changed, 2 insertions(+), 798 deletions(-) diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index b649af7..12177f9 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -12,25 +12,11 @@ from code_evaluator import CodeEvaluator class BashCodeEvaluator(CodeEvaluator): - # """Tests the Bash code obtained from Code Server""" - # def __init__(self, test_case_data, test, language, user_answer, - # ref_code_path=None, in_dir=None): - # super(BashCodeEvaluator, self).__init__(test_case_data, test, language, user_answer, - # ref_code_path, in_dir) - # self.test_case_args = self._setup() - # Private Protocol ########## def setup(self): super(BashCodeEvaluator, self).setup() self.submit_code_path = self.create_submit_code_file('submit.sh') self._set_file_as_executable(self.submit_code_path) - # get_ref_path, get_test_case_path = self.ref_code_path.strip().split(',') - # get_ref_path = get_ref_path.strip() - # get_test_case_path = get_test_case_path.strip() - # ref_path, test_case_path = self._set_test_code_file_path(get_ref_path, - # get_test_case_path) - - # return ref_path, self.submit_code_path, test_case_path def teardown(self): # Delete the created file. @@ -60,7 +46,6 @@ class BashCodeEvaluator(CodeEvaluator): the required permissions are not given to the file(s). """ - # ref_code_path = test_case_data[0] ref_code_path = test_case get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() @@ -127,89 +112,3 @@ class BashCodeEvaluator(CodeEvaluator): err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, stdnt_stdout+stdnt_stderr) return False, err - - - - # def _check_code(self, ref_path, submit_path, - # test_case_path=None): - # """ Function validates student script using instructor script as - # reference. Test cases can optionally be provided. The first argument - # ref_path, is the path to instructor script, it is assumed to - # have executable permission. The second argument submit_path, is - # the path to the student script, it is assumed to have executable - # permission. The Third optional argument is the path to test the - # scripts. Each line in this file is a test case and each test case is - # passed to the script as standard arguments. - - # Returns - # -------- - - # returns (True, "Correct answer") : If the student script passes all - # test cases/have same output, when compared to the instructor script - - # returns (False, error_msg): If the student script fails a single - # test/have dissimilar output, when compared to the instructor script. - - # Returns (False, error_msg): If mandatory arguments are not files or if - # the required permissions are not given to the file(s). - - # """ - # if not isfile(ref_path): - # return False, "No file at %s or Incorrect path" % ref_path - # if not isfile(submit_path): - # return False, "No file at %s or Incorrect path" % submit_path - # if not os.access(ref_path, os.X_OK): - # return False, "Script %s is not executable" % ref_path - # if not os.access(submit_path, os.X_OK): - # return False, "Script %s is not executable" % submit_path - - # success = False - - # if test_case_path is None or "": - # ret = self._run_command(ref_path, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, inst_stdout, inst_stderr = ret - # ret = self._run_command(submit_path, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, stdnt_stdout, stdnt_stderr = ret - # if inst_stdout == stdnt_stdout: - # return True, "Correct answer" - # else: - # err = "Error: expected %s, got %s" % (inst_stderr, - # stdnt_stderr) - # return False, err - # else: - # if not isfile(test_case_path): - # return False, "No test case at %s" % test_case_path - # if not os.access(ref_path, os.R_OK): - # return False, "Test script %s, not readable" % test_case_path - # # valid_answer is True, so that we can stop once a test case fails - # valid_answer = True - # # loop_count has to be greater than or equal to one. - # # Useful for caching things like empty test files,etc. - # loop_count = 0 - # test_cases = open(test_case_path).readlines() - # num_lines = len(test_cases) - # for test_case in test_cases: - # loop_count += 1 - # if valid_answer: - # args = [ref_path] + [x for x in test_case.split()] - # ret = self._run_command(args, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, inst_stdout, inst_stderr = ret - # args = [submit_path]+[x for x in test_case.split()] - # ret = self._run_command(args, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, stdnt_stdout, stdnt_stderr = ret - # valid_answer = inst_stdout == stdnt_stdout - # if valid_answer and (num_lines == loop_count): - # return True, "Correct answer" - # else: - # err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, - # stdnt_stdout+stdnt_stderr) - # return False, err - diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 48b97a7..2397582 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -80,15 +80,6 @@ class CodeServer(object): self.queue.put(self.port) server.serve_forever() - # # Private Protocol ########## - # def _create_evaluator_instance(self, language, json_data, in_dir): - # """Create instance of relevant EvaluateCode class based on language""" - # # set_registry() - # registry = get_registry() - # cls = registry.get_class(language) - # instance = cls.from_json(language, json_data, in_dir) - # return instance - ############################################################################### # `ServerPool` class. diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index 312467d..becf371 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -53,7 +53,6 @@ class CppCodeEvaluator(CodeEvaluator): self.write_to_submit_code_file(self.submit_code_path, user_answer) self.user_output_path, self.ref_output_path = self.set_file_paths() self.compile_command, self.compile_main = self.get_commands(clean_ref_code_path, self.user_output_path, self.ref_output_path) - # self.compiled_output = self._compile_command(self.compile_command) self.compiled_user_answer = self._run_command(self.compile_command, shell=True, stdout=subprocess.PIPE, @@ -87,21 +86,6 @@ class CppCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - # ref_code_path = test_case - # clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - - # if not isfile(clean_ref_code_path): - # return False, "No file at %s or Incorrect path" % clean_ref_code_path - # if not isfile(self.submit_code_path): - # return False, 'No file at %s or Incorrect path' % self.submit_code_path - - # success = False - # self.write_to_submit_code_file(self.submit_code_path, user_answer) - # user_output_path, ref_output_path = self.set_file_paths() - # compile_command, compile_main = self.get_commands(clean_ref_code_path, user_output_path, ref_output_path) - # ret = self._compile_command(compile_command) - # proc, stdnt_stderr = ret - # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) success = False proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -109,11 +93,6 @@ class CppCodeEvaluator(CodeEvaluator): # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - # ret = self._compile_command(self.compile_main) - # ret = self._run_command(self.compile_main, - # shell=True, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) proc, main_out, main_err = self.compiled_test_code main_err = self._remove_null_substitute_char(main_err) @@ -152,109 +131,3 @@ class CppCodeEvaluator(CodeEvaluator): err = err + "\n" + stdnt_stderr return success, err - - # def __init__(self, test_case_data, test, language, user_answer, - # ref_code_path=None, in_dir=None): - # super(CppCodeEvaluator, self).__init__(test_case_data, test, language, - # user_answer, ref_code_path, - # in_dir) - # self.test_case_args = self.setup() - - # def setup(self): - # super(CppCodeEvaluator, self).setup() - - # get_ref_path = self.ref_code_path - # ref_path, test_case_path = self._set_test_code_file_path(get_ref_path) - # self.submit_path = self.create_submit_code_file('submit.c') - - # # Set file paths #@@@ in different func get_output_file_paths - # c_user_output_path = os.getcwd() + '/output' - # c_ref_output_path = os.getcwd() + '/executable' - - # # Set command variables #@@@ This section in different func get_commands - # compile_command = 'g++ {0} -c -o {1}'.format(self.submit_path, - # c_user_output_path) - # compile_main = 'g++ {0} {1} -o {2}'.format(ref_path, - # c_user_output_path, - # c_ref_output_path) - # run_command_args = [c_ref_output_path] - # remove_user_output = c_user_output_path #@@@ not required - # remove_ref_output = c_ref_output_path #@@@ not required - - # return (ref_path, self.submit_path, compile_command, compile_main, - # run_command_args, remove_user_output, remove_ref_output) - - # def check_code(self, ref_code_path, submit_code_path, compile_command, - # compile_main, run_command_args, remove_user_output, - # remove_ref_output): - # """ Function validates student code using instructor code as - # reference.The first argument ref_code_path, is the path to - # instructor code, it is assumed to have executable permission. - # The second argument submit_code_path, is the path to the student - # code, it is assumed to have executable permission. - - # Returns - # -------- - - # returns (True, "Correct answer") : If the student function returns - # expected output when called by reference code. - - # returns (False, error_msg): If the student function fails to return - # expected output when called by reference code. - - # Returns (False, error_msg): If mandatory arguments are not files or - # if the required permissions are not given to the file(s). - - # """ - # if not isfile(ref_code_path): - # return False, "No file at %s or Incorrect path" % ref_code_path - # if not isfile(submit_code_path): - # return False, 'No file at %s or Incorrect path' % submit_code_path - - # success = False - # ret = self._compile_command(compile_command) - # proc, stdnt_stderr = ret - # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # # Only if compilation is successful, the program is executed - # # And tested with testcases - # if stdnt_stderr == '': - # ret = self._compile_command(compile_main) - # proc, main_err = ret - # main_err = self._remove_null_substitute_char(main_err) - - # if main_err == '': - # ret = self._run_command(run_command_args, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, stdout, stderr = ret - # if proc.returncode == 0: - # success, err = True, "Correct answer" - # else: - # err = stdout + "\n" + stderr - # os.remove(remove_ref_output) - # else: - # err = "Error:" - # try: - # error_lines = main_err.splitlines() - # for e in error_lines: - # if ':' in e: - # err = err + "\n" + e.split(":", 1)[1] - # else: - # err = err + "\n" + e - # except: - # err = err + "\n" + main_err - # os.remove(remove_user_output) - # else: - # err = "Compilation Error:" - # try: - # error_lines = stdnt_stderr.splitlines() - # for e in error_lines: - # if ':' in e: - # err = err + "\n" + e.split(":", 1)[1] - # else: - # err = err + "\n" + e - # except: - # err = err + "\n" + stdnt_stderr - - # return success, err diff --git a/yaksh/forms.py b/yaksh/forms.py index ddb1819..2ce2cba 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -44,17 +44,8 @@ attempts.append((-1, 'Infinite')) days_between_attempts = ((j, j) for j in range(401)) def get_object_form(model, exclude_fields=None): - # ctype = ContentType.objects.get(app_label="yaksh", model=model) - # ctype = ContentType.objects.get(pk=type_id) - # model_class = ctype.model_class() model_class = get_model_class(model) class _ObjectForm(forms.ModelForm): - # def __init__(self, *args, **kwargs): - # if "question" in kwargs: - # question = kwargs.pop("question") - # else: - # question = None - # self.fields["question"] = question class Meta: model = model_class exclude = exclude_fields diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index 9ec4209..ac2c487 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -50,18 +50,12 @@ class JavaCodeEvaluator(CodeEvaluator): user_code_directory = os.getcwd() + '/' self.write_to_submit_code_file(self.submit_code_path, user_answer) ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] - # user_output_path = "{0}{1}.class".format(user_code_directory, - # 'Test') - # ref_output_path = "{0}{1}.class".format(user_code_directory, - # ref_file_name) - # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) self.user_output_path = self.set_file_paths(user_code_directory, 'Test') self.ref_output_path = self.set_file_paths(user_code_directory, ref_file_name) compile_command, self.compile_main = self.get_commands(clean_ref_code_path, user_code_directory) self.run_command_args = "java -cp {0} {1}".format(user_code_directory, ref_file_name) - # self.compiled_output = self._compile_command(compile_command) self.compiled_user_answer = self._run_command(compile_command, shell=True, stdout=subprocess.PIPE, @@ -94,31 +88,6 @@ class JavaCodeEvaluator(CodeEvaluator): if the required permissions are not given to the file(s). """ - # ref_code_path = test_case - # clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - - # if not isfile(clean_ref_code_path): - # return False, "No file at %s or Incorrect path" % clean_ref_code_path - # if not isfile(self.submit_code_path): - # return False, 'No file at %s or Incorrect path' % self.submit_code_path - - # success = False - # user_code_directory = os.getcwd() + '/' - # self.write_to_submit_code_file(self.submit_code_path, user_answer) - # ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] - # # user_output_path = "{0}{1}.class".format(user_code_directory, - # # 'Test') - # # ref_output_path = "{0}{1}.class".format(user_code_directory, - # # ref_file_name) - # # user_output_path, ref_output_path = self.set_file_paths(user_code_directory, clean_ref_code_path) - # user_output_path = self.set_file_paths(user_code_directory, 'Test') - # ref_output_path = self.set_file_paths(user_code_directory, ref_file_name) - - # compile_command, compile_main = self.get_commands(clean_ref_code_path, user_code_directory) - # run_command_args = "java -cp {0} {1}".format(user_code_directory, - # ref_file_name) - # ret = self._compile_command(compile_command) - # proc, stdnt_stderr = ret success = False proc, stdnt_out, stdnt_stderr = self.compiled_user_answer stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) @@ -126,12 +95,6 @@ class JavaCodeEvaluator(CodeEvaluator): # Only if compilation is successful, the program is executed # And tested with testcases if stdnt_stderr == '': - # ret = self._compile_command(self.compile_main) - # ret = self._run_command(self.compile_main, - # shell=True, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - proc, main_out, main_err = self.compiled_test_code main_err = self._remove_null_substitute_char(main_err) @@ -171,118 +134,3 @@ class JavaCodeEvaluator(CodeEvaluator): err = err + "\n" + stdnt_stderr return success, err - - - # def __init__(self, test_case_data, test, language, user_answer, - # ref_code_path=None, in_dir=None): - # super(JavaCodeEvaluator, self).__init__(test_case_data, test, - # language, user_answer, - # ref_code_path, in_dir) - # self.test_case_args = self.setup() - - # def setup(self): - # super(JavaCodeEvaluator, self).setup() - - # ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) - # self.submit_path = self.create_submit_code_file('Test.java') - - # # Set file paths - # java_student_directory = os.getcwd() + '/' - # java_ref_file_name = (ref_path.split('/')[-1]).split('.')[0] - - # # Set command variables - # compile_command = 'javac {0}'.format(self.submit_path), - # compile_main = ('javac {0} -classpath ' - # '{1} -d {2}').format(ref_path, - # java_student_directory, - # java_student_directory) - # run_command_args = "java -cp {0} {1}".format(java_student_directory, - # java_ref_file_name) - # remove_user_output = "{0}{1}.class".format(java_student_directory, - # 'Test') - # remove_ref_output = "{0}{1}.class".format(java_student_directory, - # java_ref_file_name) - - # return (ref_path, self.submit_path, compile_command, compile_main, - # run_command_args, remove_user_output, remove_ref_output) - - # def teardown(self): - # # Delete the created file. - # super(JavaCodeEvaluator, self).teardown() - # os.remove(self.submit_path) - - # def check_code(self, ref_code_path, submit_code_path, compile_command, - # compile_main, run_command_args, remove_user_output, - # remove_ref_output): - # """ Function validates student code using instructor code as - # reference.The first argument ref_code_path, is the path to - # instructor code, it is assumed to have executable permission. - # The second argument submit_code_path, is the path to the student - # code, it is assumed to have executable permission. - - # Returns - # -------- - - # returns (True, "Correct answer") : If the student function returns - # expected output when called by reference code. - - # returns (False, error_msg): If the student function fails to return - # expected output when called by reference code. - - # Returns (False, error_msg): If mandatory arguments are not files or - # if the required permissions are not given to the file(s). - - # """ - # if not isfile(ref_code_path): - # return False, "No file at %s or Incorrect path" % ref_code_path - # if not isfile(submit_code_path): - # return False, 'No file at %s or Incorrect path' % submit_code_path - - # success = False - # ret = self._compile_command(compile_command) - # proc, stdnt_stderr = ret - # stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # # Only if compilation is successful, the program is executed - # # And tested with testcases - # if stdnt_stderr == '': - # ret = self._compile_command(compile_main) - # proc, main_err = ret - # main_err = self._remove_null_substitute_char(main_err) - - # if main_err == '': - # ret = self._run_command(run_command_args, shell=True, - # stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, stdout, stderr = ret - # if proc.returncode == 0: - # success, err = True, "Correct answer" - # else: - # err = stdout + "\n" + stderr - # os.remove(remove_ref_output) - # else: - # err = "Error:" - # try: - # error_lines = main_err.splitlines() - # for e in error_lines: - # if ':' in e: - # err = err + "\n" + e.split(":", 1)[1] - # else: - # err = err + "\n" + e - # except: - # err = err + "\n" + main_err - # os.remove(remove_user_output) - # else: - # err = "Compilation Error:" - # try: - # error_lines = stdnt_stderr.splitlines() - # for e in error_lines: - # if ':' in e: - # err = err + "\n" + e.split(":", 1)[1] - # else: - # err = err + "\n" + e - # except: - # err = err + "\n" + stdnt_stderr - - # return success, err diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 3205279..398e1aa 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -3,12 +3,8 @@ import importlib import json registry = None - -# def set_registry(): -# global registry -# registry = _LanguageRegistry() -def get_registry(): #@@@get_evaluator_registry +def get_registry(): global registry if registry is None: registry = _LanguageRegistry() @@ -20,11 +16,9 @@ def unpack_json(json_data): def create_evaluator_instance(language, test_case_type, json_data, in_dir): """Create instance of relevant EvaluateCode class based on language""" - # set_registry() registry = get_registry() - cls = registry.get_class(language, test_case_type) #@@@get_evaluator_for_language + cls = registry.get_class(language, test_case_type) instance = cls(in_dir) - # instance = cls.from_json(language, json_data, in_dir) return instance class _LanguageRegistry(object): diff --git a/yaksh/models.py b/yaksh/models.py index fa4121d..a200ae1 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -159,16 +159,6 @@ class Question(models.Model): # Number of points for the question. points = models.FloatField(default=1.0) - # Answer for MCQs. - # test = models.TextField(blank=True) - - # Test cases file paths (comma seperated for reference code path and test case code path) - # Applicable for CPP, C, Java and Scilab - # ref_code_path = models.TextField(blank=True) - - # # Any multiple choice options. Place one option per line. - # options = models.TextField(blank=True) - # The language for question. language = models.CharField(max_length=24, choices=languages) @@ -183,52 +173,12 @@ class Question(models.Model): # when creating a QuestionPaper. active = models.BooleanField(default=True) - # Snippet of code provided to the user. - # snippet = models.CharField(max_length=256) - # Tags for the Question. tags = TaggableManager(blank=True) # user for particular question user = models.ForeignKey(User, related_name="user") - # def consolidate_answer_data(self, test_cases, user_answer): - # def consolidate_answer_data(self, user_answer): - # test_case_data_dict = [] - # question_info_dict = {} - - # for test_case in test_cases: - # kw_args_dict = {} - # pos_args_list = [] - - # test_case_data = {} - # test_case_data['test_id'] = test_case.id - # test_case_data['func_name'] = test_case.func_name - # test_case_data['expected_answer'] = test_case.expected_answer - - # if test_case.kw_args: - # for args in test_case.kw_args.split(","): - # arg_name, arg_value = args.split("=") - # kw_args_dict[arg_name.strip()] = arg_value.strip() - - # if test_case.pos_args: - # for args in test_case.pos_args.split(","): - # pos_args_list.append(args.strip()) - - # test_case_data['kw_args'] = kw_args_dict - # test_case_data['pos_args'] = pos_args_list - # test_case_data_dict.append(test_case_data) - - # question_info_dict['language'] = self.language - # question_info_dict['id'] = self.id - # question_info_dict['user_answer'] = user_answer - # question_info_dict['test_parameter'] = test_case_data_dict - # question_info_dict['ref_code_path'] = self.ref_code_path - # question_info_dict['test'] = self.test - # question_info_dict['test_case_type'] = self.test_case_type - - # return json.dumps(question_info_dict) - def consolidate_answer_data(self, user_answer): question_data = {} test_case_data = [] @@ -239,18 +189,11 @@ class Question(models.Model): test_case_as_dict = test.get_field_value() test_case_data.append(test_case_as_dict) - # test_cases = self.testcase_set.all() - # for test in test_cases: - # test_case_child_instance = test.get_child_instance(self.test_case_type) - # test_case_instance_dict = test_case_child_instance.get_instance_as_dict() - # test_case_data.append(test_case_field_value) - question_data['test_case_data'] = test_case_data question_data['user_answer'] = user_answer return json.dumps(question_data) -<<<<<<< HEAD def dump_into_json(self, question_ids, user): questions = Question.objects.filter(id__in = question_ids, user_id = user.id) questions_dict = [] @@ -277,9 +220,6 @@ class Question(models.Model): return test_cases - # def get_test_case(self, test_case_id): - # test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) - # test_case = test_case_ctype.get_object_for_this_type(question=self, id=test_case_id) def get_test_case(self, **kwargs): test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) test_case = test_case_ctype.get_object_for_this_type(question=self, **kwargs) @@ -801,10 +741,6 @@ class AssignmentUpload(models.Model): class TestCase(models.Model): question = models.ForeignKey(Question, blank=True, null = True) - # def get_child_instance(self, type): - # return getattr(self, type) - - class StandardTestCase(TestCase): test_case = models.TextField(blank=True) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index 82b5912..ff56421 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -25,15 +25,10 @@ class PythonAssertionEvaluator(CodeEvaluator): exec submitted in self.exec_scope return self.exec_scope - # def check_code(self, test, user_answer, ref_code_path): - # def check_code(self, user_answer, test_case_data): #@@@v2 def check_code(self, user_answer, test_case): success = False try: tb = None - # submitted = compile(user_answer, '', mode='exec') - # g = {} - # exec submitted in g _tests = compile(test_case, '', mode='exec') exec _tests in self.exec_scope except AssertionError: diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index 4d89e16..b967024 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -40,12 +40,6 @@ class PythonStdoutEvaluator(CodeEvaluator): success = False tb = None - # submitted = compile(user_answer, '', mode='exec') - # with redirect_stdout() as output_buffer: - # g = {} - # exec submitted in g - # raw_output_value = output_buffer.getvalue() - # output_value = raw_output_value.strip() if expected_output in user_answer: success = False err = "Incorrect Answer: Please avoid printing the expected output directly" diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index fedfab3..3af9782 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -12,16 +12,6 @@ from code_evaluator import CodeEvaluator class ScilabCodeEvaluator(CodeEvaluator): """Tests the Scilab code obtained from Code Server""" - # def __init__(self, test_case_data, test, language, user_answer, - # ref_code_path=None, in_dir=None): - # super(ScilabCodeEvaluator, self).__init__(test_case_data, test, - # language, user_answer, - # ref_code_path, in_dir) - - # # Removes all the commands that terminates scilab - # self.user_answer, self.terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - # self.test_case_args = self.setup() - def setup(self): super(ScilabCodeEvaluator, self).setup() # ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) @@ -33,30 +23,6 @@ class ScilabCodeEvaluator(CodeEvaluator): # Delete the created file. os.remove(self.submit_code_path) - # def compile_code(self, user_answer, test_case): - # if hasattr(self, 'compiled_output'): - # return None - # else: - # ref_code_path = test_case - # clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - # user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) - - # self.write_to_submit_code_file(self.submit_code_path, user_answer) - # # Throw message if there are commmands that terminates scilab - # self.add_err = "" - # if terminate_commands: - # self.add_err = "Please do not use exit, quit and abort commands in your\ - # code.\n Otherwise your code will not be evaluated\ - # correctly.\n" - - # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) - # cmd += ' | timeout 8 scilab-cli -nb' - # self.compiled_output = self._run_command(cmd, - # shell=True, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # return self.compiled_output - def check_code(self, user_answer, test_case): ref_code_path = test_case clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) @@ -89,46 +55,12 @@ class ScilabCodeEvaluator(CodeEvaluator): if proc.returncode == 5: success, err = True, "Correct answer" else: - # err = self.add_err + stdout err = add_err + stdout else: - # err = self.add_err + stderr err = add_err + stderr return success, err - # def check_code(self, ref_path): - # success = False - - # # Throw message if there are commmands that terminates scilab - # add_err="" - # if self.terminate_commands: - # add_err = "Please do not use exit, quit and abort commands in your\ - # code.\n Otherwise your code will not be evaluated\ - # correctly.\n" - - # cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(ref_path) - # cmd += ' | timeout 8 scilab-cli -nb' - # ret = self._run_command(cmd, - # shell=True, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # proc, stdout, stderr = ret - - # # Get only the error. - # stderr = self._get_error(stdout) - # if stderr is None: - # # Clean output - # stdout = self._strip_output(stdout) - # if proc.returncode == 5: - # success, err = True, "Correct answer" - # else: - # err = add_err + stdout - # else: - # err = add_err + stderr - - # return success, err - def _remove_scilab_exit(self, string): """ Removes exit, quit and abort from the scilab code diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index bc700d1..eb9419c 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -24,11 +24,7 @@ Points:{{ form.points }}{{ form.points.errors }} Rendered:

Description: {{ form.description}} {{form.description.errors}} - Tags: {{ form.tags }} - - - Test Case Type: {{ form.test_case_type }}{{ form.test_case_type.errors }}
diff --git a/yaksh/tests.py b/yaksh/tests.py index 900a261..6e50fc5 100644 --- a/yaksh/tests.py +++ b/yaksh/tests.py @@ -107,11 +107,9 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(self.question1.summary, 'Demo question') self.assertEqual(self.question1.language, 'Python') self.assertEqual(self.question1.type, 'Code') - # self.assertFalse(self.question.options) self.assertEqual(self.question1.description, 'Write a function') self.assertEqual(self.question1.points, 1.0) self.assertTrue(self.question1.active) - # self.assertEqual(self.question.snippet, 'def myfunc()') tag_list = [] for tag in self.question1.tags.all(): tag_list.append(tag.name) @@ -149,27 +147,6 @@ class QuestionTestCases(unittest.TestCase): self.assertTrue(question_data.active) self.assertEqual(question_data.snippet, 'def fact()') -############################################################################### -# class TestCaseTestCases(unittest.TestCase): -# def setUp(self): -# self.question = Question(summary='Demo question', language='Python', -# type='Code', active=True, -# description='Write a function', points=1.0, -# snippet='def myfunc()') -# self.question.save() -# self.testcase = TestCase(question=self.question, -# func_name='def myfunc', kw_args='a=10,b=11', -# pos_args='12,13', expected_answer='15') - -# def test_testcase(self): -# """ Test question """ -# self.assertEqual(self.testcase.question, self.question) -# self.assertEqual(self.testcase.func_name, 'def myfunc') -# self.assertEqual(self.testcase.kw_args, 'a=10,b=11') -# self.assertEqual(self.testcase.pos_args, '12,13') -# self.assertEqual(self.testcase.expected_answer, '15') - - ############################################################################### class QuizTestCases(unittest.TestCase): def setUp(self): diff --git a/yaksh/views.py b/yaksh/views.py index 59a51aa..ffb08d8 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -154,11 +154,6 @@ def add_question(request): question_form = QuestionForm(request.POST) if question_form.is_valid(): new_question = question_form.save() - # tags = question_form['tags'].data.split(',') - # for i in range(0, len(tags)): - # tag = tags[i].strip() - # new_question.tags.add(tag) - # new_question.save() return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: return my_render_to_response('yaksh/add_question.html', @@ -185,11 +180,6 @@ def edit_question(request, question_id=None): question_form = QuestionForm(request.POST, instance=question_instance) if question_form.is_valid(): new_question = question_form.save(commit=False) - # tags = question_form['tags'].data.split(',') - # for i in range(0, len(tags)): - # tag = tags[i].strip() - # new_question.tags.add(tag) - # new_question.save() test_case_type = question_form.cleaned_data.get('test_case_type') test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) test_case_model_class = get_model_class(test_case_type) @@ -198,7 +188,6 @@ def edit_question(request, question_id=None): if test_case_formset.is_valid(): new_question.save() test_case_formset.save() - # return my_redirect("/exam/manage/questions") return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: return my_render_to_response('yaksh/add_question.html', @@ -220,186 +209,6 @@ def edit_question(request, question_id=None): 'question_id': question_id}, context_instance=ci) -# def add_question(request, question_id=None): -# def add_question(request, question_id=None): -# """To add a new question in the database. -# Create a new question and store it.""" -# user = request.user -# ci = RequestContext(request) -# if not user.is_authenticated() or not is_moderator(user): -# raise Http404('You are not allowed to view this page!') -# if request.method == "POST": -# form = QuestionForm(request.POST) -# if form.is_valid(): -# if question_id is None: -# if 'save_question' in request.POST: -# form.save() -# question = Question.objects.order_by("-id")[0] -# tags = form['tags'].data.split(',') -# for i in range(0, len(tags)-1): -# tag = tags[i].strip() -# question.tags.add(tag) - -# return my_redirect("/exam/manage/questions") - -# return my_render_to_response('yaksh/add_question.html', -# # {'form': form}, -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) - -# else: -# d = Question.objects.get(id=question_id) -# if 'save_question' in request.POST: -# d.summary = form['summary'].data -# d.description = form['description'].data -# d.points = form['points'].data -# # d.options = form['options'].data -# d.type = form['type'].data -# d.active = form['active'].data -# d.language = form['language'].data -# # d.snippet = form['snippet'].data -# # d.ref_code_path = form['ref_code_path'].data -# # d.test = form['test'].data -# d.save() -# question = Question.objects.get(id=question_id) -# for tag in question.tags.all(): -# question.tags.remove(tag) -# tags = form['tags'].data.split(',') -# for i in range(0, len(tags)-1): -# tag = tags[i].strip() -# question.tags.add(tag) - -# return my_redirect("/exam/manage/questions") - -# return my_render_to_response('yaksh/add_question.html', -# # {'form': form}, -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) - -# else: -# return my_render_to_response('yaksh/add_question.html', -# # {'form': form}, -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) -# else: -# form = QuestionForm() -# if question_id is None: -# form = QuestionForm() -# return my_render_to_response('yaksh/add_question.html', -# # {'form': form}, -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) -# else: -# d = Question.objects.get(id=question_id) -# form = QuestionForm() -# form.initial['summary'] = d.summary -# form.initial['description'] = d.description -# form.initial['points'] = d.points -# # form.initial['options'] = d.options -# form.initial['type'] = d.type -# form.initial['active'] = d.active -# form.initial['language'] = d.language -# # form.initial['snippet'] = d.snippet -# # form.initial['ref_code_path'] = d.ref_code_path -# # form.initial['test'] = d.test -# form_tags = d.tags.all() -# form_tags_split = form_tags.values('name') -# initial_tags = "" -# for tag in form_tags_split: -# initial_tags = initial_tags + str(tag['name']).strip() + "," -# if (initial_tags == ","): -# initial_tags = "" -# form.initial['tags'] = initial_tags - -# return my_render_to_response('yaksh/add_question.html', -# # {'form': form}, -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) - -# @login_required -# def show_testcase(request, question_id=None): -# """Show all test cases related to Questions""" - -# user = request.user -# ci = RequestContext(request) -# if not user.is_authenticated() or not is_moderator(user): -# raise Http404('You are not allowed to view this page!') -# if not question_id: -# raise Http404('No Question Found') - -# question = Question.objects.get(id=question_id) -# test_cases = question.get_test_cases() - -# if request.POST.get('delete') == 'delete': -# data = request.POST.getlist('test_case') -# for i in data: -# for t in test_cases: -# if int(i) == t.id: -# test_case_deleted = t.delete() -# test_cases = question.get_test_cases() - -# return my_render_to_response('yaksh/show_testcase.html', -# {'test_cases': test_cases, -# 'question_id': question_id}, -# context_instance=ci) - - -# @login_required -# def add_testcase(request, question_id=None, test_case_id=None): -# """To add new test case for a question""" - -# user = request.user -# ci = RequestContext(request) -# if not user.is_authenticated() or not is_moderator(user): -# raise Http404('You are not allowed to view this page!') -# if not question_id: -# raise Http404('No Question Found') - -# question = Question.objects.get(id=question_id) -# test_case_type = question.test_case_type - -# if test_case_id: -# instance = question.get_test_case(test_case_id) -# else: -# instance = None - -# # test_cases = self.testcase_set.all() -# # for test in test_cases: -# # test_case_child_instance = test.get_child_instance(test_case_type) - -# test_case_form_object = get_object_form(model=test_case_type, exclude_fields=['question']) - -# # if test_case_type == "standardtestcase": -# # from yaksh.forms import StandardTestCaseForm -# # if request.method == "POST": -# # form = StandardTestCaseForm(request.POST) -# # form = StandardTestCaseForm(initial) - -# if request.method == "POST": -# form = test_case_form_object(request.POST, instance=instance) -# if form.is_valid(): -# form_data = form.save(commit=False) -# form_data.question = question -# form_data.save() -# return my_redirect("/exam/manage/showtestcase/{0}".format(question_id)) -# else: -# return my_render_to_response('yaksh/add_testcase.html', -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) - -# else: -# form = test_case_form_object(initial={"question": question}, instance=instance) -# return my_render_to_response('yaksh/add_testcase.html', -# {'form': form, -# 'question_id': question_id}, -# context_instance=ci) - @login_required def add_quiz(request, quiz_id=None): """To add a new quiz in the database. @@ -669,36 +478,6 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None): return show_question(request, question, paper) -# def validate_answer(user, user_answer, question, json_data=None): -# """ -# Checks whether the answer submitted by the user is right or wrong. -# If right then returns correct = True, success and -# message = Correct answer. -# success is True for MCQ's and multiple correct choices because -# only one attempt are allowed for them. -# For code questions success is True only if the answer is correct. -# """ - -# result = {'success': True, 'error': 'Incorrect answer'} -# correct = False - -# if user_answer is not None: -# if question.type == 'mcq': -# if user_answer.strip() == question.test.strip(): -# correct = True -# elif question.type == 'mcc': -# answers = set(question.test.splitlines()) -# if set(user_answer) == answers: -# correct = True -# elif question.type == 'code': -# user_dir = get_user_dir(user) -# json_result = code_server.run_code(question.language, question.test_case_type, json_data, user_dir) -# result = json.loads(json_result) -# if result.get('success'): -# correct = True - -# return correct, result - def validate_answer(user, user_answer, question, json_data=None): """ Checks whether the answer submitted by the user is right or wrong. @@ -721,7 +500,6 @@ def validate_answer(user, user_answer, question, json_data=None): expected_answers = [] for opt in question.get_test_cases(correct=True): expected_answers.append(opt.options) - # answers = set(question.test.splitlines()) if set(user_answer) == set(expected_answers): correct = True elif question.type == 'code': -- cgit From d953f6f9e62671eeb5d6ea6498475167301dfe91 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 2 May 2016 14:38:13 +0530 Subject: - Fix tests - Fix minor views.py - Fix minor errors caused due to conflict resolution, rebasing - Fix errors in forms, views - Minor Button label change in addquestion.html - Add snippet to question - Remove commented code --- yaksh/code_evaluator.py | 61 +++---------------------- yaksh/evaluator_tests/test_bash_evaluation.py | 40 ---------------- yaksh/evaluator_tests/test_code_evaluation.py | 25 ---------- yaksh/evaluator_tests/test_java_evaluation.py | 43 ----------------- yaksh/evaluator_tests/test_python_evaluation.py | 3 -- yaksh/evaluator_tests/test_scilab_evaluation.py | 20 -------- yaksh/forms.py | 11 ----- yaksh/models.py | 8 ++-- yaksh/python_assertion_evaluator.py | 2 - yaksh/scilab_code_evaluator.py | 4 -- yaksh/templates/yaksh/add_question.html | 2 +- yaksh/tests.py | 59 ++++++++++++++---------- yaksh/urls.py | 2 +- yaksh/views.py | 9 +++- 14 files changed, 57 insertions(+), 232 deletions(-) diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index b7a523c..1c11c00 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -5,14 +5,13 @@ import os import stat from os.path import isdir, dirname, abspath, join, isfile, exists import signal +import traceback from multiprocessing import Process, Queue import subprocess import re -# import json # Local imports. from settings import SERVER_TIMEOUT - MY_DIR = abspath(dirname(__file__)) @@ -50,36 +49,12 @@ def delete_signal_handler(): class CodeEvaluator(object): """Tests the code obtained from Code Server""" - # def __init__(self, test_case_data, test, language, user_answer, - # ref_code_path=None, in_dir=None): def __init__(self, in_dir=None): msg = 'Code took more than %s seconds to run. You probably '\ 'have an infinite loop in your code.' % SERVER_TIMEOUT self.timeout_msg = msg - # self.test_case_data = test_case_data - # self.language = language.lower() #@@@remove - # self.user_answer = user_answer #@@@specific to check-code - # self.ref_code_path = ref_code_path #@@@specific to check-code - # self.test = test #@@@specific to check-code - self.in_dir = in_dir #@@@Common for all, no change - # self.test_case_args = None #@@@no change - - # Public Protocol ########## - # @classmethod - # def from_json(cls, language, json_data, in_dir): - # json_data = json.loads(json_data) - # # test_case_data = json_data.get("test_case_data") - # user_answer = json_data.get("user_answer") - # ref_code_path = json_data.get("ref_code_path") - # test = json_data.get("test") - - # # instance = cls(test_case_data, test, language, user_answer, ref_code_path, - # # in_dir) - # instance = cls(test, language, user_answer, ref_code_path, - # in_dir) - # return instance - - # def evaluate(self): + self.in_dir = in_dir + def evaluate(self, **kwargs): """Evaluates given code with the test cases based on given arguments in test_case_data. @@ -103,7 +78,6 @@ class CodeEvaluator(object): """ self.setup() - # success, err = self.safe_evaluate(self.test_case_args) success, err = self.safe_evaluate(**kwargs) self.teardown() @@ -114,18 +88,13 @@ class CodeEvaluator(object): def setup(self): self._change_dir(self.in_dir) - # def safe_evaluate(self, args): - # def safe_evaluate(self, **kwargs): #@@@v2 def safe_evaluate(self, user_answer, test_case_data): # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False - # args = args or [] # Do whatever testing needed. try: - # success, err = self.check_code(*args) - # success, err = self.check_code(**kwargs) #@@@v2 for test_case in test_case_data: self.compile_code(user_answer, **test_case) success, err = self.check_code(user_answer, **test_case) @@ -134,9 +103,9 @@ class CodeEvaluator(object): except TimeoutException: err = self.timeout_msg - except: - _type, value = sys.exc_info()[:2] - err = "Error: {0}".format(repr(value)) + except Exception: + err = "Error: {0}".format(traceback.format_exc(limit=0)) + finally: # Set back any original signal handler. set_original_signal_handler(prev_handler) @@ -198,24 +167,6 @@ class CodeEvaluator(object): raise return proc, stdout, stderr - # def _compile_command(self, cmd, *args, **kw): - # """Compiles C/C++/java code and returns errors if any. - # Run a command in a subprocess while blocking, the process is killed - # if it takes more than 2 seconds to run. Return the Popen object, the - # stderr. - # """ - # try: - # proc_compile = subprocess.Popen(cmd, shell=True, stdin=None, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE) - # out, err = proc_compile.communicate() - # except TimeoutException: - # # Runaway code, so kill it. - # proc_compile.kill() - # # Re-raise exception. - # raise - # return proc_compile, err - def _change_dir(self, in_dir): if in_dir is not None and isdir(in_dir): os.chdir(in_dir) diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 39247f7..7c58c43 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -10,15 +10,6 @@ class BashEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - # def setUp(self): - # self.language = "bash" - # self.ref_code_path = "bash_files/sample.sh,bash_files/sample.args" - # self.in_dir = "/tmp" - # self.test_case_data = [] - # self.timeout_msg = ("Code took more than {0} seconds to run. " - # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - # self.test = None - def test_correct_answer(self): user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" get_class = BashCodeEvaluator(self.in_dir) @@ -49,37 +40,6 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) - # def test_infinite_loop(self): - # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" - # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertFalse(result.get("success")) - # self.assertEquals(result.get("error"), self.timeout_msg) - - # def test_correct_answer(self): - # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" - # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertTrue(result.get("success")) - # self.assertEqual(result.get("error"), "Correct answer") - - # def test_error(self): - # user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" - # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertFalse(result.get("success")) - # self.assertTrue("Error" in result.get("error")) - - # def test_infinite_loop(self): - # user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" - # get_class = BashCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertFalse(result.get("success")) - # self.assertEquals(result.get("error"), self.timeout_msg) if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index e4f129c..ace6115 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -23,28 +23,3 @@ class RegistryTestCase(unittest.TestCase): if __name__ == '__main__': unittest.main() - - -# import unittest -# import os -# from yaksh import cpp_code_evaluator -# from yaksh.language_registry import _LanguageRegistry, get_registry -# from yaksh.settings import SERVER_TIMEOUT - - -# class RegistryTestCase(unittest.TestCase): -# def setUp(self): -# self.registry_object = get_registry() -# self.language_registry = _LanguageRegistry() - -# def test_set_register(self): -# class_name = getattr(cpp_code_evaluator, 'CppCodeEvaluator') -# self.registry_object.register("c", {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}) -# self.assertEquals(self.registry_object.get_class("c", "standardtestcase"), class_name) - -# def tearDown(self): -# self.registry_object = None - - -# if __name__ == '__main__': -# unittest.main() diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index eacd62e..76a3fcf 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -57,49 +57,6 @@ class JavaEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg) - # def setUp(self): - # self.language = "java" - # self.ref_code_path = "java_files/main_square.java" - # self.in_dir = "/tmp" - # self.test_case_data = [] - # evaluator.SERVER_TIMEOUT = 9 - # self.timeout_msg = ("Code took more than {0} seconds to run. " - # "You probably have an infinite loop in " - # "your code.").format(evaluator.SERVER_TIMEOUT) - # self.test = None - - # def tearDown(self): - # evaluator.SERVER_TIMEOUT = 2 - - # def test_correct_answer(self): - # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a;\n\t}\n}" - # get_class = JavaCodeEvaluator(self.test_case_data, self.test, - # self.language, user_answer, - # self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertTrue(result.get("success")) - # self.assertEqual(result.get("error"), "Correct answer") - - # def test_error(self): - # user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a*a" - # get_class = JavaCodeEvaluator(self.test_case_data, self.test, - # self.language, user_answer, - # self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertFalse(result.get("success")) - # self.assertTrue("Error" in result.get("error")) - - # def test_infinite_loop(self): - # user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" - # get_class = JavaCodeEvaluator(self.test_case_data, self.test, - # self.language, user_answer, - # self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertFalse(result.get("success")) - # self.assertEquals(result.get("error"), self.timeout_msg) if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 39d4723..f6ac0bf 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -16,7 +16,6 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - # {u'user_answer': u'def adder(a,b):\r\n return a', u'test_case_data': [u'assert(adder(1,2)==3)']} user_answer = "def add(a,b):\n\treturn a + b" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, @@ -156,7 +155,6 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - # user_answer = "a = 'Hello'\nb = 'World'\nprint '{0} {1}'.format(a, b)" user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, @@ -197,7 +195,6 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), 'Incorrect Answer') ->>>>>>> - Add test cases for multiple python evaluators if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 928cd7e..24d6740 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -11,16 +11,6 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - # def setUp(self): - # self.language = "scilab" - # self.ref_code_path = "scilab_files/test_add.sce" - # self.in_dir = "/tmp" - # self.test_case_data = [] - # self.timeout_msg = ("Code took more than {0} seconds to run. " - # "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) - # self.test = None - - def test_correct_answer(self): user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" get_class = ScilabCodeEvaluator(self.in_dir) @@ -31,16 +21,6 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) - # def test_correct_answer(self): - # user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" - # get_class = ScilabCodeEvaluator(self.test_case_data, self.test, - # self.language, user_answer, - # self.ref_code_path, self.in_dir) - # result = get_class.evaluate() - - # self.assertTrue(result.get("success")) - # self.assertEqual(result.get("error"), "Correct answer") - def test_error(self): user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction" get_class = ScilabCodeEvaluator(self.in_dir) diff --git a/yaksh/forms.py b/yaksh/forms.py index 2ce2cba..808262b 100644 --- a/yaksh/forms.py +++ b/yaksh/forms.py @@ -224,14 +224,3 @@ class ProfileForm(forms.ModelForm): class UploadFileForm(forms.Form): file = forms.FileField() - -class StandardTestCaseForm(forms.ModelForm): - class Meta: - model = StandardTestCase - fields = ['test_case'] - - -class StdoutBasedTestCaseForm(forms.ModelForm): - class Meta: - model = StdoutBasedTestCase - fields = ['output'] diff --git a/yaksh/models.py b/yaksh/models.py index a200ae1..dd2fb5f 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -176,6 +176,9 @@ class Question(models.Model): # Tags for the Question. tags = TaggableManager(blank=True) + # Snippet of code provided to the user. + snippet = models.CharField(max_length=256, blank=True) + # user for particular question user = models.ForeignKey(User, related_name="user") @@ -199,10 +202,9 @@ class Question(models.Model): questions_dict = [] for question in questions: q_dict = {'summary': question.summary, 'description': question.description, - 'points': question.points, 'test': question.test, - 'ref_code_path': question.ref_code_path, - 'options': question.options, 'language': question.language, + 'points': question.points, 'language': question.language, 'type': question.type, 'active': question.active, + 'test_case_type': question.test_case_type, 'snippet': question.snippet} questions_dict.append(q_dict) diff --git a/yaksh/python_assertion_evaluator.py b/yaksh/python_assertion_evaluator.py index ff56421..bf6a4be 100644 --- a/yaksh/python_assertion_evaluator.py +++ b/yaksh/python_assertion_evaluator.py @@ -39,8 +39,6 @@ class PythonAssertionEvaluator(CodeEvaluator): err = "{0} {1} in: {2}".format(type.__name__, str(value), text) except TimeoutException: raise - except Exception: - err = traceback.format_exc(limit=0) else: success = True err = 'Correct answer' diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index 3af9782..f4aa5f8 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -14,9 +14,7 @@ class ScilabCodeEvaluator(CodeEvaluator): """Tests the Scilab code obtained from Code Server""" def setup(self): super(ScilabCodeEvaluator, self).setup() - # ref_path, test_case_path = self._set_test_code_file_path(self.ref_code_path) self.submit_code_path = self.create_submit_code_file('function.sci') - # return ref_path, # Return as a tuple def teardown(self): super(ScilabCodeEvaluator, self).teardown() @@ -43,8 +41,6 @@ class ScilabCodeEvaluator(CodeEvaluator): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # success = False - # proc, stdout, stderr = self.compiled_output proc, stdout, stderr = ret # Get only the error. diff --git a/yaksh/templates/yaksh/add_question.html b/yaksh/templates/yaksh/add_question.html index eb9419c..d38aa1c 100644 --- a/yaksh/templates/yaksh/add_question.html +++ b/yaksh/templates/yaksh/add_question.html @@ -40,7 +40,7 @@
- +
diff --git a/yaksh/tests.py b/yaksh/tests.py index 6e50fc5..150a8b0 100644 --- a/yaksh/tests.py +++ b/yaksh/tests.py @@ -79,27 +79,23 @@ class QuestionTestCases(unittest.TestCase): type='Code', active=True, test_case_type='standardtestcase', description='Write a function', points=1.0, - user=self.user1) + snippet='def myfunc()', user=self.user1) self.question1.save() self.question2 = Question(summary='Demo Json', language='python', type='code', active=True, description='factorial of a no', points=2.0, - user=self.user2) + snippet='def fact()', user=self.user2) self.question2.save() self.question1.tags.add('python', 'function') self.assertion_testcase = StandardTestCase(question=self.question1, test_case='assert myfunc(12, 13) == 15') - answer_data = {"user_answer": "demo_answer", - "test_case_data": ["assert myfunc(12, 13) == 15"], - } - self.answer_data_json = json.dumps(answer_data) self.user_answer = "demo_answer" questions_data = [{"snippet": "def fact()", "active": True, "points": 1.0, - "ref_code_path": "", "description": "factorial of a no", - "language": "Python", "test": "", "type": "Code", - "options": "", "summary": "Json Demo"}] + "description": "factorial of a no", + "language": "Python", "type": "Code", + "summary": "Json Demo"}] self.json_questions_data = json.dumps(questions_data) def test_question(self): @@ -110,16 +106,12 @@ class QuestionTestCases(unittest.TestCase): self.assertEqual(self.question1.description, 'Write a function') self.assertEqual(self.question1.points, 1.0) self.assertTrue(self.question1.active) + self.assertEqual(self.question1.snippet, 'def myfunc()') tag_list = [] for tag in self.question1.tags.all(): tag_list.append(tag.name) self.assertEqual(tag_list, ['python', 'function']) - def test_consolidate_answer_data(self): - """ Test consolidate_answer_data function """ - result = self.question1.consolidate_answer_data(self.user_answer) - self.assertEqual(result, self.answer_data_json) - def test_dump_questions_into_json(self): """ Test dump questions into json """ question = Question() @@ -138,7 +130,7 @@ class QuestionTestCases(unittest.TestCase): """ Test load questions into database from json """ question = Question() result = question.load_from_json(self.json_questions_data, self.user1) - question_data = Question.objects.get(pk=27) + question_data = Question.objects.get(pk=25) self.assertEqual(question_data.summary, 'Json Demo') self.assertEqual(question_data.language, 'Python') self.assertEqual(question_data.type, 'Code') @@ -479,22 +471,43 @@ class CourseTestCases(unittest.TestCase): ############################################################################### class TestCaseTestCases(unittest.TestCase): def setUp(self): - self.question = Question(summary='Demo question', language='Python', + self.user = User.objects.get(pk=1) + self.question1 = Question(summary='Demo question 1', language='Python', type='Code', active=True, description='Write a function', points=1.0, + test_case_type="standardtestcase", user=self.user, + snippet='def myfunc()' + ) + self.question2 = Question(summary='Demo question 2', language='Python', + type='Code', active=True, + description='Write to standard output', points=1.0, + test_case_type="stdoutbasedtestcase", user=self.user, + snippet='def myfunc()' ) - self.question.save() - self.assertion_testcase = StandardTestCase(question=self.question, + self.question1.save() + self.question2.save() + self.assertion_testcase = StandardTestCase(question=self.question1, test_case='assert myfunc(12, 13) == 15') - self.stdout_based_testcase = StdoutBasedTestCase(question=self.question, - output='Hello World') + self.stdout_based_testcase = StdoutBasedTestCase(question=self.question2, + expected_output='Hello World') + self.assertion_testcase.save() + self.stdout_based_testcase.save() + answer_data = {"user_answer": "demo_answer", + "test_case_data": [{"test_case": "assert myfunc(12, 13) == 15"}], + } + self.answer_data_json = json.dumps(answer_data) def test_assertion_testcase(self): """ Test question """ - self.assertEqual(self.assertion_testcase.question, self.question) + self.assertEqual(self.assertion_testcase.question, self.question1) self.assertEqual(self.assertion_testcase.test_case, 'assert myfunc(12, 13) == 15') def test_stdout_based_testcase(self): """ Test question """ - self.assertEqual(self.stdout_based_testcase.question, self.question) - self.assertEqual(self.stdout_based_testcase.output, 'Hello World') + self.assertEqual(self.stdout_based_testcase.question, self.question2) + self.assertEqual(self.stdout_based_testcase.expected_output, 'Hello World') + + def test_consolidate_answer_data(self): + """ Test consolidate answer data model method """ + result = self.question1.consolidate_answer_data(user_answer="demo_answer") + self.assertEqual(result, self.answer_data_json) \ No newline at end of file diff --git a/yaksh/urls.py b/yaksh/urls.py index 6a33888..feac8c1 100644 --- a/yaksh/urls.py +++ b/yaksh/urls.py @@ -43,7 +43,7 @@ urlpatterns += [ url(r'^manage/addquestion/$', views.add_question), url(r'^manage/addquestion/(?P\d+)/$', views.edit_question), url(r'^manage/addquiz/$', views.add_quiz), - url(r'^manage/addquiz/(?P\d+)/$', views.edit_quiz), + url(r'^manage/addquiz/(?P\d+)/$', views.add_quiz), url(r'^manage/gradeuser/$', views.grade_user), url(r'^manage/gradeuser/(?P\d+)/$',views.grade_user), url(r'^manage/gradeuser/(?P\d+)/(?P\d+)/$',views.grade_user), diff --git a/yaksh/views.py b/yaksh/views.py index ffb08d8..2a3adbf 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -153,7 +153,9 @@ def add_question(request): if request.method == "POST" and 'save_question' in request.POST: question_form = QuestionForm(request.POST) if question_form.is_valid(): - new_question = question_form.save() + new_question = question_form.save(commit=False) + new_question.user = user + new_question.save() return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: return my_render_to_response('yaksh/add_question.html', @@ -190,6 +192,11 @@ def edit_question(request, question_id=None): test_case_formset.save() return my_redirect("/exam/manage/addquestion/{0}".format(new_question.id)) else: + test_case_type = question_form.cleaned_data.get('test_case_type') + test_case_form_class = get_object_form(model=test_case_type, exclude_fields=['question']) + test_case_model_class = get_model_class(test_case_type) + TestCaseInlineFormSet = inlineformset_factory(Question, test_case_model_class, form=test_case_form_class, extra=1) + test_case_formset = TestCaseInlineFormSet(request.POST, request.FILES, instance=question_instance) return my_render_to_response('yaksh/add_question.html', {'form': question_form, 'test_case_formset': test_case_formset, -- cgit From c384c60c6d7fb5d30f3f929c518e0b41e084c4c4 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 9 May 2016 13:00:04 +0530 Subject: - Adhere to 80 columns - add docstrings - Fix further tests --- yaksh/admin.py | 4 +- yaksh/bash_code_evaluator.py | 68 +++++++---- yaksh/code_evaluator.py | 5 + yaksh/code_server.py | 7 +- yaksh/cpp_code_evaluator.py | 148 +++++++++++----------- yaksh/evaluator_tests/test_bash_evaluation.py | 17 ++- yaksh/evaluator_tests/test_c_cpp_evaluation.py | 7 +- yaksh/evaluator_tests/test_code_evaluation.py | 35 +++++- yaksh/evaluator_tests/test_java_evaluation.py | 7 +- yaksh/evaluator_tests/test_python_evaluation.py | 107 ++++++++++------ yaksh/evaluator_tests/test_scilab_evaluation.py | 15 ++- yaksh/java_code_evaluator.py | 47 ++++--- yaksh/language_registry.py | 1 + yaksh/models.py | 52 +++++--- yaksh/python_stdout_evaluator.py | 4 +- yaksh/scilab_code_evaluator.py | 20 +-- yaksh/settings.py | 18 +-- yaksh/tests.py | 155 ++++++++++++++++-------- 18 files changed, 463 insertions(+), 254 deletions(-) diff --git a/yaksh/admin.py b/yaksh/admin.py index d223cd4..c31b99b 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -1,5 +1,5 @@ -from yaksh.models import Question, Quiz, TestCase,\ - StandardTestCase, StdoutBasedTestCase +from yaksh.models import Question, Quiz +from yaksh.models import TestCase, StandardTestCase, StdoutBasedTestCase from django.contrib import admin admin.site.register(Question) diff --git a/yaksh/bash_code_evaluator.py b/yaksh/bash_code_evaluator.py index 12177f9..a0af0e2 100644 --- a/yaksh/bash_code_evaluator.py +++ b/yaksh/bash_code_evaluator.py @@ -50,41 +50,52 @@ class BashCodeEvaluator(CodeEvaluator): get_ref_path, get_test_case_path = ref_code_path.strip().split(',') get_ref_path = get_ref_path.strip() get_test_case_path = get_test_case_path.strip() - clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(get_ref_path, - get_test_case_path) + clean_ref_code_path, clean_test_case_path = \ + self._set_test_code_file_path(get_ref_path, get_test_case_path) if not isfile(clean_ref_code_path): - return False, "No file at %s or Incorrect path" % clean_ref_code_path + msg = "No file at %s or Incorrect path" % clean_ref_code_path + return False, msg if not isfile(self.submit_code_path): - return False, "No file at %s or Incorrect path" % self.submit_code_path + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg if not os.access(clean_ref_code_path, os.X_OK): - return False, "Script %s is not executable" % clean_ref_code_path + msg = "Script %s is not executable" % clean_ref_code_path + return False, msg if not os.access(self.submit_code_path, os.X_OK): - return False, "Script %s is not executable" % self.submit_code_path + msg = "Script %s is not executable" % self.submit_code_path + return False, msg success = False self.write_to_submit_code_file(self.submit_code_path, user_answer) if clean_test_case_path is None or "": - ret = self._run_command(clean_ref_code_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ret = self._run_command(clean_ref_code_path, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) proc, inst_stdout, inst_stderr = ret - ret = self._run_command(self.submit_code_path, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ret = self._run_command(self.submit_code_path, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) proc, stdnt_stdout, stdnt_stderr = ret if inst_stdout == stdnt_stdout: return True, "Correct answer" else: err = "Error: expected %s, got %s" % (inst_stderr, - stdnt_stderr) + stdnt_stderr + ) return False, err else: if not isfile(clean_test_case_path): - return False, "No test case at %s" % clean_test_case_path + msg = "No test case at %s" % clean_test_case_path + return False, msg if not os.access(clean_ref_code_path, os.R_OK): - return False, "Test script %s, not readable" % clean_test_case_path + msg = "Test script %s, not readable" % clean_test_case_path + return False, msg # valid_answer is True, so that we can stop once a test case fails valid_answer = True # loop_count has to be greater than or equal to one. @@ -95,20 +106,27 @@ class BashCodeEvaluator(CodeEvaluator): for test_case in test_cases: loop_count += 1 if valid_answer: - args = [clean_ref_code_path] + [x for x in test_case.split()] - ret = self._run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + args = [clean_ref_code_path] + \ + [x for x in test_case.split()] + ret = self._run_command(args, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) proc, inst_stdout, inst_stderr = ret - args = [self.submit_code_path]+[x for x in test_case.split()] - ret = self._run_command(args, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + args = [self.submit_code_path] + \ + [x for x in test_case.split()] + ret = self._run_command(args, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) proc, stdnt_stdout, stdnt_stderr = ret valid_answer = inst_stdout == stdnt_stdout if valid_answer and (num_lines == loop_count): return True, "Correct answer" else: - err = "Error:expected %s, got %s" % (inst_stdout+inst_stderr, - stdnt_stdout+stdnt_stderr) + err = ("Error:expected" + " %s, got %s").format(inst_stdout+inst_stderr, + stdnt_stdout+stdnt_stderr + ) return False, err diff --git a/yaksh/code_evaluator.py b/yaksh/code_evaluator.py index 1c11c00..aab99eb 100644 --- a/yaksh/code_evaluator.py +++ b/yaksh/code_evaluator.py @@ -89,6 +89,11 @@ class CodeEvaluator(object): self._change_dir(self.in_dir) def safe_evaluate(self, user_answer, test_case_data): + """ + Handles code evaluation along with compilation, signal handling + and Exception handling + """ + # Add a new signal handler for the execution of this code. prev_handler = create_signal_handler() success = False diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 2397582..2d8567e 100755 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -62,8 +62,11 @@ class CodeServer(object): """Calls relevant EvaluateCode class based on language to check the answer code """ - code_evaluator = create_evaluator_instance(language, test_case_type, json_data, - in_dir) + code_evaluator = create_evaluator_instance(language, + test_case_type, + json_data, + in_dir + ) data = unpack_json(json_data) result = code_evaluator.evaluate(**data) diff --git a/yaksh/cpp_code_evaluator.py b/yaksh/cpp_code_evaluator.py index becf371..b869442 100644 --- a/yaksh/cpp_code_evaluator.py +++ b/yaksh/cpp_code_evaluator.py @@ -43,91 +43,103 @@ class CppCodeEvaluator(CodeEvaluator): return None else: ref_code_path = test_case - clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + clean_ref_code_path, clean_test_case_path = \ + self._set_test_code_file_path(ref_code_path) if not isfile(clean_ref_code_path): - return False, "No file at %s or Incorrect path" % clean_ref_code_path + msg = "No file at %s or Incorrect path" % clean_ref_code_path + return False, msg if not isfile(self.submit_code_path): - return False, 'No file at %s or Incorrect path' % self.submit_code_path + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg self.write_to_submit_code_file(self.submit_code_path, user_answer) self.user_output_path, self.ref_output_path = self.set_file_paths() - self.compile_command, self.compile_main = self.get_commands(clean_ref_code_path, self.user_output_path, self.ref_output_path) - self.compiled_user_answer = self._run_command(self.compile_command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - self.compiled_test_code = self._run_command(self.compile_main, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - + self.compile_command, self.compile_main = self.get_commands( + clean_ref_code_path, + self.user_output_path, + self.ref_output_path + ) + self.compiled_user_answer = self._run_command( + self.compile_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + self.compiled_test_code = self._run_command( + self.compile_main, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) return self.compiled_user_answer, self.compiled_test_code def check_code(self, user_answer, test_case): - """ Function validates student code using instructor code as - reference.The first argument ref_code_path, is the path to - instructor code, it is assumed to have executable permission. - The second argument submit_code_path, is the path to the student - code, it is assumed to have executable permission. - - Returns - -------- - - returns (True, "Correct answer") : If the student function returns - expected output when called by reference code. - - returns (False, error_msg): If the student function fails to return - expected output when called by reference code. - - Returns (False, error_msg): If mandatory arguments are not files or - if the required permissions are not given to the file(s). - - """ - success = False - proc, stdnt_out, stdnt_stderr = self.compiled_user_answer - stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) - - # Only if compilation is successful, the program is executed - # And tested with testcases - if stdnt_stderr == '': - proc, main_out, main_err = self.compiled_test_code - main_err = self._remove_null_substitute_char(main_err) - - if main_err == '': - ret = self._run_command([self.ref_output_path], stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc, stdout, stderr = ret - if proc.returncode == 0: - success, err = True, "Correct answer" - else: - err = stdout + "\n" + stderr - os.remove(self.ref_output_path) + """ Function validates student code using instructor code as + reference.The first argument ref_code_path, is the path to + instructor code, it is assumed to have executable permission. + The second argument submit_code_path, is the path to the student + code, it is assumed to have executable permission. + + Returns + -------- + + returns (True, "Correct answer") : If the student function returns + expected output when called by reference code. + + returns (False, error_msg): If the student function fails to return + expected output when called by reference code. + + Returns (False, error_msg): If mandatory arguments are not files or + if the required permissions are not given to the file(s). + """ + + success = False + proc, stdnt_out, stdnt_stderr = self.compiled_user_answer + stdnt_stderr = self._remove_null_substitute_char(stdnt_stderr) + + # Only if compilation is successful, the program is executed + # And tested with testcases + if stdnt_stderr == '': + proc, main_out, main_err = self.compiled_test_code + main_err = self._remove_null_substitute_char(main_err) + + if main_err == '': + ret = self._run_command([self.ref_output_path], + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + proc, stdout, stderr = ret + if proc.returncode == 0: + success, err = True, "Correct answer" else: - err = "Error:" - try: - error_lines = main_err.splitlines() - for e in error_lines: - if ':' in e: - err = err + "\n" + e.split(":", 1)[1] - else: - err = err + "\n" + e - except: - err = err + "\n" + main_err - os.remove(self.user_output_path) + err = stdout + "\n" + stderr + os.remove(self.ref_output_path) else: - err = "Compilation Error:" + err = "Error:" try: - error_lines = stdnt_stderr.splitlines() + error_lines = main_err.splitlines() for e in error_lines: if ':' in e: err = err + "\n" + e.split(":", 1)[1] else: err = err + "\n" + e except: - err = err + "\n" + stdnt_stderr + err = err + "\n" + main_err + os.remove(self.user_output_path) + else: + err = "Compilation Error:" + try: + error_lines = stdnt_stderr.splitlines() + for e in error_lines: + if ':' in e: + err = err + "\n" + e.split(":", 1)[1] + else: + err = err + "\n" + e + except: + err = err + "\n" + stdnt_stderr - return success, err + return success, err diff --git a/yaksh/evaluator_tests/test_bash_evaluation.py b/yaksh/evaluator_tests/test_bash_evaluation.py index 7c58c43..4ff3e0a 100644 --- a/yaksh/evaluator_tests/test_bash_evaluation.py +++ b/yaksh/evaluator_tests/test_bash_evaluation.py @@ -5,13 +5,18 @@ from yaksh.settings import SERVER_TIMEOUT class BashEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = [{"test_case": "bash_files/sample.sh,bash_files/sample.args"}] + self.test_case_data = [ + {"test_case": "bash_files/sample.sh,bash_files/sample.args"} + ] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]]" + " && echo $(( $1 + $2 )) && exit $(( $1 + $2 ))" + ) get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -21,7 +26,8 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertEquals(result.get('error'), "Correct answer") def test_error(self): - user_answer = "#!/bin/bash\n[[ $# -eq 2 ]] && echo $(( $1 - $2 )) && exit $(( $1 - $2 ))" + user_answer = ("#!/bin/bash\n[[ $# -eq 2 ]] " + "&& echo $(( $1 - $2 )) && exit $(( $1 - $2 ))") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -31,7 +37,8 @@ class BashEvaluationTestCases(unittest.TestCase): self.assertTrue("Error" in result.get("error")) def test_infinite_loop(self): - user_answer = "#!/bin/bash\nwhile [ 1 ] ; do echo "" > /dev/null ; done" + user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" + " do echo "" > /dev/null ; done") get_class = BashCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data diff --git a/yaksh/evaluator_tests/test_c_cpp_evaluation.py b/yaksh/evaluator_tests/test_c_cpp_evaluation.py index a07150d..71fb843 100644 --- a/yaksh/evaluator_tests/test_c_cpp_evaluation.py +++ b/yaksh/evaluator_tests/test_c_cpp_evaluation.py @@ -8,14 +8,15 @@ class CEvaluationTestCases(unittest.TestCase): self.test_case_data = [{"test_case": "c_cpp_files/main.cpp"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in your" + " code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "int add(int a, int b)\n{return a+b;}" get_class = CppCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEquals(result.get('error'), "Correct answer") diff --git a/yaksh/evaluator_tests/test_code_evaluation.py b/yaksh/evaluator_tests/test_code_evaluation.py index ace6115..51c0c51 100644 --- a/yaksh/evaluator_tests/test_code_evaluation.py +++ b/yaksh/evaluator_tests/test_code_evaluation.py @@ -2,20 +2,43 @@ import unittest import os from yaksh import python_assertion_evaluator from yaksh.language_registry import _LanguageRegistry, get_registry -from yaksh.settings import SERVER_TIMEOUT +from yaksh.settings import SERVER_TIMEOUT, code_evaluators class RegistryTestCase(unittest.TestCase): def setUp(self): self.registry_object = get_registry() self.language_registry = _LanguageRegistry() + assertion_evaluator_path = ("yaksh.python_assertion_evaluator" + ".PythonAssertionEvaluator" + ) + stdout_evaluator_path = ("yaksh.python_stdout_evaluator." + "PythonStdoutEvaluator" + ) + code_evaluators['python'] = \ + {"standardtestcase": assertion_evaluator_path, + "stdoutbasedtestcase": stdout_evaluator_path + } def test_set_register(self): - class_name = getattr(python_assertion_evaluator, 'PythonAssertionEvaluator') - self.registry_object.register("python", {"standardtestcase": "yaksh.python_assertion_evaluator.PythonAssertionEvaluator", - "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" - }) - self.assertEquals(self.registry_object.get_class("python", "standardtestcase"), class_name) + evaluator_class = self.registry_object.get_class("python", + "standardtestcase" + ) + assertion_evaluator_path = ("yaksh.python_assertion_evaluator" + ".PythonAssertionEvaluator" + ) + stdout_evaluator_path = ("yaksh.python_stdout_evaluator." + "PythonStdoutEvaluator" + ) + class_name = getattr(python_assertion_evaluator, + 'PythonAssertionEvaluator' + ) + self.registry_object.register("python", + {"standardtestcase": assertion_evaluator_path, + "stdoutbasedtestcase": stdout_evaluator_path + } + ) + self.assertEquals(evaluator_class, class_name) def tearDown(self): self.registry_object = None diff --git a/yaksh/evaluator_tests/test_java_evaluation.py b/yaksh/evaluator_tests/test_java_evaluation.py index 76a3fcf..801277f 100644 --- a/yaksh/evaluator_tests/test_java_evaluation.py +++ b/yaksh/evaluator_tests/test_java_evaluation.py @@ -7,11 +7,14 @@ from yaksh.settings import SERVER_TIMEOUT class JavaEvaluationTestCases(unittest.TestCase): def setUp(self): - self.test_case_data = [{"test_case": "java_files/main_square.java"}] + self.test_case_data = [ + {"test_case": "java_files/main_square.java"} + ] self.in_dir = "/tmp" evaluator.SERVER_TIMEOUT = 9 self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(evaluator.SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(evaluator.SERVER_TIMEOUT) def tearDown(self): evaluator.SERVER_TIMEOUT = 2 diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index f6ac0bf..1e867a3 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -9,11 +9,12 @@ from textwrap import dedent class PythonAssertionEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"test_case": 'assert(add(1,2)==3)'}, - {"test_case": 'assert(add(-1,2)==1)'}, - {"test_case": 'assert(add(-1,-2)==-3)'}, - ] + {"test_case": 'assert(add(-1,2)==1)'}, + {"test_case": 'assert(add(-1,-2)==-3)'}, + ] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop in" + " your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "def add(a,b):\n\treturn a + b" @@ -33,14 +34,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) - self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)") + self.assertEqual(result.get('error'), + "AssertionError in: assert(add(1,2)==3)" + ) def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg) @@ -50,12 +53,18 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b); return a + b """) - syntax_error_msg = ["Traceback", "call", "File", "line", "", - "SyntaxError", "invalid syntax"] + syntax_error_msg = ["Traceback", + "call", + "File", + "line", + "", + "SyntaxError", + "invalid syntax" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -68,12 +77,17 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b): return a + b """) - indent_error_msg = ["Traceback", "call", "File", "line", "", - "IndentationError", "indented block"] + indent_error_msg = ["Traceback", "call", + "File", + "line", + "", + "IndentationError", + "indented block" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -83,11 +97,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def test_name_error(self): user_answer = "" - name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] + name_error_msg = ["Traceback", + "call", + "NameError", + "name", + "defined" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -100,12 +119,15 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a, b): return add(3, 3) """) - recursion_error_msg = ["Traceback", "call", "RuntimeError", - "maximum recursion depth exceeded"] + recursion_error_msg = ["Traceback", + "call", + "RuntimeError", + "maximum recursion depth exceeded" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -118,11 +140,16 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): def add(a): return a + b """) - type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] + type_error_msg = ["Traceback", + "call", + "TypeError", + "exactly", + "argument" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) @@ -136,7 +163,12 @@ class PythonAssertionEvaluationTestCases(unittest.TestCase): c = 'a' return int(a) + int(b) + int(c) """) - value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] + value_error_msg = ["Traceback", + "call", + "ValueError", + "invalid literal", + "base" + ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -152,14 +184,15 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): def setUp(self): self.test_case_data = [{"expected_output": "0 1 1 2 3"}] self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint a,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success')) @@ -168,19 +201,21 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "a,b=0,1\nfor i in range(5):\n\tprint b,\n\ta,b=b,a+b" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), "Incorrect Answer") def test_direct_printed_answer(self): user_answer = "print '0 1 1 2 3'" - error_msg = "Incorrect Answer: Please avoid printing the expected output directly" + error_msg = ("Incorrect Answer: Please avoid printing" + " the expected output directly" + ) get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), error_msg) @@ -189,8 +224,8 @@ class PythonStdoutEvaluationTestCases(unittest.TestCase): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonStdoutEvaluator() kwargs = {'user_answer': user_answer, - 'test_case_data': self.test_case_data - } + 'test_case_data': self.test_case_data + } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), 'Incorrect Answer') diff --git a/yaksh/evaluator_tests/test_scilab_evaluation.py b/yaksh/evaluator_tests/test_scilab_evaluation.py index 24d6740..242f260 100644 --- a/yaksh/evaluator_tests/test_scilab_evaluation.py +++ b/yaksh/evaluator_tests/test_scilab_evaluation.py @@ -9,10 +9,12 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.test_case_data = [{"test_case": "scilab_files/test_add.sce"}] self.in_dir = "/tmp" self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) + "You probably have an infinite loop" + " in your code.").format(SERVER_TIMEOUT) def test_correct_answer(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a+b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -22,7 +24,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(result.get('success')) def test_error(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a+b;\ndis(\tendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a+b;\ndis(\tendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -33,7 +36,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): def test_incorrect_answer(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a-b;\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a-b;\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data @@ -44,7 +48,8 @@ class ScilabEvaluationTestCases(unittest.TestCase): self.assertTrue(result.get('error').splitlines > 1) def test_infinite_loop(self): - user_answer = "funcprot(0)\nfunction[c]=add(a,b)\n\tc=a;\nwhile(1==1)\nend\nendfunction" + user_answer = ("funcprot(0)\nfunction[c]=add(a,b)" + "\n\tc=a;\nwhile(1==1)\nend\nendfunction") get_class = ScilabCodeEvaluator(self.in_dir) kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data diff --git a/yaksh/java_code_evaluator.py b/yaksh/java_code_evaluator.py index ac2c487..c64aa3b 100644 --- a/yaksh/java_code_evaluator.py +++ b/yaksh/java_code_evaluator.py @@ -40,31 +40,46 @@ class JavaCodeEvaluator(CodeEvaluator): return None else: ref_code_path = test_case - clean_ref_code_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) + clean_ref_code_path, clean_test_case_path = \ + self._set_test_code_file_path(ref_code_path) if not isfile(clean_ref_code_path): - return False, "No file at %s or Incorrect path" % clean_ref_code_path + msg = "No file at %s or Incorrect path" % clean_ref_code_path + return False, msg if not isfile(self.submit_code_path): - return False, 'No file at %s or Incorrect path' % self.submit_code_path + msg = "No file at %s or Incorrect path" % self.submit_code_path + return False, msg user_code_directory = os.getcwd() + '/' - self.write_to_submit_code_file(self.submit_code_path, user_answer) + self.write_to_submit_code_file(self.submit_code_path, + user_answer + ) ref_file_name = (clean_ref_code_path.split('/')[-1]).split('.')[0] - self.user_output_path = self.set_file_paths(user_code_directory, 'Test') - self.ref_output_path = self.set_file_paths(user_code_directory, ref_file_name) - - compile_command, self.compile_main = self.get_commands(clean_ref_code_path, user_code_directory) - self.run_command_args = "java -cp {0} {1}".format(user_code_directory, - ref_file_name) + self.user_output_path = self.set_file_paths(user_code_directory, + 'Test' + ) + self.ref_output_path = self.set_file_paths(user_code_directory, + ref_file_name + ) + compile_command, self.compile_main = self.get_commands( + clean_ref_code_path, + user_code_directory + ) + self.run_command_args = "java -cp {0} {1}".format( + user_code_directory, + ref_file_name + ) self.compiled_user_answer = self._run_command(compile_command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) self.compiled_test_code = self._run_command(self.compile_main, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) return self.compiled_user_answer, self.compiled_test_code diff --git a/yaksh/language_registry.py b/yaksh/language_registry.py index 398e1aa..24aef7d 100644 --- a/yaksh/language_registry.py +++ b/yaksh/language_registry.py @@ -36,6 +36,7 @@ class _LanguageRegistry(object): test_case_register = self._register[language] cls = test_case_register.get(test_case_type) module_name, class_name = cls.rsplit(".", 1) + import yaksh.python_assertion_evaluator # load the module, will raise ImportError if module cannot be loaded get_module = importlib.import_module(module_name) # get the class, will raise AttributeError if class cannot be found diff --git a/yaksh/models.py b/yaksh/models.py index dd2fb5f..32fb0d0 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -201,11 +201,14 @@ class Question(models.Model): questions = Question.objects.filter(id__in = question_ids, user_id = user.id) questions_dict = [] for question in questions: - q_dict = {'summary': question.summary, 'description': question.description, - 'points': question.points, 'language': question.language, - 'type': question.type, 'active': question.active, - 'test_case_type': question.test_case_type, - 'snippet': question.snippet} + q_dict = {'summary': question.summary, + 'description': question.description, + 'points': question.points, + 'language': question.language, + 'type': question.type, + 'active': question.active, + 'test_case_type': question.test_case_type, + 'snippet': question.snippet} questions_dict.append(q_dict) return json.dumps(questions_dict, indent=2) @@ -217,14 +220,24 @@ class Question(models.Model): Question.objects.get_or_create(**question) def get_test_cases(self, **kwargs): - test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) - test_cases = test_case_ctype.get_all_objects_for_this_type(question=self, **kwargs) + test_case_ctype = ContentType.objects.get(app_label="yaksh", + model=self.test_case_type + ) + test_cases = test_case_ctype.get_all_objects_for_this_type( + question=self, + **kwargs + ) return test_cases def get_test_case(self, **kwargs): - test_case_ctype = ContentType.objects.get(app_label="yaksh", model=self.test_case_type) - test_case = test_case_ctype.get_object_for_this_type(question=self, **kwargs) + test_case_ctype = ContentType.objects.get(app_label="yaksh", + model=self.test_case_type + ) + test_case = test_case_ctype.get_object_for_this_type( + question=self, + **kwargs + ) return test_case @@ -368,10 +381,13 @@ class QuestionPaper(models.Model): def make_answerpaper(self, user, ip, attempt_num): """Creates an answer paper for the user to attempt the quiz""" - ans_paper = AnswerPaper(user=user, user_ip=ip, attempt_number=attempt_num) + ans_paper = AnswerPaper(user=user, + user_ip=ip, + attempt_number=attempt_num + ) ans_paper.start_time = datetime.now() - ans_paper.end_time = ans_paper.start_time \ - + timedelta(minutes=self.quiz.duration) + ans_paper.end_time = ans_paper.start_time + \ + timedelta(minutes=self.quiz.duration) ans_paper.question_paper = self ans_paper.save() questions = self._get_questions_for_answerpaper() @@ -750,7 +766,9 @@ class StandardTestCase(TestCase): return {"test_case": self.test_case} def __unicode__(self): - return u'Question: {0} | Test Case: {1}'.format(self.question, self.test_case) + return u'Question: {0} | Test Case: {1}'.format(self.question, + self.test_case + ) class StdoutBasedTestCase(TestCase): @@ -760,7 +778,9 @@ class StdoutBasedTestCase(TestCase): return {"expected_output": self.expected_output} def __unicode__(self): - return u'Question: {0} | Exp. Output: {1}'.format(self.question, self.expected_output) + return u'Question: {0} | Exp. Output: {1}'.format(self.question, + self.expected_output + ) class McqTestCase(TestCase): @@ -771,4 +791,6 @@ class McqTestCase(TestCase): return {"options": self.options, "correct": self.correct} def __unicode__(self): - return u'Question: {0} | Correct: {1}'.format(self.question, self.correct) + return u'Question: {0} | Correct: {1}'.format(self.question, + self.correct + ) diff --git a/yaksh/python_stdout_evaluator.py b/yaksh/python_stdout_evaluator.py index b967024..6606581 100644 --- a/yaksh/python_stdout_evaluator.py +++ b/yaksh/python_stdout_evaluator.py @@ -42,7 +42,9 @@ class PythonStdoutEvaluator(CodeEvaluator): tb = None if expected_output in user_answer: success = False - err = "Incorrect Answer: Please avoid printing the expected output directly" + err = ("Incorrect Answer: Please avoid " + "printing the expected output directly" + ) elif self.output_value == expected_output: success = True err = "Correct answer" diff --git a/yaksh/scilab_code_evaluator.py b/yaksh/scilab_code_evaluator.py index f4aa5f8..91b4cb3 100644 --- a/yaksh/scilab_code_evaluator.py +++ b/yaksh/scilab_code_evaluator.py @@ -14,7 +14,8 @@ class ScilabCodeEvaluator(CodeEvaluator): """Tests the Scilab code obtained from Code Server""" def setup(self): super(ScilabCodeEvaluator, self).setup() - self.submit_code_path = self.create_submit_code_file('function.sci') + self.submit_code_path = \ + self.create_submit_code_file('function.sci') def teardown(self): super(ScilabCodeEvaluator, self).teardown() @@ -23,8 +24,10 @@ class ScilabCodeEvaluator(CodeEvaluator): def check_code(self, user_answer, test_case): ref_code_path = test_case - clean_ref_path, clean_test_case_path = self._set_test_code_file_path(ref_code_path) - user_answer, terminate_commands = self._remove_scilab_exit(user_answer.lstrip()) + clean_ref_path, clean_test_case_path = \ + self._set_test_code_file_path(ref_code_path) + user_answer, terminate_commands = \ + self._remove_scilab_exit(user_answer.lstrip()) success = False self.write_to_submit_code_file(self.submit_code_path, user_answer) @@ -35,12 +38,15 @@ class ScilabCodeEvaluator(CodeEvaluator): code.\n Otherwise your code will not be evaluated\ correctly.\n" - cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format(clean_ref_path) + cmd = 'printf "lines(0)\nexec(\'{0}\',2);\nquit();"'.format( + clean_ref_path + ) cmd += ' | timeout 8 scilab-cli -nb' ret = self._run_command(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) proc, stdout, stderr = ret # Get only the error. diff --git a/yaksh/settings.py b/yaksh/settings.py index f8baa2c..70e5471 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -20,12 +20,12 @@ SERVER_TIMEOUT = 2 URL_ROOT = '' code_evaluators = { - "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", - "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" - }, - "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, - "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"}, - "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, - "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, - } + "python": {"standardtestcase": "python_assertion_evaluator.PythonAssertionEvaluator", + "stdoutbasedtestcase": "python_stdout_evaluator.PythonStdoutEvaluator" + }, + "c": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, + "cpp": {"standardtestcase": "cpp_code_evaluator.CppCodeEvaluator"}, + "java": {"standardtestcase": "java_code_evaluator.JavaCodeEvaluator"}, + "bash": {"standardtestcase": "bash_code_evaluator.BashCodeEvaluator"}, + "scilab": {"standardtestcase": "scilab_code_evaluator.ScilabCodeEvaluator"}, +} diff --git a/yaksh/tests.py b/yaksh/tests.py index 150a8b0..f2083dd 100644 --- a/yaksh/tests.py +++ b/yaksh/tests.py @@ -75,22 +75,33 @@ class QuestionTestCases(unittest.TestCase): # Single question details self.user1 = User.objects.get(pk=1) self.user2 = User.objects.get(pk=2) - self.question1 = Question(summary='Demo question', language='Python', - type='Code', active=True, - test_case_type='standardtestcase', - description='Write a function', points=1.0, - snippet='def myfunc()', user=self.user1) + self.question1 = Question(summary='Demo question', + language='Python', + type='Code', + active=True, + test_case_type='standardtestcase', + description='Write a function', + points=1.0, + snippet='def myfunc()', + user=self.user1 + ) self.question1.save() - self.question2 = Question(summary='Demo Json', language='python', - type='code', active=True, - description='factorial of a no', points=2.0, - snippet='def fact()', user=self.user2) + self.question2 = Question(summary='Demo Json', + language='python', + type='code', + active=True, + description='factorial of a no', + points=2.0, + snippet='def fact()', + user=self.user2 + ) self.question2.save() self.question1.tags.add('python', 'function') self.assertion_testcase = StandardTestCase(question=self.question1, - test_case='assert myfunc(12, 13) == 15') + test_case='assert myfunc(12, 13) == 15' + ) self.user_answer = "demo_answer" questions_data = [{"snippet": "def fact()", "active": True, "points": 1.0, "description": "factorial of a no", @@ -182,7 +193,9 @@ class QuestionPaperTestCases(unittest.TestCase): # create question paper self.question_paper = QuestionPaper.objects.create(quiz=self.quiz, - total_marks=0.0, shuffle_questions=True) + total_marks=0.0, + shuffle_questions=True + ) # add fixed set of questions to the question paper self.question_paper.fixed_questions.add(self.questions[3], @@ -190,23 +203,29 @@ class QuestionPaperTestCases(unittest.TestCase): # create two QuestionSet for random questions # QuestionSet 1 self.question_set_1 = QuestionSet.objects.create(marks=2, - num_questions=2) + num_questions=2 + ) # add pool of questions for random sampling - self.question_set_1.questions.add(self.questions[6], self.questions[7], - self.questions[8], self.questions[9]) + self.question_set_1.questions.add(self.questions[6], + self.questions[7], + self.questions[8], + self.questions[9] + ) # add question set 1 to random questions in Question Paper self.question_paper.random_questions.add(self.question_set_1) # QuestionSet 2 self.question_set_2 = QuestionSet.objects.create(marks=3, - num_questions=3) + num_questions=3 + ) # add pool of questions self.question_set_2.questions.add(self.questions[11], - self.questions[12], - self.questions[13], - self.questions[14]) + self.questions[12], + self.questions[13], + self.questions[14] + ) # add question set 2 self.question_paper.random_questions.add(self.question_set_2) @@ -215,8 +234,10 @@ class QuestionPaperTestCases(unittest.TestCase): self.user = User.objects.get(pk=1) - self.attempted_papers = AnswerPaper.objects.filter(question_paper=self.question_paper, - user=self.user) + self.attempted_papers = AnswerPaper.objects.filter( + question_paper=self.question_paper, + user=self.user + ) def test_questionpaper(self): """ Test question paper""" @@ -235,7 +256,8 @@ class QuestionPaperTestCases(unittest.TestCase): """ Test get_random_questions() method of Question Paper""" random_questions_set_1 = self.question_set_1.get_random_questions() random_questions_set_2 = self.question_set_2.get_random_questions() - total_random_questions = len(random_questions_set_1 + random_questions_set_2) + total_random_questions = len(random_questions_set_1 + \ + random_questions_set_2) self.assertEqual(total_random_questions, 5) # To check whether random questions are from random_question_set @@ -286,12 +308,15 @@ class AnswerPaperTestCases(unittest.TestCase): # create answerpaper self.answerpaper = AnswerPaper(user=self.user, - question_paper=self.question_paper, - start_time=self.start_time, - end_time=self.end_time, - user_ip=self.ip) - self.attempted_papers = AnswerPaper.objects.filter(question_paper=self.question_paper, - user=self.user) + question_paper=self.question_paper, + start_time=self.start_time, + end_time=self.end_time, + user_ip=self.ip + ) + self.attempted_papers = AnswerPaper.objects.filter( + question_paper=self.question_paper, + user=self.user + ) already_attempted = self.attempted_papers.count() self.answerpaper.attempt_number = already_attempted + 1 self.answerpaper.save() @@ -300,9 +325,14 @@ class AnswerPaperTestCases(unittest.TestCase): self.answerpaper.save() # answers for the Answer Paper self.answer_right = Answer(question=Question.objects.get(id=1), - answer="Demo answer", correct=True, marks=1) + answer="Demo answer", + correct=True, marks=1 + ) self.answer_wrong = Answer(question=Question.objects.get(id=2), - answer="My answer", correct=False, marks=0) + answer="My answer", + correct=False, + marks=0 + ) self.answer_right.save() self.answer_wrong.save() self.answerpaper.answers.add(self.answer_right) @@ -448,12 +478,14 @@ class CourseTestCases(unittest.TestCase): def test_get_quizzes(self): """ Test get_quizzes method of Courses""" - self.assertSequenceEqual(self.course.get_quizzes(), [self.quiz1, self.quiz2]) + self.assertSequenceEqual(self.course.get_quizzes(), + [self.quiz1, self.quiz2]) def test_add_teachers(self): """ Test to add teachers to a course""" self.course.add_teachers(self.student1, self.student2) - self.assertSequenceEqual(self.course.get_teachers(), [self.student1, self.student2]) + self.assertSequenceEqual(self.course.get_teachers(), + [self.student1, self.student2]) def test_remove_teachers(self): """ Test to remove teachers from a course""" @@ -472,42 +504,61 @@ class CourseTestCases(unittest.TestCase): class TestCaseTestCases(unittest.TestCase): def setUp(self): self.user = User.objects.get(pk=1) - self.question1 = Question(summary='Demo question 1', language='Python', - type='Code', active=True, - description='Write a function', points=1.0, - test_case_type="standardtestcase", user=self.user, - snippet='def myfunc()' - ) - self.question2 = Question(summary='Demo question 2', language='Python', - type='Code', active=True, - description='Write to standard output', points=1.0, - test_case_type="stdoutbasedtestcase", user=self.user, - snippet='def myfunc()' - ) + self.question1 = Question(summary='Demo question 1', + language='Python', + type='Code', + active=True, + description='Write a function', + points=1.0, + test_case_type="standardtestcase", + user=self.user, + snippet='def myfunc()' + ) + self.question2 = Question(summary='Demo question 2', + language='Python', + type='Code', + active=True, + description='Write to standard output', + points=1.0, + test_case_type="stdoutbasedtestcase", + user=self.user, + snippet='def myfunc()' + ) self.question1.save() self.question2.save() - self.assertion_testcase = StandardTestCase(question=self.question1, - test_case='assert myfunc(12, 13) == 15') - self.stdout_based_testcase = StdoutBasedTestCase(question=self.question2, - expected_output='Hello World') + self.assertion_testcase = StandardTestCase( + question=self.question1, + test_case='assert myfunc(12, 13) == 15' + ) + self.stdout_based_testcase = StdoutBasedTestCase( + question=self.question2, + expected_output='Hello World' + ) self.assertion_testcase.save() self.stdout_based_testcase.save() answer_data = {"user_answer": "demo_answer", - "test_case_data": [{"test_case": "assert myfunc(12, 13) == 15"}], - } + "test_case_data": [ + {"test_case": "assert myfunc(12, 13) == 15"} + ] + } self.answer_data_json = json.dumps(answer_data) def test_assertion_testcase(self): """ Test question """ self.assertEqual(self.assertion_testcase.question, self.question1) - self.assertEqual(self.assertion_testcase.test_case, 'assert myfunc(12, 13) == 15') + self.assertEqual(self.assertion_testcase.test_case, + 'assert myfunc(12, 13) == 15') def test_stdout_based_testcase(self): """ Test question """ self.assertEqual(self.stdout_based_testcase.question, self.question2) - self.assertEqual(self.stdout_based_testcase.expected_output, 'Hello World') + self.assertEqual(self.stdout_based_testcase.expected_output, + 'Hello World' + ) def test_consolidate_answer_data(self): """ Test consolidate answer data model method """ - result = self.question1.consolidate_answer_data(user_answer="demo_answer") + result = self.question1.consolidate_answer_data( + user_answer="demo_answer" + ) self.assertEqual(result, self.answer_data_json) \ No newline at end of file -- cgit