Change test data files, views, models, template

- Fix upload marks for a quiz column missing error
- Add upload marks task to celery
- Minor fixes to avoid errors in regrade and custom_filters

15 files changed, 208 insertions, 137 deletions

diff --git a/yaksh/fixtures/marks_correct.csv b/yaksh/fixtures/marks_correct.csv
index 9134da5..d739644 100644
--- a/yaksh/fixtures/marks_correct.csv
+++ b/yaksh/fixtures/marks_correct.csv
@@ -1,4 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,1,good work,1,nice
-student2,1,good work,0,bad
-
+student2,1,good work,0,bad
\ No newline at end of file
diff --git a/yaksh/fixtures/marks_header_missing.csv b/yaksh/fixtures/marks_header_missing.csv
index 8c3a747..81b0c77 100644
--- a/yaksh/fixtures/marks_header_missing.csv
+++ b/yaksh/fixtures/marks_header_missing.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks
+user__username,Q-1212-Dummy1-1.0-marks
 student1,0.9
 student2,1
diff --git a/yaksh/fixtures/marks_header_modified.csv b/yaksh/fixtures/marks_header_modified.csv
index 08ba31d..f6d6859 100644
--- a/yaksh/fixtures/marks_header_modified.csv
+++ b/yaksh/fixtures/marks_header_modified.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummmy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-1212-Dummmy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,0.75,fine work,1,not nice
 student2,1,good work,0,not okay
diff --git a/yaksh/fixtures/marks_invalid_data.csv b/yaksh/fixtures/marks_invalid_data.csv
index 44fb2bb..b4af15b 100644
--- a/yaksh/fixtures/marks_invalid_data.csv
+++ b/yaksh/fixtures/marks_invalid_data.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,NA,good work,1,nice
 student2,1,good work,0,bad
diff --git a/yaksh/fixtures/marks_invalid_question_id.csv b/yaksh/fixtures/marks_invalid_question_id.csv
index eb1d921..629a673 100644
--- a/yaksh/fixtures/marks_invalid_question_id.csv
+++ b/yaksh/fixtures/marks_invalid_question_id.csv
@@ -1,3 +1,3 @@
-username,Q-12112-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-12112-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,1,good work,1,nice
 student2,1,good work,0,bad
diff --git a/yaksh/fixtures/marks_invalid_user.csv b/yaksh/fixtures/marks_invalid_user.csv
index bd31071..5d5c200 100644
--- a/yaksh/fixtures/marks_invalid_user.csv
+++ b/yaksh/fixtures/marks_invalid_user.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,1,good work,1,nice
 student452,1,good work,0,bad
diff --git a/yaksh/fixtures/marks_not_attempted_question.csv b/yaksh/fixtures/marks_not_attempted_question.csv
index 3c3e2e7..ecce363 100644
--- a/yaksh/fixtures/marks_not_attempted_question.csv
+++ b/yaksh/fixtures/marks_not_attempted_question.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
+user__username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments,Q-1213-Dummy2-1.0-marks,Q-1213-Dummy2-comments
 student1,1,good work,1,nice
 student2,0.3,very good,1,good
diff --git a/yaksh/fixtures/marks_single_question.csv b/yaksh/fixtures/marks_single_question.csv
index 9677730..00b74fe 100644
--- a/yaksh/fixtures/marks_single_question.csv
+++ b/yaksh/fixtures/marks_single_question.csv
@@ -1,3 +1,3 @@
-username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments
+user__username,Q-1212-Dummy1-1.0-marks,Q-1212-Dummy1-comments
 student1,0.5,okay work
 student2,1,good work
diff --git a/yaksh/models.py b/yaksh/models.py
index 11ddf8a..77b3684 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -2267,7 +2267,7 @@ class AnswerPaper(models.Model):
         ans_data = None
         if not df.empty:
             ans_data = df.groupby("question_id").tail(1)
-        for que_summary, que_id in question_ids:
+        for que_summary, que_id, que_comments in question_ids:
             if ans_data is not None:
                 ans = ans_data['question_id'].to_list()
                 marks = ans_data['marks'].to_list()
@@ -2278,6 +2278,7 @@ class AnswerPaper(models.Model):
                     que_data[que_summary] = 0
             else:
                 que_data[que_summary] = 0
+            que_data[que_comments] = "NA"
         return que_data
 
     def current_question(self):
@@ -2576,25 +2577,17 @@ class AnswerPaper(models.Model):
                 self.user, self.question_paper.quiz.description,
                 question_id
             )
-            return False, msg + 'Question not in the answer paper.'
+            return False, f'{msg} Question not in the answer paper.'
         user_answer = self.answers.filter(question=question).last()
-        if not user_answer:
-            return False, msg + 'Did not answer.'
+        if not user_answer or not user_answer.answer:
+            return False, f'{msg} Did not answer.'
         if question.type in ['mcc', 'arrange']:
             try:
                 answer = literal_eval(user_answer.answer)
                 if type(answer) is not list:
-                    return (False,
-                            msg + '{0} answer not a list.'.format(
-                                                            question.type
-                                                            )
-                            )
+                    return (False, f'{msg} {question.type} answer not a list.')
             except Exception:
-                return (False,
-                        msg + '{0} answer submission error'.format(
-                                                             question.type
-                                                             )
-                        )
+                return (False, f'{msg} {question.type} answer submission error')
         else:
             answer = user_answer.answer
         json_data = question.consolidate_answer_data(answer) \
diff --git a/yaksh/tasks.py b/yaksh/tasks.py
index 1c4658b..5068c64 100644
--- a/yaksh/tasks.py
+++ b/yaksh/tasks.py
@@ -1,6 +1,8 @@
 # Python Imports
 from __future__ import absolute_import, unicode_literals
 from textwrap import dedent
+import csv
+import json
 
 # Django and celery imports
 from celery import shared_task
@@ -8,7 +10,10 @@ from django.urls import reverse
 from django.shortcuts import get_object_or_404
 
 # Local imports
-from .models import Course, QuestionPaper, Quiz, AnswerPaper, CourseStatus
+from .models import (
+    Course, QuestionPaper, Quiz, AnswerPaper, CourseStatus, User, Question,
+    Answer
+)
 from notifications_plugin.models import NotificationMessage, Notification
 
 
@@ -80,3 +85,112 @@ def regrade_papers(data):
     notification = Notification.objects.add_single_notification(
         user_id, nm.id
     )
+
+
+@shared_task
+def update_user_marks(data):
+    request_user = data.get("user_id")
+    course_id = data.get("course_id")
+    questionpaper_id = data.get("questionpaper_id")
+    csv_data = data.get("csv_data")
+    question_paper = QuestionPaper.objects.get(id=questionpaper_id)
+    def _get_header_info(reader):
+        question_ids = []
+        fields = reader.fieldnames
+        for field in fields:
+            if field.startswith('Q') and field.count('-') > 0:
+                qid = int(field.split('-')[1])
+                if qid not in question_ids:
+                    question_ids.append(qid)
+        return question_ids
+    try:
+        reader = csv.DictReader(csv_data)
+        question_ids = _get_header_info(reader)
+        _read_marks_csv(
+            reader, request_user, course_id, question_paper, question_ids
+        )
+    except TypeError:
+        url = reverse(
+            "yaksh:monitor", args=[question_paper.quiz_id, course_id]
+        )
+        message = dedent("""
+            Unable to update quiz marks. Please re-upload correct CSV file
+            Click <a href="{0}">here</a> to view
+            """.format(url)
+        )
+        nm = NotificationMessage.objects.add_single_message(
+            request_user, "{0} marks update status".format(
+                question_paper.quiz.description
+            ), message, "warning"
+        )
+        notification = Notification.objects.add_single_notification(
+            request_user, nm.id
+        )
+
+
+def _read_marks_csv(
+        reader, request_user, course_id, question_paper, question_ids):
+    update_status = []
+    for row in reader:
+        username = row['user__username']
+        user = User.objects.filter(username=username).first()
+        if user:
+            answerpapers = question_paper.answerpaper_set.filter(
+                course_id=course_id, user_id=user.id)
+        else:
+            update_status.append(f'{username} user not found!')
+            continue
+        answerpaper = answerpapers.last()
+        if not answerpaper:
+            update_status.append(f'{username} has no answerpaper!')
+            continue
+        answers = answerpaper.answers.all()
+        questions = answerpaper.questions.values_list('id', flat=True)
+        for qid in question_ids:
+            question = Question.objects.filter(id=qid).first()
+            if not question:
+                update_status.append(f'{qid} is an invalid question id!')
+                continue
+            if qid in questions:
+                answer = answers.filter(question_id=qid).last()
+                if not answer:
+                    answer = Answer(question_id=qid, marks=0, correct=False,
+                                    answer='', error=json.dumps([]))
+                    answer.save()
+                    answerpaper.answers.add(answer)
+                key1 = 'Q-{0}-{1}-{2}-marks'.format(qid, question.summary,
+                                                    question.points)
+                key2 = 'Q-{0}-{1}-comments'.format(qid, question.summary)
+                if key1 in reader.fieldnames:
+                    try:
+                        answer.set_marks(float(row[key1]))
+                    except ValueError:
+                        update_status.append(f'{row[key1]} invalid marks!')
+                if key2 in reader.fieldnames:
+                    answer.set_comment(row[key2])
+                answer.save()
+        answerpaper.update_marks(state='completed')
+        answerpaper.save()
+        update_status.append(
+            'Updated successfully for user: {0}, question: {1}'.format(
+            username, question.summary)
+        )
+    url = reverse(
+        "yaksh:grade_user",
+        args=[question_paper.quiz_id, course_id]
+    )
+    message = dedent("""
+        Quiz mark update is complete.
+        Click <a href="{0}">here</a> to view
+        <br><br>{1}
+        """.format(url, "\n".join(update_status))
+    )
+    summary = "{0} marks update status".format(
+        question_paper.quiz.description
+    )
+    nm = NotificationMessage.objects.add_single_message(
+        request_user, summary, message, "info"
+    )
+    notification = Notification.objects.add_single_notification(
+        request_user, nm.id
+    )
diff --git a/yaksh/templates/yaksh/grade_user.html b/yaksh/templates/yaksh/grade_user.html
index 4e1db2b..32cf09c 100644
--- a/yaksh/templates/yaksh/grade_user.html
+++ b/yaksh/templates/yaksh/grade_user.html
@@ -559,6 +559,9 @@ function searchNames() {
                                   {% endif %}
                                 </div>
                               </div>
+                              <br>
+                              <b>Comment:</b>
+                              <textarea class="form-control" readonly="">{{ans.answer.comment}}</textarea>
                             </div>
                           </div>
                           <br>
diff --git a/yaksh/templates/yaksh/monitor.html b/yaksh/templates/yaksh/monitor.html
index c7755e7..6fd3cb1 100644
--- a/yaksh/templates/yaksh/monitor.html
+++ b/yaksh/templates/yaksh/monitor.html
@@ -52,7 +52,8 @@ $(document).ready(function()
   {% if quiz %}
     {% if papers %}
     <div class="row">
-      <div class="card col-md-3">
+      <div class="col-md-3">
+      <div class="card">
         <div class="card-body">
           <div class="table-responsive">
             <table id="course-detail" class="table">
@@ -83,34 +84,41 @@ $(document).ready(function()
             </table>
           </div>
         </div>
-      </div>
-      <div class="col-md-9">
-        <div class="row">
-          <div class="col-md-4">
+        <div class="card-body">
+          <div class="col">
+            <div class="badge badge-info">
+              Auto-Refreshes every 5 minutes
+            </div>
+          </div>
+          <br>
+          <div class="col">
             <button type="button" class="btn btn-info" data-toggle="modal" data-target="#csvModal">
               <i class="fa fa-download"></i>&nbsp;Download CSV
             </button>
           </div>
-          <div class="col-md-4">
+          <br>
+          <div class="col">
             <a href="{% url 'yaksh:show_statistics' papers.0.question_paper.id course.id %}" class="btn btn-primary">
               <i class="fa fa-line-chart"></i>&nbsp;Question Statistics
             </a>
           </div>
-          <div class="col-md-4">
-            <div class="badge badge-info">
-              Auto-Refreshes every 5 minutes
-            </div>
-          </div>
         </div>
-        <hr>
+      </div>
+      </div>
+      <div class="col-md-9">
         <div class="row">
-          <div class="col-md-4">
-          <p>
-          <b>
-          - Download the CSV file from the button above<br />
-          - Edit and upload the same <br />
-          </b>
-          </p>
+          <div class="col-md-5">
+            <ul>
+              <li>
+                Download the CSV file from the button
+              </li>
+              <li>
+                Edit and upload the same
+              </li>
+              <li>
+                <b>Note: Do not change the CSV Headers</b>
+              </li>
+            </ul>
           </div>
           <div class="col-md-6">
           <form id="upload_users" action="{% url 'yaksh:upload_marks' course.id papers.0.question_paper.id %}" method="POST" enctype="multipart/form-data">
@@ -123,7 +131,7 @@ $(document).ready(function()
           </form>
           </div>
         </div>
-        <br>
+        <hr>
         <div class="row">
           <div class="col-md-3">
             <b>Select Attempt number:</b>
diff --git a/yaksh/templatetags/custom_filters.py b/yaksh/templatetags/custom_filters.py
index b404758..bd97d2e 100644
--- a/yaksh/templatetags/custom_filters.py
+++ b/yaksh/templatetags/custom_filters.py
@@ -84,12 +84,15 @@ def get_answer_for_arrange_options(ans, question):
         ans = ans.decode("utf-8")
     else:
         ans = str(ans)
-    answer = literal_eval(ans)
-    testcases = []
-    for answer_id in answer:
-        tc = question.get_test_case(id=int(answer_id))
-        testcases.append(tc)
-    return testcases
+    try:
+        answer = literal_eval(ans)
+        testcases = []
+        for answer_id in answer:
+            tc = question.get_test_case(id=int(answer_id))
+            testcases.append(tc)
+        return testcases
+    except Exception:
+        return None
 
 
 @register.filter(name='replace_spaces')
diff --git a/yaksh/test_views.py b/yaksh/test_views.py
index 58b7506..ccd2fbc 100644
--- a/yaksh/test_views.py
+++ b/yaksh/test_views.py
@@ -23,7 +23,6 @@ from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.files import File
 from django.contrib.messages import get_messages
 from django.contrib.contenttypes.models import ContentType
-from celery.contrib.testing.worker import start_worker
 from django.test import SimpleTestCase
 
 
@@ -41,6 +40,8 @@ from online_test.celery_settings import app
 
 from notifications_plugin.models import Notification
 
+app.conf.update(CELERY_ALWAYS_EAGER=True)
+
 
 class TestUserRegistration(TestCase):
     def setUp(self):
@@ -4418,9 +4419,6 @@ class TestGrader(SimpleTestCase):
             end_time=timezone.now()+timezone.timedelta(minutes=20),
             )
 
-        self.celery_worker = start_worker(app)
-        self.celery_worker.__enter__()
-
     def tearDown(self):
         User.objects.all().delete()
         Course.objects.all().delete()
@@ -4429,7 +4427,6 @@ class TestGrader(SimpleTestCase):
         QuestionPaper.objects.all().delete()
         AnswerPaper.objects.all().delete()
         self.mod_group.delete()
-        self.celery_worker.__exit__(None, None, None)
 
     def test_regrade_denies_anonymous(self):
         # Given
diff --git a/yaksh/views.py b/yaksh/views.py
index 1965191..bddea26 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -33,6 +33,7 @@ except ImportError:
     from io import BytesIO as string_io
 import re
 # Local imports.
+from online_test.celery_settings import app
 from yaksh.code_server import get_result as get_result_from_code_server
 from yaksh.models import (
     Answer, AnswerPaper, AssignmentUpload, Course, FileUpload, FloatTestCase,
@@ -57,7 +58,7 @@ from .file_utils import extract_files, is_csv
 from .send_emails import (send_user_mail,
                           generate_activation_key, send_bulk_mail)
 from .decorators import email_verified, has_profile
-from .tasks import regrade_papers
+from .tasks import regrade_papers, update_user_marks
 from notifications_plugin.models import Notification
 
 
@@ -1842,7 +1843,10 @@ def download_quiz_csv(request, course_id, quiz_id):
             attempt_number=attempt_number
         ).order_by("user__first_name")
         que_summaries = [
-            (f"Q-{que.id}-{que.summary}-{que.points}-marks", que.id) for que in questions
+            (f"Q-{que.id}-{que.summary}-{que.points}-marks", que.id,
+                f"Q-{que.id}-{que.summary}-comments"
+                )
+            for que in questions
         ]
         user_data = list(answerpapers.values(
             "user__username", "user__first_name", "user__last_name",
@@ -2224,20 +2228,24 @@ def regrade(request, course_id, questionpaper_id, question_id=None,
                                   course.is_teacher(user)):
         raise Http404('You are not allowed to view this page!')
     questionpaper = get_object_or_404(QuestionPaper, pk=questionpaper_id)
-    details = []
     quiz = questionpaper.quiz
     data = {"user_id": user.id, "course_id": course_id,
             "questionpaper_id": questionpaper_id, "question_id": question_id,
             "answerpaper_id": answerpaper_id, "quiz_id": quiz.id,
             "quiz_name": quiz.description, "course_name": course.name
             }
-    regrade_papers.delay(data)
-    msg = dedent("""
-            {0} is submitted for re-evaluation. You will receive a
-            notification for the re-evaluation status
-            """.format(quiz.description)
-        )
-    messages.info(request, msg)
+    is_celery_alive = app.control.ping()
+    if is_celery_alive:
+        regrade_papers.delay(data)
+        msg = dedent("""
+                {0} is submitted for re-evaluation. You will receive a
+                notification for the re-evaluation status
+                """.format(quiz.description)
+            )
+        messages.info(request, msg)
+    else:
+        msg = "Unable to submit for regrade. Please contact admin"
+        messages.warning(request, msg)
     return redirect(
         reverse("yaksh:grade_user", args=[quiz.id, course_id])
     )
@@ -4045,78 +4053,24 @@ def upload_marks(request, course_id, questionpaper_id):
         if not is_csv_file:
             messages.warning(request, "The file uploaded is not a CSV file.")
             return redirect('yaksh:monitor', quiz.id, course_id)
-        try:
-            reader = csv.DictReader(
-                csv_file.read().decode('utf-8').splitlines(),
-                dialect=dialect)
-        except TypeError:
-            messages.warning(request, "Bad CSV file")
-            return redirect('yaksh:monitor', quiz.id, course_id)
-        question_ids = _get_header_info(reader)
-        _read_marks_csv(request, reader, course, question_paper, question_ids)
-    return redirect('yaksh:monitor', quiz.id, course_id)
-
-
-def _get_header_info(reader):
-    user_ids, question_ids = [], []
-    fields = reader.fieldnames
-    for field in fields:
-        if field.startswith('Q') and field.count('-') > 0:
-            qid = int(field.split('-')[1])
-            if qid not in question_ids:
-                question_ids.append(qid)
-    return question_ids
-
-
-def _read_marks_csv(request, reader, course, question_paper, question_ids):
-    messages.info(request, 'Marks Uploaded!')
-    for row in reader:
-        username = row['username']
-        user = User.objects.filter(username=username).first()
-        if user:
-            answerpapers = question_paper.answerpaper_set.filter(course=course,
-                                                         user_id=user.id)
+        data = {
+            "course_id": course_id, "questionpaper_id": questionpaper_id,
+            "csv_data": csv_file.read().decode('utf-8').splitlines(),
+            "user_id": request.user.id
+        }
+        is_celery_alive = app.control.ping()
+        if is_celery_alive:
+            update_user_marks.delay(data)
+            msg = dedent("""
+                {0} is submitted for marks update. You will receive a
+                notification for the update status
+                """.format(quiz.description)
+            )
+            messages.info(request, msg)
         else:
-            messages.info(request, '{0} user not found!'.format(username))
-            continue
-        answerpaper = answerpapers.last()
-        if not answerpaper:
-            messages.info(request, '{0} has no answerpaper!'.format(username))
-            continue
-        answers = answerpaper.answers.all()
-        questions = answerpaper.questions.all().values_list('id', flat=True)
-        for qid in question_ids:
-            question = Question.objects.filter(id=qid).first()
-            if not question:
-                messages.info(request,
-                             '{0} is an invalid question id!'.format(qid))
-                continue
-            if qid in questions:
-                answer = answers.filter(question_id=qid).last()
-                if not answer:
-                    answer = Answer(question_id=qid, marks=0, correct=False,
-                                    answer='Created During Marks Update!',
-                                    error=json.dumps([]))
-                    answer.save()
-                    answerpaper.answers.add(answer)
-                key1 = 'Q-{0}-{1}-{2}-marks'.format(qid, question.summary,
-                                                    question.points)
-                key2 = 'Q-{0}-{1}-comments'.format(qid, question.summary,
-                                                   question.points)
-                if key1 in reader.fieldnames:
-                    try:
-                        answer.set_marks(float(row[key1]))
-                    except ValueError:
-                        messages.info(request,
-                                     '{0} invalid marks!'.format(row[key1]))
-                if key2 in reader.fieldnames:
-                    answer.set_comment(row[key2])
-                answer.save()
-        answerpaper.update_marks(state='completed')
-        answerpaper.save()
-        messages.info(request,
-            'Updated successfully for user: {0}, question: {1}'.format(
-            username, question.summary))
+            msg = "Unable to submit for marks update. Please check with admin"
+            messages.warning(request, msg)
+    return redirect('yaksh:monitor', quiz.id, course_id)
 
 
 @login_required