Modify file upload with js client dropzone

9 files changed, 215 insertions, 117 deletions

diff --git a/online_test/settings.py b/online_test/settings.py
index bca24d5..4bd800d 100644
--- a/online_test/settings.py
+++ b/online_test/settings.py
@@ -30,6 +30,9 @@ DEBUG = True
 
 ALLOWED_HOSTS = []
 
+# This is a required field
+DOMAIN_HOST = "http://127.0.0.1:8000"
+
 URL_ROOT = ''
 
 # Application definition
@@ -256,9 +259,6 @@ CORS_ALLOW_CREDENTIALS = True
 # AWS Credentials
 USE_AWS = False
 if USE_AWS:
-    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-    STATICFILES_STORAGE = 'storages.backends.s3boto3.S3StaticStorage'
-
     AWS_ACCESS_KEY_ID = "access-key"
     AWS_SECRET_ACCESS_KEY = "secret-key"
     AWS_S3_REGION_NAME = "ap-south-1"
@@ -272,11 +272,7 @@ if USE_AWS:
     AWS_STATIC_LOCATION = 'static'
     STATICFILES_STORAGE = 'yaksh.storage_backends.StaticStorage'
     STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_STATIC_LOCATION}/"
-
     # Media Public
     AWS_PUBLIC_MEDIA_LOCATION = 'media/public'
     DEFAULT_FILE_STORAGE = 'yaksh.storage_backends.PublicMediaStorage'
 
-    # Media Private
-    AWS_PRIVATE_MEDIA_LOCATION = 'media/private'
-    PRIVATE_FILE_STORAGE = 'yaksh.storage_backends.PrivateMediaStorage'
diff --git a/requirements/requirements-common.txt b/requirements/requirements-common.txt
index 9805dc5..320b145 100644
--- a/requirements/requirements-common.txt
+++ b/requirements/requirements-common.txt
@@ -21,3 +21,5 @@ django-cors-headers==3.1.0
 Pillow
 pandas
 more-itertools==8.4.0
+django-storages==1.11.1
+boto3==1.17.17
diff --git a/yaksh/models.py b/yaksh/models.py
index 613bf00..51c2aa8 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -43,7 +43,8 @@ from django.template import Context, Template
 from django.conf import settings
 from django.forms.models import model_to_dict
 from django.db.models import Count
-
+from django.db.models.signals import pre_delete
+from django.db.models.fields.files import FieldFile
 # Local Imports
 from yaksh.code_server import (
     submit, get_result as get_result_from_code_server
@@ -51,7 +52,6 @@ from yaksh.code_server import (
 from yaksh.settings import SERVER_POOL_PORT, SERVER_HOST_NAME
 from .file_utils import extract_files, delete_files
 from grades.models import GradingSystem
-from yaksh.storage_backends import PublicMediaStorage
 
 
 languages = (
@@ -272,6 +272,17 @@ def is_valid_time_format(time):
     return status
 
 
+def file_cleanup(sender, instance, *args, **kwargs):
+    '''
+        Deletes the file(s) associated with a model instance. The model
+        is not saved after deletion of the file(s) since this is meant
+        to be used with the pre_delete signal.
+    '''
+    for field_name, _ in instance.__dict__.items():
+        field = getattr(instance, field_name)
+        if issubclass(field.__class__, FieldFile) and field.name:
+            field.delete(save=False)
+
 ###############################################################################
 class CourseManager(models.Manager):
 
@@ -1424,11 +1435,10 @@ class Question(models.Model):
         ]
     }
 
-    def consolidate_answer_data(self, user_answer, user=None):
+    def consolidate_answer_data(self, user_answer, user=None, regrade=False):
         question_data = {}
         metadata = {}
         test_case_data = []
-
         test_cases = self.get_test_cases()
 
         for test in test_cases:
@@ -1441,20 +1451,34 @@ class Question(models.Model):
         metadata['partial_grading'] = self.partial_grading
         files = FileUpload.objects.filter(question=self)
         if files:
-            metadata['file_paths'] = [(file.file.path, file.extract)
-                                      for file in files]
-        if self.type == "upload":
-            assignment_files = AssignmentUpload.objects.filter(
-                assignmentQuestion=self
-                )
-            if assignment_files:
-                metadata['assign_files'] = [(file.assignmentFile.url, False)
-                                            for file in assignment_files]
-                print(metadata['assign_files'])
+            if settings.USE_AWS:
+                metadata['file_paths'] = [
+                    (file.file.url, file.extract)
+                     for file in files
+                ]
+            else:
+                metadata['file_paths'] = [
+                    (self.get_file_url(file.file.url), file.extract)
+                     for file in files
+                ]
+        if self.type == "upload" and regrade:
+            file = AssignmentUpload.objects.only(
+                "assignmentFile").filter(
+                assignmentQuestion_id=self.id, answer_paper__user_id=user.id
+                ).order_by("-id").first()
+            if file:
+                if settings.USE_AWS:
+                    metadata['assign_files'] = [file.assignmentFile.url]
+                else:
+                    metadata['assign_files'] = [
+                        self.get_file_url(file.assignmentFile.url)
+                    ]
         question_data['metadata'] = metadata
-
         return json.dumps(question_data)
 
+    def get_file_url(self, path):
+        return f'{settings.DOMAIN_HOST}{path}'
+
     def dump_questions(self, question_ids, user):
         questions = Question.objects.filter(id__in=question_ids,
                                             user_id=user.id, active=True
@@ -1693,7 +1717,7 @@ class FileUpload(models.Model):
     def get_filename(self):
         return os.path.basename(self.file.name)
 
-
+pre_delete.connect(file_cleanup, sender=FileUpload)
 ###############################################################################
 class Answer(models.Model):
     """Answers submitted by the users."""
@@ -2599,7 +2623,7 @@ class AnswerPaper(models.Model):
                 return (False, f'{msg} {question.type} answer submission error')
         else:
             answer = user_answer.answer
-        json_data = question.consolidate_answer_data(answer) \
+        json_data = question.consolidate_answer_data(answer, self.user, True) \
             if question.type == 'code' else None
         result = self.validate_answer(answer, question,
                                       json_data, user_answer.id,
@@ -2649,7 +2673,11 @@ class AssignmentUploadManager(models.Manager):
                         answer_paper__question_paper=qp,
                         answer_paper__course_id=course_id
                         )
-            file_name = User.objects.get(id=user_id).get_full_name()
+            user_name = assignment_files.values_list(
+                "answer_paper__user__first_name",
+                "answer_paper__user__last_name"
+            )[0]
+            file_name = "_".join(user_name)
         else:
             assignment_files = AssignmentUpload.objects.filter(
                         answer_paper__question_paper=qp,
@@ -2676,6 +2704,8 @@ class AssignmentUpload(models.Model):
     def __str__(self):
         return f'Assignment File of the user {self.answer_paper.user}'
 
+pre_delete.connect(file_cleanup, sender=AssignmentUpload)
+
 
 ##############################################################################
 class TestCase(models.Model):
diff --git a/yaksh/static/yaksh/js/requesthandler.js b/yaksh/static/yaksh/js/requesthandler.js
index 80b67fb..0a06c79 100644
--- a/yaksh/static/yaksh/js/requesthandler.js
+++ b/yaksh/static/yaksh/js/requesthandler.js
@@ -145,8 +145,49 @@ var global_editor = {};
 var csrftoken = jQuery("[name=csrfmiddlewaretoken]").val();
 var err_lineno;
 var marker;
+Dropzone.autoDiscover = false;
+var submitfiles;
 $(document).ready(function(){
-  if(is_exercise == "True" && can_skip == "False"){
+  var filezone = $("div#dropzone_file").dropzone({
+    url: $("#code").attr("action"),
+    parallelUploads: 10,
+    uploadMultiple: true,
+    maxFiles:20,
+    paramName: "assignment",
+    autoProcessQueue: false,
+    init: function() {
+      var submitButton = document.querySelector("#check");
+      myDropzone = this;
+      submitButton.addEventListener("click", function(e) {
+        if (myDropzone.getQueuedFiles().length === 0) {
+          $("#upload_alert").modal("show");
+          e.preventDefault();
+          return;
+        }
+        if (myDropzone.getAcceptedFiles().length > 0) {
+          if (submitfiles === true) {
+              submitfiles = false;
+              return;
+          }
+          e.preventDefault();
+          myDropzone.processQueue();
+          myDropzone.on("complete", function () {
+              submitfiles = true;
+              $('#check').trigger('click');
+          });
+        }
+      });
+    },
+    success: function (file, response) {
+      document.open();
+      document.write(response);
+      document.close();
+    },
+    headers: {
+        "X-CSRFToken": document.getElementById("code").elements[0].value
+    }
+  });
+  if(is_exercise == "True" && can_skip == "False") {
       setTimeout(function() {show_solution();}, delay_time*1000);
   }
   // Codemirror object, language modes and initial content
diff --git a/yaksh/storage_backends.py b/yaksh/storage_backends.py
index 4d08c8c..206e456 100644
--- a/yaksh/storage_backends.py
+++ b/yaksh/storage_backends.py
@@ -3,16 +3,16 @@ from storages.backends.s3boto3 import S3Boto3Storage
 
 
 class StaticStorage(S3Boto3Storage):
-    location = settings.AWS_STATIC_LOCATION if settings.USE_AWS else settings.STATIC_URL
+    if settings.USE_AWS:
+        location = settings.AWS_STATIC_LOCATION
+    else:
+        pass
 
 
 class PublicMediaStorage(S3Boto3Storage):
-    location = settings.AWS_PUBLIC_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL
-    file_overwrite = False
-
-
-class PrivateMediaStorage(S3Boto3Storage):
-    location = settings.AWS_PRIVATE_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL
-    default_acl = 'private'
-    file_overwrite = False
-    custom_domain = False
\ No newline at end of file
+    if settings.USE_AWS:
+        location = settings.AWS_PUBLIC_MEDIA_LOCATION
+        file_overwrite = True
+        default_acl = 'public-read'
+    else:
+        pass
diff --git a/yaksh/templates/yaksh/complete.html b/yaksh/templates/yaksh/complete.html
index 4d921e1..79a1392 100644
--- a/yaksh/templates/yaksh/complete.html
+++ b/yaksh/templates/yaksh/complete.html
@@ -20,58 +20,58 @@
     </div>
 </center>
 {% endif %}
-{% csrf_token %}
-    {% if paper.questions_answered.all or paper.questions_unanswered.all %}
-        <center>
-        <div class="col-md-8">
-            <h3>Submission Status</h3>
-            <table class="table table-dark table-responsive-sm" >
-                <thead class="thead-dark">
+{% if paper.questions_answered.all or paper.questions_unanswered.all %}
+    <center>
+    <div class="col-md-8">
+        <h3>Submission Status</h3>
+        <table class="table table-dark table-responsive-sm" >
+            <thead class="thead-dark">
+                <tr>
+                    <th> Question</th>
+                    <th> Status </th>
+                </tr>
+            </thead>
+            <tbody class="list">
+            {% for question in paper.questions.all %}
+                {% if question in paper.questions_answered.all %}
                     <tr>
-                        <th> Question</th>
-                        <th> Status </th>
-                    </tr>
-                </thead>
-                <tbody class="list">
-                {% for question in paper.questions.all %}
-                    {% if question in paper.questions_answered.all %}
-                        <tr>
-                        <td> {{ question.summary }} </td>
-                        <td> <span class="badge badge-success">Attempted</span> </td>
-                    {% else %}
-                        <tr>
-                        <td> {{ question }} </td>
-                        <td> <span class="badge badge-warning">Not Attempted</span> </td>
-                    {% endif %}
-                    </tr>
-                {% endfor %}
-                </tbody>
-            </table>
-        </div>
-        </center>
-    {% endif %}
-    <br><br>
-	<center class="container">
-        <h5>
-            <span class="alert alert-success">{{message}}</span>
-        </h5>
+                    <td> {{ question.summary }} </td>
+                    <td> <span class="badge badge-success">Attempted</span> </td>
+                {% else %}
+                    <tr>
+                    <td> {{ question }} </td>
+                    <td> <span class="badge badge-warning">Not Attempted</span> </td>
+                {% endif %}
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
     </center>
-    <center>
-    <br>
-    {% if module_id and not paper.course.is_trial %}
-        {% if first_unit %}
-            <a href="{% url 'yaksh:next_unit' course_id module_id learning_unit.id '1' %}" class="btn btn-info btn-lg" id="Next"> Next
-                <span class="fa fa-step-forward">
-                </span>
-            </a>
-        {% else %}
-            <a href="{% url 'yaksh:next_unit' course_id module_id learning_unit.id %}" class="btn btn-info btn-lg" id="Next"> Next
-                <span class="fa fa-step-forward">
-                </span>
-            </a>
-        {% endif %}
+{% endif %}
+<br><br>
+<center class="container">
+    <h5>
+        <span class="alert alert-success">{{message}}</span>
+    </h5>
+</center>
+<center>
+<br>
+{% if module_id and not paper.course.is_trial %}
+    {% if first_unit %}
+        <a href="{% url 'yaksh:next_unit' course_id module_id learning_unit.id '1' %}" class="btn btn-info btn-lg" id="Next"> Next
+            <span class="fa fa-step-forward">
+            </span>
+        </a>
     {% else %}
-        <a href="{% url 'yaksh:index' %}" id="home" class="btn btn-primary btn-lg"> Home </a>
+        <a href="{% url 'yaksh:next_unit' course_id module_id learning_unit.id %}" class="btn btn-info btn-lg" id="Next"> Next
+            <span class="fa fa-step-forward">
+            </span>
+        </a>
     {% endif %}
-    </center>
+{% else %}
+    <a href="{% url 'yaksh:index' %}" id="home" class="btn btn-primary btn-lg"> Home </a>
+{% endif %}
+</center>
+<br><br>
 {% endblock content %}
diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html
index 56edcf3..2779db2 100644
--- a/yaksh/templates/yaksh/question.html
+++ b/yaksh/templates/yaksh/question.html
@@ -7,6 +7,8 @@
 <link rel="stylesheet" href="{% static 'yaksh/css/question.css' %}" type="text/css" />
 <link rel="stylesheet" href="{% static 'yaksh/css/codemirror/lib/codemirror.css' %}" type="text/css" />
 <link rel="stylesheet" href="{% static 'yaksh/css/exam.css' %}" type="text/css" />
+<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.8.1/dropzone.min.css">
+<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.8.1/basic.min.css">
 <style>
     .CodeMirror{
         border-style: groove;
@@ -21,6 +23,7 @@
 {% block script %}
 
 
+<script src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.8.1/min/dropzone.min.js"></script>
 <script src="{% static 'yaksh/js/requesthandler.js' %}"></script>
 <script src="{% static 'yaksh/js/codemirror/lib/codemirror.js' %}"></script>
 <script src="{% static 'yaksh/js/codemirror/mode/python/python.js' %}"></script>
@@ -29,6 +32,7 @@
 <script src="{% static 'yaksh/js/codemirror/mode/r/r.js' %}"></script>
 <script type="text/javascript" src="{% static 'yaksh/js/mathjax/MathJax.js' %}?config=TeX-MML-AM_CHTML"></script>
 <script src="{% static 'yaksh/js/jquery-sortable.js' %}"></script>
+
 <script>
 init_val = '{{ last_attempt|escape_quotes|safe }}';
 lang = "{{ question.language }}"
@@ -142,7 +146,6 @@ question_type = "{{ question.type }}";
         </div>
       {% endif %}
     </center>
-  </div>
   <form id="code" action="{% url 'yaksh:check' question.id paper.attempt_number module.id paper.question_paper.id course.id %}" method="post" enctype="multipart/form-data">
     {% csrf_token %}
     <input type=hidden name="question_id" id="question_id" value="{{ question.id }}"></input>
@@ -174,6 +177,9 @@ question_type = "{{ question.type }}";
               <small><strong>Marks: {{ question.points }}</strong></small>
             </span>
           </div>
+          <div class="badge badge-warning">
+            Last submitted answer is considered for evaluation
+          </div>
         </div>
       </div>
       <div class="card-body">
@@ -181,10 +187,10 @@ question_type = "{{ question.type }}";
           {{ question.description|safe }}
         </div>
         {% if files %}
-          <div class="col-md-5">
+          <div class="col-md-8">
               <div class="card">
                 <div class="card-header">
-                  <span> Files to download for this question </span>
+                  <span class="badge badge-info"> Files to download for this question </span>
                 </div>
                 <div class="card-body">
                   {% for f_name in files %}
@@ -265,8 +271,16 @@ question_type = "{{ question.type }}";
           <!-- Upload type question -->
           {% if question.type == "upload" %}
             <p>Upload assignment file for the said question<p>
-            <input type=file id="assignment" name="assignment" multiple="">
+            <div class="dropzone needsclick dz-clickable" id="dropzone_file">
+              <div class="dz-message needsclick">    
+                <button type="button" class="dz-button">
+                  Drop files here or click to upload.
+                </button>
+              </div>
+            </div>
+            <br>
             {% if assignment_files %}
+            <div>
                 <ul class="list-group">
                 {% for as_file in assignment_files %}
                     <li class="list-group-item">
@@ -274,6 +288,7 @@ question_type = "{{ question.type }}";
                     </li>
                 {% endfor %}
                 </ul>
+            </div>
             {% endif %}
           {% endif %}
 
diff --git a/yaksh/templates/yaksh/quit.html b/yaksh/templates/yaksh/quit.html
index 828ad60..e829b3c 100644
--- a/yaksh/templates/yaksh/quit.html
+++ b/yaksh/templates/yaksh/quit.html
@@ -57,4 +57,5 @@
         {% endif %}
     </center>
 	</form>
+    <br><br>
 {% endblock content %}
diff --git a/yaksh/views.py b/yaksh/views.py
index 7b1e038..f0fe2e8 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -21,6 +21,7 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.contrib import messages
 from taggit.models import Tag
 from django.urls import reverse
+from django.conf import settings
 import json
 from textwrap import dedent
 import zipfile
@@ -251,9 +252,7 @@ def add_question(request, question_id=None):
         extract_files_id = request.POST.getlist('extract')
         hide_files_id = request.POST.getlist('hide')
         if remove_files_id:
-            files = FileUpload.objects.filter(id__in=remove_files_id)
-            for file in files:
-                file.remove()
+            files = FileUpload.objects.filter(id__in=remove_files_id).delete()
         if extract_files_id:
             files = FileUpload.objects.filter(id__in=extract_files_id)
             for file in files:
@@ -847,8 +846,11 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None,
         elif current_question.type == 'upload':
             # if time-up at upload question then the form is submitted without
             # validation
-            assignment_filename = request.FILES.getlist('assignment')
-            if not assignment_filename:
+            assign_files = []
+            assignments = request.FILES
+            for i in range(len(assignments)):
+                assign_files.append(assignments[f"assignment[{i}]"])
+            if not assign_files:
                 msg = "Please upload assignment file"
                 return show_question(
                     request, current_question, paper, notification=msg,
@@ -856,26 +858,29 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None,
                     previous_question=current_question
                 )
             uploaded_files = []
-            for fname in assignment_filename:
+            AssignmentUpload.objects.filter(
+                assignmentQuestion_id=current_question.id,
+                answer_paper_id=paper.id
+                ).delete()
+            for fname in assign_files:
                 fname._name = fname._name.replace(" ", "_")
                 uploaded_files.append(AssignmentUpload(
-                                    assignmentQuestion=current_question,
+                                    assignmentQuestion_id=current_question.id,
                                     assignmentFile=fname,
                                     answer_paper_id=paper.id
                                 ))
             AssignmentUpload.objects.bulk_create(uploaded_files)
             user_answer = 'ASSIGNMENT UPLOADED'
-            if not current_question.grade_assignment_upload:
-                new_answer = Answer(
-                    question=current_question, answer=user_answer,
-                    correct=False, error=json.dumps([])
-                )
-                new_answer.save()
-                paper.answers.add(new_answer)
-                next_q = paper.add_completed_question(current_question.id)
-                return show_question(request, next_q, paper,
-                                     course_id=course_id, module_id=module_id,
-                                     previous_question=current_question)
+            new_answer = Answer(
+                question=current_question, answer=user_answer,
+                correct=False, error=json.dumps([])
+            )
+            new_answer.save()
+            paper.answers.add(new_answer)
+            next_q = paper.add_completed_question(current_question.id)
+            return show_question(request, next_q, paper,
+                                 course_id=course_id, module_id=module_id,
+                                 previous_question=current_question)
         else:
             user_answer = request.POST.get('answer')
         if not is_valid_answer(user_answer):
@@ -902,12 +907,11 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None,
         # questions, we obtain the results via XML-RPC with the code executed
         # safely in a separate process (the code_server.py) running as nobody.
         json_data = current_question.consolidate_answer_data(
-            user_answer, user) if current_question.type == 'code' or \
-            current_question.type == 'upload' else None
+            user_answer, user) if current_question.type == 'code' else None
         result = paper.validate_answer(
             user_answer, current_question, json_data, uid
         )
-        if current_question.type in ['code', 'upload']:
+        if current_question.type == 'code':
             if (paper.time_left() <= 0 and not
                     paper.question_paper.quiz.is_exercise):
                 url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT)
@@ -2388,12 +2392,21 @@ def download_assignment_file(request, quiz_id, course_id,
     for f_name in assignment_files:
         folder = f_name.answer_paper.user.get_full_name().replace(" ", "_")
         sub_folder = f_name.assignmentQuestion.summary.replace(" ", "_")
-        folder_name = os.sep.join((folder, sub_folder, os.path.basename(
-                        f_name.assignmentFile.name))
-                        )
-        zip_file.write(
-            f_name.assignmentFile.path, folder_name
-        )
+        if settings.USE_AWS:
+            folder_name = os.sep.join((folder, sub_folder))
+            download_url = f_name.assignmentFile.url
+            zip_file.writestr(
+                os.path.join(folder_name, os.path.basename(download_url)),
+                f_name.assignmentFile.read()
+            )
+        else:
+            folder_name = os.sep.join((folder, sub_folder, os.path.basename(
+                            f_name.assignmentFile.name))
+                            )
+            zip_file.write(
+                f_name.assignmentFile.path, folder_name
+            )
+
     zip_file.close()
     zipfile_name.seek(0)
     response = HttpResponse(content_type='application/zip')