From 542433598aad0efffee7619e1f113425147bcec0 Mon Sep 17 00:00:00 2001 From: adityacp Date: Mon, 1 Mar 2021 11:13:07 +0530 Subject: Add AWS support for file upload --- online_test/settings.py | 34 ++++++++++++++++++++++++++++++++-- yaksh/admin.py | 3 ++- yaksh/models.py | 9 +++++++-- yaksh/storage_backends.py | 18 ++++++++++++++++++ yaksh/templates/yaksh/add_quiz.html | 4 ++-- yaksh/views.py | 18 +++++++++--------- 6 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 yaksh/storage_backends.py diff --git a/online_test/settings.py b/online_test/settings.py index e7e19a0..bca24d5 100644 --- a/online_test/settings.py +++ b/online_test/settings.py @@ -52,7 +52,8 @@ INSTALLED_APPS = ( 'rest_framework', 'api', 'corsheaders', - 'rest_framework.authtoken' + 'rest_framework.authtoken', + 'storages' ) MIDDLEWARE = ( @@ -162,7 +163,7 @@ PRODUCTION_URL = 'your_project_url' IS_DEVELOPMENT = True # Video File upload size -MAX_UPLOAD_SIZE = 52428800 +MAX_UPLOAD_SIZE = 524288000 DEFAULT_FROM_EMAIL = EMAIL_HOST_USER @@ -250,3 +251,32 @@ REST_FRAMEWORK = { CORS_ORIGIN_ALLOW_ALL = True CORS_ALLOW_CREDENTIALS = True + + +# AWS Credentials +USE_AWS = False +if USE_AWS: + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + STATICFILES_STORAGE = 'storages.backends.s3boto3.S3StaticStorage' + + AWS_ACCESS_KEY_ID = "access-key" + AWS_SECRET_ACCESS_KEY = "secret-key" + AWS_S3_REGION_NAME = "ap-south-1" + AWS_STORAGE_BUCKET_NAME = "yaksh-django" + AWS_S3_CUSTOM_DOMAIN = f'{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com' + AWS_DEFAULT_ACL = 'public-read' + AWS_S3_SIGNATURE_VERSION = 's3v4' + AWS_S3_ADDRESSING_STYLE = "virtual" + + # Static Location + AWS_STATIC_LOCATION = 'static' + STATICFILES_STORAGE = 'yaksh.storage_backends.StaticStorage' + STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_STATIC_LOCATION}/" + + # Media Public + AWS_PUBLIC_MEDIA_LOCATION = 'media/public' + DEFAULT_FILE_STORAGE = 'yaksh.storage_backends.PublicMediaStorage' + + # Media Private + AWS_PRIVATE_MEDIA_LOCATION = 'media/private' + PRIVATE_FILE_STORAGE = 'yaksh.storage_backends.PrivateMediaStorage' diff --git a/yaksh/admin.py b/yaksh/admin.py index e98c7c5..011e24f 100644 --- a/yaksh/admin.py +++ b/yaksh/admin.py @@ -2,7 +2,7 @@ from yaksh.models import Question, Quiz, QuestionPaper, Profile from yaksh.models import (TestCase, StandardTestCase, StdIOBasedTestCase, Course, AnswerPaper, CourseStatus, LearningModule, Lesson, Post, Comment, Topic, TableOfContents, - LessonQuizAnswer, Answer + LessonQuizAnswer, Answer, AssignmentUpload ) from django.contrib import admin @@ -64,3 +64,4 @@ admin.site.register(Topic) admin.site.register(TableOfContents) admin.site.register(LessonQuizAnswer) admin.site.register(Answer) +admin.site.register(AssignmentUpload) \ No newline at end of file diff --git a/yaksh/models.py b/yaksh/models.py index 11ddf8a..dd3838a 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -51,6 +51,8 @@ from yaksh.code_server import ( from yaksh.settings import SERVER_POOL_PORT, SERVER_HOST_NAME from .file_utils import extract_files, delete_files from grades.models import GradingSystem +from yaksh.storage_backends import PublicMediaStorage + languages = ( ("python", "Python"), @@ -1442,8 +1444,9 @@ class Question(models.Model): assignmentQuestion=self, user=user ) if assignment_files: - metadata['assign_files'] = [(file.assignmentFile.path, False) + metadata['assign_files'] = [(file.assignmentFile.url, False) for file in assignment_files] + print(metadata['assign_files']) question_data['metadata'] = metadata return json.dumps(question_data) @@ -2661,7 +2664,9 @@ class AssignmentUploadManager(models.Manager): class AssignmentUpload(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) assignmentQuestion = models.ForeignKey(Question, on_delete=models.CASCADE) - assignmentFile = models.FileField(upload_to=get_assignment_dir, max_length=255) + assignmentFile = models.FileField( + upload_to=get_assignment_dir, max_length=255 + ) question_paper = models.ForeignKey(QuestionPaper, blank=True, null=True, on_delete=models.CASCADE) course = models.ForeignKey(Course, null=True, blank=True, diff --git a/yaksh/storage_backends.py b/yaksh/storage_backends.py new file mode 100644 index 0000000..4d08c8c --- /dev/null +++ b/yaksh/storage_backends.py @@ -0,0 +1,18 @@ +from django.conf import settings +from storages.backends.s3boto3 import S3Boto3Storage + + +class StaticStorage(S3Boto3Storage): + location = settings.AWS_STATIC_LOCATION if settings.USE_AWS else settings.STATIC_URL + + +class PublicMediaStorage(S3Boto3Storage): + location = settings.AWS_PUBLIC_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL + file_overwrite = False + + +class PrivateMediaStorage(S3Boto3Storage): + location = settings.AWS_PRIVATE_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL + default_acl = 'private' + file_overwrite = False + custom_domain = False \ No newline at end of file diff --git a/yaksh/templates/yaksh/add_quiz.html b/yaksh/templates/yaksh/add_quiz.html index 1609639..d43c1c6 100644 --- a/yaksh/templates/yaksh/add_quiz.html +++ b/yaksh/templates/yaksh/add_quiz.html @@ -65,11 +65,11 @@

You can check the quiz by attempting it in the following modes:

- + Try as student - + Try as teacher diff --git a/yaksh/views.py b/yaksh/views.py index 1965191..8422110 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -858,15 +858,15 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None, ) for fname in assignment_filename: fname._name = fname._name.replace(" ", "_") - assignment_files = AssignmentUpload.objects.filter( - assignmentQuestion=current_question, course_id=course_id, - assignmentFile__icontains=fname, user=user, - question_paper=questionpaper_id) - if assignment_files.exists(): - assign_file = assignment_files.first() - if os.path.exists(assign_file.assignmentFile.path): - os.remove(assign_file.assignmentFile.path) - assign_file.delete() + # assignment_files = AssignmentUpload.objects.filter( + # assignmentQuestion=current_question, course_id=course_id, + # assignmentFile__icontains=fname, user=user, + # question_paper=questionpaper_id) + # if assignment_files.exists(): + # assign_file = assignment_files.first() + # if os.path.exists(assign_file.assignmentFile.path): + # os.remove(assign_file.assignmentFile.path) + # assign_file.delete() AssignmentUpload.objects.create( user=user, assignmentQuestion=current_question, course_id=course_id, -- cgit From cc101bd195cc456f7bace4e9b646e2190975dde3 Mon Sep 17 00:00:00 2001 From: adityacp Date: Tue, 23 Mar 2021 11:53:39 +0530 Subject: Modify file upload with js client dropzone --- online_test/settings.py | 10 +--- requirements/requirements-common.txt | 2 + yaksh/models.py | 66 +++++++++++++++------ yaksh/static/yaksh/js/requesthandler.js | 43 +++++++++++++- yaksh/storage_backends.py | 20 +++---- yaksh/templates/yaksh/complete.html | 100 ++++++++++++++++---------------- yaksh/templates/yaksh/question.html | 23 ++++++-- yaksh/templates/yaksh/quit.html | 1 + yaksh/views.py | 67 ++++++++++++--------- 9 files changed, 215 insertions(+), 117 deletions(-) diff --git a/online_test/settings.py b/online_test/settings.py index bca24d5..4bd800d 100644 --- a/online_test/settings.py +++ b/online_test/settings.py @@ -30,6 +30,9 @@ DEBUG = True ALLOWED_HOSTS = [] +# This is a required field +DOMAIN_HOST = "http://127.0.0.1:8000" + URL_ROOT = '' # Application definition @@ -256,9 +259,6 @@ CORS_ALLOW_CREDENTIALS = True # AWS Credentials USE_AWS = False if USE_AWS: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - STATICFILES_STORAGE = 'storages.backends.s3boto3.S3StaticStorage' - AWS_ACCESS_KEY_ID = "access-key" AWS_SECRET_ACCESS_KEY = "secret-key" AWS_S3_REGION_NAME = "ap-south-1" @@ -272,11 +272,7 @@ if USE_AWS: AWS_STATIC_LOCATION = 'static' STATICFILES_STORAGE = 'yaksh.storage_backends.StaticStorage' STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_STATIC_LOCATION}/" - # Media Public AWS_PUBLIC_MEDIA_LOCATION = 'media/public' DEFAULT_FILE_STORAGE = 'yaksh.storage_backends.PublicMediaStorage' - # Media Private - AWS_PRIVATE_MEDIA_LOCATION = 'media/private' - PRIVATE_FILE_STORAGE = 'yaksh.storage_backends.PrivateMediaStorage' diff --git a/requirements/requirements-common.txt b/requirements/requirements-common.txt index 9805dc5..320b145 100644 --- a/requirements/requirements-common.txt +++ b/requirements/requirements-common.txt @@ -21,3 +21,5 @@ django-cors-headers==3.1.0 Pillow pandas more-itertools==8.4.0 +django-storages==1.11.1 +boto3==1.17.17 diff --git a/yaksh/models.py b/yaksh/models.py index 613bf00..51c2aa8 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -43,7 +43,8 @@ from django.template import Context, Template from django.conf import settings from django.forms.models import model_to_dict from django.db.models import Count - +from django.db.models.signals import pre_delete +from django.db.models.fields.files import FieldFile # Local Imports from yaksh.code_server import ( submit, get_result as get_result_from_code_server @@ -51,7 +52,6 @@ from yaksh.code_server import ( from yaksh.settings import SERVER_POOL_PORT, SERVER_HOST_NAME from .file_utils import extract_files, delete_files from grades.models import GradingSystem -from yaksh.storage_backends import PublicMediaStorage languages = ( @@ -272,6 +272,17 @@ def is_valid_time_format(time): return status +def file_cleanup(sender, instance, *args, **kwargs): + ''' + Deletes the file(s) associated with a model instance. The model + is not saved after deletion of the file(s) since this is meant + to be used with the pre_delete signal. + ''' + for field_name, _ in instance.__dict__.items(): + field = getattr(instance, field_name) + if issubclass(field.__class__, FieldFile) and field.name: + field.delete(save=False) + ############################################################################### class CourseManager(models.Manager): @@ -1424,11 +1435,10 @@ class Question(models.Model): ] } - def consolidate_answer_data(self, user_answer, user=None): + def consolidate_answer_data(self, user_answer, user=None, regrade=False): question_data = {} metadata = {} test_case_data = [] - test_cases = self.get_test_cases() for test in test_cases: @@ -1441,20 +1451,34 @@ class Question(models.Model): metadata['partial_grading'] = self.partial_grading files = FileUpload.objects.filter(question=self) if files: - metadata['file_paths'] = [(file.file.path, file.extract) - for file in files] - if self.type == "upload": - assignment_files = AssignmentUpload.objects.filter( - assignmentQuestion=self - ) - if assignment_files: - metadata['assign_files'] = [(file.assignmentFile.url, False) - for file in assignment_files] - print(metadata['assign_files']) + if settings.USE_AWS: + metadata['file_paths'] = [ + (file.file.url, file.extract) + for file in files + ] + else: + metadata['file_paths'] = [ + (self.get_file_url(file.file.url), file.extract) + for file in files + ] + if self.type == "upload" and regrade: + file = AssignmentUpload.objects.only( + "assignmentFile").filter( + assignmentQuestion_id=self.id, answer_paper__user_id=user.id + ).order_by("-id").first() + if file: + if settings.USE_AWS: + metadata['assign_files'] = [file.assignmentFile.url] + else: + metadata['assign_files'] = [ + self.get_file_url(file.assignmentFile.url) + ] question_data['metadata'] = metadata - return json.dumps(question_data) + def get_file_url(self, path): + return f'{settings.DOMAIN_HOST}{path}' + def dump_questions(self, question_ids, user): questions = Question.objects.filter(id__in=question_ids, user_id=user.id, active=True @@ -1693,7 +1717,7 @@ class FileUpload(models.Model): def get_filename(self): return os.path.basename(self.file.name) - +pre_delete.connect(file_cleanup, sender=FileUpload) ############################################################################### class Answer(models.Model): """Answers submitted by the users.""" @@ -2599,7 +2623,7 @@ class AnswerPaper(models.Model): return (False, f'{msg} {question.type} answer submission error') else: answer = user_answer.answer - json_data = question.consolidate_answer_data(answer) \ + json_data = question.consolidate_answer_data(answer, self.user, True) \ if question.type == 'code' else None result = self.validate_answer(answer, question, json_data, user_answer.id, @@ -2649,7 +2673,11 @@ class AssignmentUploadManager(models.Manager): answer_paper__question_paper=qp, answer_paper__course_id=course_id ) - file_name = User.objects.get(id=user_id).get_full_name() + user_name = assignment_files.values_list( + "answer_paper__user__first_name", + "answer_paper__user__last_name" + )[0] + file_name = "_".join(user_name) else: assignment_files = AssignmentUpload.objects.filter( answer_paper__question_paper=qp, @@ -2676,6 +2704,8 @@ class AssignmentUpload(models.Model): def __str__(self): return f'Assignment File of the user {self.answer_paper.user}' +pre_delete.connect(file_cleanup, sender=AssignmentUpload) + ############################################################################## class TestCase(models.Model): diff --git a/yaksh/static/yaksh/js/requesthandler.js b/yaksh/static/yaksh/js/requesthandler.js index 80b67fb..0a06c79 100644 --- a/yaksh/static/yaksh/js/requesthandler.js +++ b/yaksh/static/yaksh/js/requesthandler.js @@ -145,8 +145,49 @@ var global_editor = {}; var csrftoken = jQuery("[name=csrfmiddlewaretoken]").val(); var err_lineno; var marker; +Dropzone.autoDiscover = false; +var submitfiles; $(document).ready(function(){ - if(is_exercise == "True" && can_skip == "False"){ + var filezone = $("div#dropzone_file").dropzone({ + url: $("#code").attr("action"), + parallelUploads: 10, + uploadMultiple: true, + maxFiles:20, + paramName: "assignment", + autoProcessQueue: false, + init: function() { + var submitButton = document.querySelector("#check"); + myDropzone = this; + submitButton.addEventListener("click", function(e) { + if (myDropzone.getQueuedFiles().length === 0) { + $("#upload_alert").modal("show"); + e.preventDefault(); + return; + } + if (myDropzone.getAcceptedFiles().length > 0) { + if (submitfiles === true) { + submitfiles = false; + return; + } + e.preventDefault(); + myDropzone.processQueue(); + myDropzone.on("complete", function () { + submitfiles = true; + $('#check').trigger('click'); + }); + } + }); + }, + success: function (file, response) { + document.open(); + document.write(response); + document.close(); + }, + headers: { + "X-CSRFToken": document.getElementById("code").elements[0].value + } + }); + if(is_exercise == "True" && can_skip == "False") { setTimeout(function() {show_solution();}, delay_time*1000); } // Codemirror object, language modes and initial content diff --git a/yaksh/storage_backends.py b/yaksh/storage_backends.py index 4d08c8c..206e456 100644 --- a/yaksh/storage_backends.py +++ b/yaksh/storage_backends.py @@ -3,16 +3,16 @@ from storages.backends.s3boto3 import S3Boto3Storage class StaticStorage(S3Boto3Storage): - location = settings.AWS_STATIC_LOCATION if settings.USE_AWS else settings.STATIC_URL + if settings.USE_AWS: + location = settings.AWS_STATIC_LOCATION + else: + pass class PublicMediaStorage(S3Boto3Storage): - location = settings.AWS_PUBLIC_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL - file_overwrite = False - - -class PrivateMediaStorage(S3Boto3Storage): - location = settings.AWS_PRIVATE_MEDIA_LOCATION if settings.USE_AWS else settings.MEDIA_URL - default_acl = 'private' - file_overwrite = False - custom_domain = False \ No newline at end of file + if settings.USE_AWS: + location = settings.AWS_PUBLIC_MEDIA_LOCATION + file_overwrite = True + default_acl = 'public-read' + else: + pass diff --git a/yaksh/templates/yaksh/complete.html b/yaksh/templates/yaksh/complete.html index 4d921e1..79a1392 100644 --- a/yaksh/templates/yaksh/complete.html +++ b/yaksh/templates/yaksh/complete.html @@ -20,58 +20,58 @@ {% endif %} -{% csrf_token %} - {% if paper.questions_answered.all or paper.questions_unanswered.all %} -
-
-

Submission Status

- - +{% if paper.questions_answered.all or paper.questions_unanswered.all %} +
+
+

Submission Status

+
+ + + + + + + + {% for question in paper.questions.all %} + {% if question in paper.questions_answered.all %} - - - - - - {% for question in paper.questions.all %} - {% if question in paper.questions_answered.all %} - - - - {% else %} - - - - {% endif %} - - {% endfor %} - -
Question Status
Question Status
{{ question.summary }} Attempted
{{ question }} Not Attempted
-
-
- {% endif %} -

-
-
- {{message}} -
+ {{ question.summary }} + Attempted + {% else %} + + {{ question }} + Not Attempted + {% endif %} + + {% endfor %} + + +
-
-
- {% if module_id and not paper.course.is_trial %} - {% if first_unit %} - - {% else %} - - {% endif %} +{% endif %} +

+
+
+ {{message}} +
+
+
+
+{% if module_id and not paper.course.is_trial %} + {% if first_unit %} + {% else %} - Home + {% endif %} -
+{% else %} + Home +{% endif %} +
+

{% endblock content %} diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html index 56edcf3..2779db2 100644 --- a/yaksh/templates/yaksh/question.html +++ b/yaksh/templates/yaksh/question.html @@ -7,6 +7,8 @@ + +