Merge pull request #378 from prathamesh920/upload_users_csv

Upload users csv

11 files changed, 378 insertions, 3 deletions

diff --git a/yaksh/file_utils.py b/yaksh/file_utils.py
index f41c531..b178eeb 100644
--- a/yaksh/file_utils.py
+++ b/yaksh/file_utils.py
@@ -2,7 +2,7 @@ import shutil
 import os
 import zipfile
 import tempfile
-
+import csv
 
 def copy_files(file_paths):
     """ Copy Files to current directory, takes
@@ -50,3 +50,19 @@ def extract_files(zip_file, path=None):
         zip_file.close()
         return zfiles, extract_path
 
+
+def is_csv(document):
+    ''' Check if document is csv with ',' as the delimiter'''
+    try:
+        try:
+            content = document.read(1024).decode('utf-8')
+        except AttributeError:
+            document.seek(0)
+            content = document.read(1024)
+        sniffer = csv.Sniffer()
+        dialect = sniffer.sniff(content, delimiters=',')
+        document.seek(0)
+    except (csv.Error, UnicodeDecodeError):
+        return False, None
+    return True, dialect
+
diff --git a/yaksh/fixtures/sample_user_upload.csv b/yaksh/fixtures/sample_user_upload.csv
new file mode 100644
index 0000000..2edf710
--- /dev/null
+++ b/yaksh/fixtures/sample_user_upload.csv
@@ -0,0 +1,2 @@
+firstname,lastname,email,username,password,institute,department,roll_no,remove
+sample,user,sampleuser@xyz.com,sample_user,sample,sample,sample,sample123,False
diff --git a/yaksh/fixtures/users_add_update_reject.csv b/yaksh/fixtures/users_add_update_reject.csv
new file mode 100644
index 0000000..43881d8
--- /dev/null
+++ b/yaksh/fixtures/users_add_update_reject.csv
@@ -0,0 +1,4 @@
+firstname, lastname, email, institute,department,roll_no,remove,password,username
+test, test, test@g.com, TEST, TEST, TEST101, FALSE, TEST, test
+test2, test, test@g.com, TEST, TEST, TEST101, FALSE, TEST, test
+test2, test, test@g.com, TEST, TEST, TEST101, TRUE, TEST, test
diff --git a/yaksh/fixtures/users_correct.csv b/yaksh/fixtures/users_correct.csv
new file mode 100644
index 0000000..7b2a977
--- /dev/null
+++ b/yaksh/fixtures/users_correct.csv
@@ -0,0 +1,2 @@
+firstname, lastname, email
+abc, abc, abc@xyz.com
diff --git a/yaksh/fixtures/users_some_headers_missing.csv b/yaksh/fixtures/users_some_headers_missing.csv
new file mode 100644
index 0000000..ddcaa88
--- /dev/null
+++ b/yaksh/fixtures/users_some_headers_missing.csv
@@ -0,0 +1,2 @@
+noname, lastname, email
+abc, abc, abc@xyz.com
diff --git a/yaksh/fixtures/users_some_values_missing.csv b/yaksh/fixtures/users_some_values_missing.csv
new file mode 100644
index 0000000..d60ed8d
--- /dev/null
+++ b/yaksh/fixtures/users_some_values_missing.csv
@@ -0,0 +1,4 @@
+firstname, lastname, email
+abc, , abc@xyz.com
+dummy, dummy , dummy@xyz.com
+dummy, dummy , dummy@xyz.com
diff --git a/yaksh/fixtures/users_with_no_values.csv b/yaksh/fixtures/users_with_no_values.csv
new file mode 100644
index 0000000..db08ba2
--- /dev/null
+++ b/yaksh/fixtures/users_with_no_values.csv
@@ -0,0 +1 @@
+firstname, lastname, email
diff --git a/yaksh/templates/yaksh/course_detail.html b/yaksh/templates/yaksh/course_detail.html
index d8aeb2e..93a7048 100644
--- a/yaksh/templates/yaksh/course_detail.html
+++ b/yaksh/templates/yaksh/course_detail.html
@@ -45,6 +45,27 @@
     </div>
 </div>
 <div class="col-md-9 col-md-offset-2 main">
+    <form id="upload_users" action="{{ URL_ROOT }}/exam/manage/upload_users/{{course.id}}/"
+        method="POST" enctype="multipart/form-data">
+        {% csrf_token %}
+        <input type="file" name="csv_file" />
+        <button class="btn btn-primary" type=submit> Upload Users <span class="glyphicon glyphicon-open"/></button>
+    </form>
+    <div class="alert alert-info" role="alert">
+        <p>
+        - The uploaded csv should have headers exactly same as mentioned below:<br />
+        <b>firstname, lastname, email, username, password, institute, roll_no, department,
+           remove</b><br />
+        - Mandatory fields are <b> firstname, lastname and email. </b><br />
+        - Other fields are optional. <br />
+        - If username and password are not provided then
+        <b>Users created will have username and password same as their email</b>
+        </p>
+        <p>
+        <b> Click <a class="btn btn-success" href="{{ URL_ROOT }}/exam/manage/download_sample_csv/
+">here</a> to download a sample CSV, edit and upload it</b>
+        </p>
+    </div>
     <div class="row">
         {% if message %}
         <div class="alert alert-warning" role="alert">
@@ -53,6 +74,14 @@
         </center>
         </div>
         {% endif %}
+        {% if upload_details %}
+        <div class="alert alert-info" role="info">
+            {% for detail in upload_details %}
+            <strong> {{ detail }} </strong><br>
+            {% endfor %}
+        </div>
+        {% endif %}
+        <hr>
         {% if state == 'mail' %}
             <div id="enrolled-students">
             <center><b><u>Send Mails to Students</u></b></center><br>
@@ -218,6 +247,7 @@
         {% endif %}
     </div>
 </div>
+
 <!-- Dialog to display error message -->
 <div id="dialog" title="Alert">
   <p id="error_msg"></p>
diff --git a/yaksh/test_views.py b/yaksh/test_views.py
index 652f44c..2dddcef 100644
--- a/yaksh/test_views.py
+++ b/yaksh/test_views.py
@@ -1539,6 +1539,8 @@ class TestCourses(TestCase):
         self.user1_course.delete()
         self.user2_course.delete()
 
+
+
     def test_courses_denies_anonymous(self):
         """
         If not logged in redirect to login page
@@ -1790,6 +1792,163 @@ class TestCourseDetail(TestCase):
         self.student.delete()
         self.user1_course.delete()
 
+    def test_upload_users_with_correct_csv(self):
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH, "users_correct.csv")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                    kwargs={'course_id': self.user1_course.id}),
+                                    data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        uploaded_user = User.objects.filter(email="abc@xyz.com")
+        self.assertEqual(uploaded_user.count(), 1)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('upload_details', response.context)
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
+    def test_upload_users_add_update_reject(self):
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH,
+                                     "users_add_update_reject.csv")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                    kwargs={'course_id': self.user1_course.id}),
+                                    data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        uploaded_user = User.objects.filter(username="test")
+        user = uploaded_user[0]
+        self.assertEqual(uploaded_user.count(), 1)
+        self.assertEqual(user.first_name, "test2")
+        self.assertIn(user, self.user1_course.get_rejected())
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('upload_details', response.context)
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
+    def test_upload_users_with_wrong_csv(self):
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH, "demo_questions.zip")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+        message = "The file uploaded is not a CSV file."
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                            kwargs={'course_id': self.user1_course.id}),
+                                            data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn('upload_details', response.context)
+        self.assertIn('message', response.context)
+        self.assertEqual(response.context['message'], message)
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
+    def test_upload_users_csv_with_missing_headers(self):
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH, "users_some_headers_missing.csv")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+        message = "The CSV file does not contain the required headers"
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                            kwargs={'course_id': self.user1_course.id}),
+                                            data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn('upload_details', response.context)
+        self.assertIn('message', response.context)
+        self.assertEqual(response.context['message'], message)
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
+    def test_upload_users_csv_with_no_values(self):
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH, "users_with_no_values.csv")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                            kwargs={'course_id': self.user1_course.id}),
+                                            data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('upload_details', response.context)
+        self.assertNotIn('message', response.context)
+        self.assertIn("No rows in the CSV file", response.context['upload_details'])
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
+    def test_upload_users_csv_with_missing_values(self):
+        '''
+            This test takes csv with 3 row values.
+            1st row has a missing row.
+            2nd has a proper row.
+            3rd has a same row has 2nd
+
+            Only 2nd user will be added.
+
+            This test proves that:
+            - Row with missing values is ignored and continued with next row.
+            - Duplicate user is not created.
+        '''
+        # Given
+        self.client.login(
+            username=self.user1.username,
+            password=self.user1_plaintext_pass
+        )
+        csv_file_path = os.path.join(FIXTURES_DIR_PATH, "users_some_values_missing.csv")
+        csv_file = open(csv_file_path, 'rb')
+        upload_file = SimpleUploadedFile(csv_file_path, csv_file.read())
+
+        # When
+        response = self.client.post(reverse('yaksh:upload_users',
+                                    kwargs={'course_id': self.user1_course.id}),
+                                    data={'csv_file': upload_file})
+        csv_file.close()
+
+        # Then
+        uploaded_user = User.objects.filter(email="dummy@xyz.com")
+        self.assertEqual(uploaded_user.count(), 1)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('upload_details', response.context)
+        self.assertNotIn('message', response.context)
+        self.assertTemplateUsed(response, 'yaksh/course_detail.html')
+
     def test_course_detail_denies_anonymous(self):
         """
         If not logged in redirect to login page
diff --git a/yaksh/urls.py b/yaksh/urls.py
index 9288956..7f484b9 100644
--- a/yaksh/urls.py
+++ b/yaksh/urls.py
@@ -68,6 +68,7 @@ urlpatterns = [
             name="enroll_user"),
     url(r'manage/enroll/rejected/(?P<course_id>\d+)/(?P<user_id>\d+)/$',
         views.enroll, {'was_rejected': True}),
+    url(r'manage/upload_users/(?P<course_id>\d+)/$', views.upload_users, name="upload_users"),
     url(r'manage/send_mail/(?P<course_id>\d+)/$', views.send_mail, name="send_mail"),
     url(r'manage/reject/(?P<course_id>\d+)/(?P<user_id>\d+)/$', views.reject,
             name="reject_user"),
@@ -110,4 +111,6 @@ urlpatterns = [
         views.download_assignment_file, name="download_quiz_assignment"),
     url(r'^manage/courses/download_yaml_template/',
         views.download_yaml_template, name="download_yaml_template"),
+    url(r'^manage/download_sample_csv/',
+        views.download_sample_csv, name="download_sample_csv"),
 ]
diff --git a/yaksh/views.py b/yaksh/views.py
index df002a4..bc03ca2 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -37,7 +37,7 @@ from yaksh.models import (
     HookTestCase, IntegerTestCase, McqTestCase, Profile,
     QuestionPaper, QuestionSet, Quiz, Question, StandardTestCase,
     StdIOBasedTestCase, StringTestCase, TestCase, User,
-    get_model_class
+    get_model_class, FIXTURES_DIR_PATH
 )
 from yaksh.forms import (
     UserRegisterForm, UserLoginForm, QuizForm, QuestionForm,
@@ -45,7 +45,7 @@ from yaksh.forms import (
     UploadFileForm, get_object_form, FileForm, QuestionPaperForm
 )
 from .settings import URL_ROOT
-from .file_utils import extract_files
+from .file_utils import extract_files, is_csv
 from .send_emails import send_user_mail, generate_activation_key, send_bulk_mail
 from .decorators import email_verified, has_profile
 
@@ -1850,6 +1850,158 @@ def download_assignment_file(request, quiz_id, question_id=None, user_id=None):
 
 @login_required
 @email_verified
+def upload_users(request, course_id):
+    user = request.user
+    ci = RequestContext(request)
+    course = get_object_or_404(Course, pk=course_id)
+    context = {'course': course}
+
+    if not (course.is_teacher(user) or course.is_creator(user)):
+        msg = 'You do not have permissions to this course.'
+        return complete(request, reason=msg)
+    if request.method == 'POST':
+        if 'csv_file' not in request.FILES:
+            context['message'] = "Please upload a CSV file."
+            return my_render_to_response('yaksh/course_detail.html', context,
+                                         context_instance=ci)
+        csv_file = request.FILES['csv_file']
+        is_csv_file, dialect = is_csv(csv_file)
+        if not is_csv_file:
+            context['message'] = "The file uploaded is not a CSV file."
+            return my_render_to_response('yaksh/course_detail.html', context,
+                                         context_instance=ci)
+        required_fields = ['firstname', 'lastname', 'email']
+        try:
+            reader = csv.DictReader(csv_file.read().decode('utf-8').splitlines(),
+                                    dialect=dialect)
+        except TypeError:
+            context['message'] = "Bad CSV file"
+            return my_render_to_response('yaksh/course_detail.html', context,
+                                         context_instance=ci)
+        stripped_fieldnames = [field.strip().lower() for field in reader.fieldnames]
+        for field in required_fields:
+            if field not in stripped_fieldnames:
+                context['message'] = "The CSV file does not contain the required headers"
+                return my_render_to_response('yaksh/course_detail.html', context,
+                                             context_instance=ci)
+        reader.fieldnames = stripped_fieldnames
+        context['upload_details'] = _read_user_csv(reader, course)
+    return my_render_to_response('yaksh/course_detail.html', context,
+                                 context_instance=ci)
+
+
+def _read_user_csv(reader, course):
+    fields = reader.fieldnames
+    upload_details = ["Upload Summary:"]
+    counter = 0
+    for row in reader:
+        counter += 1
+        (username, email, first_name, last_name, password, roll_no, institute,
+         department, remove) = _get_csv_values(row, fields)
+        if not email or not first_name or not last_name:
+            upload_details.append("{0} -- Missing Values".format(counter))
+            continue
+        users = User.objects.filter(username=username)
+        if users.exists():
+            user = users[0]
+            if remove.strip().lower() == 'true':
+                if _remove_from_course(user, course):
+                    upload_details.append("{0} -- {1} -- User rejected".format(
+                                          counter, user.username))
+                    continue
+            else:
+                if _add_to_course(user, course):
+                    upload_details.append("{0} -- {1} -- User rejected".format(
+                                          counter, user.username))
+            if user not in course.get_enrolled():
+                upload_details.append("{0} -- {1} not added to course".format(
+                    counter, user))
+                continue
+        user_defaults = {'email': email, 'first_name': first_name,
+                         'last_name': last_name}
+        user, created = _create_or_update_user(username, password, user_defaults)
+        profile_defaults = {'institute': institute, 'roll_number': roll_no,
+                            'department': department, 'is_email_verified': True}
+        _create_or_update_profile(user, profile_defaults)
+        if created:
+            state = "Added"
+            course.students.add(user)
+        else:
+            state = "Updated"
+        upload_details.append("{0} -- {1} -- User {2} Successfully".format(
+                              counter, user.username, state))
+    if counter == 0:
+        upload_details.append("No rows in the CSV file")
+    return upload_details
+
+
+def _get_csv_values(row, fields):
+    roll_no, institute, department = "", "", ""
+    remove = "false"
+    email, first_name, last_name = map(str.strip, [row['email'],
+                                       row['firstname'],
+                                       row['lastname']])
+    password = email
+    username = email
+    if 'password' in fields and row['password']:
+        password = row['password'].strip()
+    if 'roll_no' in fields:
+        roll_no = row['roll_no'].strip()
+    if 'institute' in fields:
+        institute = row['institute'].strip()
+    if 'department' in fields:
+        department = row['department'].strip()
+    if 'remove' in fields:
+        remove = row['remove'].strip()
+    if 'username' in fields and row['username']:
+        username = row['username'].strip()
+    if 'remove' in fields:
+        remove = row['remove']
+    return (username, email, first_name, last_name, password, roll_no, institute,
+            department, remove)
+
+
+def _remove_from_course(user, course):
+    if user in course.get_enrolled():
+        course.reject(True, user)
+        return True
+
+
+def _add_to_course(user, course):
+    if user in course.get_rejected():
+        course.enroll(True, user)
+        return True
+
+
+def _create_or_update_user(username, password, defaults):
+    user, created = User.objects.update_or_create(username=username,
+                                                  defaults=defaults)
+    user.set_password(password)
+    user.save()
+    return user, created
+
+
+def _create_or_update_profile(user, defaults):
+    Profile.objects.update_or_create(user=user, defaults=defaults)
+
+
+@login_required
+@email_verified
+def download_sample_csv(request):
+    user = request.user
+    if not is_moderator(user):
+        raise Http404('You are not allowed to view this page!')
+    csv_file_path = os.path.join(FIXTURES_DIR_PATH,
+                                 "sample_user_upload.csv")
+    with open(csv_file_path, 'rb') as csv_file:
+        response = HttpResponse(csv_file.read(), content_type='text/csv')
+        response['Content-Disposition'] = 'attachment;\
+                                          filename="sample_user_upload"'
+        return response
+
+
+@login_required
+@email_verified
 def duplicate_course(request, course_id):
     user = request.user
     course = get_object_or_404(Course, pk=course_id)