Merge pull request #312 from maheshgudi/yaml_questions

Yaml serialized questions

11 files changed, 220 insertions, 82 deletions

diff --git a/requirements/requirements-common.txt b/requirements/requirements-common.txt
index 53a44a4..100d693 100644
--- a/requirements/requirements-common.txt
+++ b/requirements/requirements-common.txt
@@ -6,3 +6,4 @@ tornado
 selenium==2.53.6
 coverage
 psutil
+ruamel.yaml==0.15.23
diff --git a/yaksh/fixtures/demo_questions.zip b/yaksh/fixtures/demo_questions.zip
index c68e7ef..4e86485 100644
--- a/yaksh/fixtures/demo_questions.zip
+++ b/yaksh/fixtures/demo_questions.zip
diff --git a/yaksh/live_server_tests/selenium_test.py b/yaksh/live_server_tests/selenium_test.py
index 277f08e..bc400fd 100644
--- a/yaksh/live_server_tests/selenium_test.py
+++ b/yaksh/live_server_tests/selenium_test.py
@@ -23,6 +23,7 @@ class SeleniumTest():
             self.driver.get(self.url)
             self.login(username, password)
             self.open_quiz()
+            self.quit_quiz()
             self.close_quiz()
             self.logout()
             self.driver.close()
@@ -111,9 +112,20 @@ class SeleniumTest():
         )
         start_exam_elem.click()
 
-        self.test_c_question(question_label=2)
-        self.test_python_question(question_label=3)
-        self.test_bash_question(question_label=1)
+        self.test_c_question(question_label=7)
+        self.test_python_question(question_label=5)
+        self.test_bash_question(question_label=4)
+
+    def quit_quiz(self):
+        quit_link_elem = WebDriverWait(self.driver, 5).until(
+            EC.presence_of_element_located((By.NAME, "quit"))
+        )
+        quit_link_elem.click()
+
+        quit_link_elem = WebDriverWait(self.driver, 5).until(
+            EC.presence_of_element_located((By.NAME, "yes"))
+        )
+        quit_link_elem.click()
 
     def close_quiz(self):
         quit_link_elem = WebDriverWait(self.driver, 5).until(
diff --git a/yaksh/models.py b/yaksh/models.py
index 30ecde0..979740d 100644
--- a/yaksh/models.py
+++ b/yaksh/models.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 from datetime import datetime, timedelta
 import json
+import ruamel.yaml
+from ruamel.yaml.scalarstring import PreservedScalarString
+from ruamel.yaml.comments import CommentedMap
 from random import sample
 from collections import Counter
 from django.db import models
@@ -23,9 +26,11 @@ import shutil
 import zipfile
 import tempfile
 from textwrap import dedent
+from ast import literal_eval
 from .file_utils import extract_files, delete_files
 from yaksh.xmlrpc_clients import code_server
 from django.conf import settings
+from django.forms.models import model_to_dict
 
 
 languages = (
@@ -97,6 +102,18 @@ def get_upload_dir(instance, filename):
         'question_%s' % (instance.question.id), filename
     ))
 
+def dict_to_yaml(dictionary):
+    for k,v in dictionary.items():
+        if isinstance(v, list):
+            for  nested_v in v:
+                if isinstance(nested_v, dict):
+                    dict_to_yaml(nested_v)
+        elif v and isinstance(v,str):
+            dictionary[k] = PreservedScalarString(v)
+    return ruamel.yaml.round_trip_dump(dictionary, explicit_start=True,
+                                       default_flow_style=False,
+                                       allow_unicode=True,
+                                       )
 
 ###############################################################################
 class CourseManager(models.Manager):
@@ -375,52 +392,57 @@ class Question(models.Model):
         return json.dumps(question_data)
 
     def dump_questions(self, question_ids, user):
-        questions = Question.objects.filter(
-            id__in=question_ids, user_id=user.id, active=True
-        )
+        questions = Question.objects.filter(id__in=question_ids,
+                                            user_id=user.id, active=True
+                                            )
         questions_dict = []
         zip_file_name = string_io()
         zip_file = zipfile.ZipFile(zip_file_name, "a")
         for question in questions:
             test_case = question.get_test_cases()
             file_names = question._add_and_get_files(zip_file)
-            q_dict = {
-                'summary': question.summary,
-                'description': question.description,
-                'points': question.points, 'language': question.language,
-                'type': question.type, 'active': question.active,
-                'snippet': question.snippet,
-                'testcase': [case.get_field_value() for case in test_case],
-                'files': file_names
-            }
+            q_dict = model_to_dict(question, exclude=['id', 'user'])
+            testcases = []
+            for case in test_case:
+                testcases.append(case.get_field_value())
+            q_dict['testcase'] = testcases
+            q_dict['files'] = file_names
+            q_dict['tags'] = [tags.tag.name for tags in q_dict['tags']]
             questions_dict.append(q_dict)
-        question._add_json_to_zip(zip_file, questions_dict)
+        question._add_yaml_to_zip(zip_file, questions_dict)
         return zip_file_name
 
     def load_questions(self, questions_list, user, file_path=None,
                        files_list=None):
         try:
-            questions = json.loads(questions_list)
-        except ValueError as exc_msg:
-            msg = "Error Parsing Json: {0}".format(exc_msg)
-            return msg
-        for question in questions:
-            question['user'] = user
-            file_names = question.pop('files')
-            test_cases = question.pop('testcase')
-            que, result = Question.objects.get_or_create(**question)
-            if file_names:
-                que._add_files_to_db(file_names, file_path)
-            for test_case in test_cases:
-                test_case_type = test_case.pop('test_case_type')
-                model_class = get_model_class(test_case_type)
-                new_test_case, obj_create_status = \
-                    model_class.objects.get_or_create(
-                        question=que, **test_case
-                    )
-                new_test_case.type = test_case_type
-                new_test_case.save()
-        return "Questions Uploaded Successfully"
+            questions = ruamel.yaml.safe_load_all(questions_list)
+            msg = "Questions Uploaded Successfully"
+            for question in questions:
+                question['user'] = user
+                file_names = question.pop('files')
+                test_cases = question.pop('testcase')
+                tags = question.pop('tags')
+                que, result = Question.objects.get_or_create(**question)
+                if file_names:
+                    que._add_files_to_db(file_names, file_path)
+                if tags:
+                    que.tags.add(*tags)
+                for test_case in test_cases:
+                    try:
+                        test_case_type = test_case.pop('test_case_type')
+                        model_class = get_model_class(test_case_type)
+                        new_test_case, obj_create_status = \
+                            model_class.objects.get_or_create(
+                                question=que, **test_case
+                            )
+                        new_test_case.type = test_case_type
+                        new_test_case.save()
+
+                    except:
+                        msg = "File not correct."
+        except Exception as exc_msg:
+            msg = "Error Parsing Yaml: {0}".format(exc_msg)
+        return msg
 
     def get_test_cases(self, **kwargs):
         tc_list = []
@@ -476,25 +498,30 @@ class Question(models.Model):
                 file_upload.extract = extract
                 file_upload.file.save(file_name, django_file, save=True)
 
-    def _add_json_to_zip(self, zip_file, q_dict):
-        json_data = json.dumps(q_dict, indent=2)
+    def _add_yaml_to_zip(self, zip_file, q_dict,path_to_file=None):
+        
         tmp_file_path = tempfile.mkdtemp()
-        json_path = os.path.join(tmp_file_path, "questions_dump.json")
-        with open(json_path, "w") as json_file:
-            json_file.write(json_data)
-        zip_file.write(json_path, os.path.basename(json_path))
+        yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml")
+        for elem in q_dict:
+            sorted_dict = CommentedMap(sorted(elem.items(), key=lambda x:x[0]))
+            yaml_block = dict_to_yaml(sorted_dict)
+            with open(yaml_path, "a") as yaml_file:
+                yaml_file.write(yaml_block)
+        zip_file.write(yaml_path, os.path.basename(yaml_path))
         zip_file.close()
         shutil.rmtree(tmp_file_path)
 
-    def read_json(self, file_path, user, files=None):
-        json_file = os.path.join(file_path, "questions_dump.json")
+    def read_yaml(self, file_path, user, files=None):
+        yaml_file = os.path.join(file_path, "questions_dump.yaml")
         msg = ""
-        if os.path.exists(json_file):
-            with open(json_file, 'r') as q_file:
+        if os.path.exists(yaml_file):
+            with open(yaml_file, 'r') as q_file:
                 questions_list = q_file.read()
-                msg = self.load_questions(questions_list, user, file_path, files)
+                msg = self.load_questions(questions_list, user, 
+                                          file_path, files
+                                          )
         else:
-            msg = "Please upload zip file with questions_dump.json in it."
+            msg = "Please upload zip file with questions_dump.yaml in it."
 
         if files:
             delete_files(files, file_path)
@@ -505,7 +532,7 @@ class Question(models.Model):
             settings.FIXTURE_DIRS, 'demo_questions.zip'
         )
         files, extract_path = extract_files(zip_file_path)
-        self.read_json(extract_path, user, files)
+        self.read_yaml(extract_path, user, files)
 
     def __str__(self):
         return self.summary
@@ -880,8 +907,13 @@ class QuestionPaper(models.Model):
                                                       total_marks=6.0,
                                                       shuffle_questions=True
                                                       )
+        summaries = ['Roots of quadratic equation', 'Print Output',
+                     'Adding decimals', 'For Loop over String',
+                     'Hello World in File', 'Extract columns from files',
+                     'Check Palindrome', 'Add 3 numbers', 'Reverse a string'
+                     ]
         questions = Question.objects.filter(active=True,
-                                            summary="Yaksh Demo Question",
+                                            summary__in=summaries,
                                             user=user)
         q_order = [str(que.id) for que in questions]
         question_paper.fixed_question_order = ",".join(q_order)
diff --git a/yaksh/static/yaksh/js/show_question.js b/yaksh/static/yaksh/js/show_question.js
index e3ed1cc..e7cd817 100644
--- a/yaksh/static/yaksh/js/show_question.js
+++ b/yaksh/static/yaksh/js/show_question.js
@@ -37,3 +37,7 @@ function confirm_edit(frm)
     else
         return true;
 }
+$(document).ready(function()
+    { 
+        $("#questions-table").tablesorter({sortList: [[0,0], [4,0]]});
+    });
\ No newline at end of file
diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html
index ee33523..74dd8c3 100644
--- a/yaksh/templates/yaksh/question.html
+++ b/yaksh/templates/yaksh/question.html
@@ -238,8 +238,8 @@ function call_skip(url)
                      {% endif %}
         <div class="from-group">
 
-         {% if question.type == "mcq" or "mcc" or "integer" or "float" or "string" %}
-         <br><button class="btn btn-primary" type="submit" name="check" id="check">Submit Answer</button>&nbsp;&nbsp;<br/>
+         {% if question.type == "mcq" or  question.type == "mcc" or question.type == "integer" or question.type == "float" or question.type ==  "string" %}
+         <br><button class="btn btn-primary" type="submit" name="check" id="check">Submit Answer</button>&nbsp;&nbsp;
          {% elif question.type == "upload" %}
          <br><button class="btn btn-primary" type="submit" name="check" id="check" onClick="return validate();">Upload</button>&nbsp;&nbsp;
 
diff --git a/yaksh/templates/yaksh/showquestions.html b/yaksh/templates/yaksh/showquestions.html
index a136ddf..78be301 100644
--- a/yaksh/templates/yaksh/showquestions.html
+++ b/yaksh/templates/yaksh/showquestions.html
@@ -2,31 +2,62 @@
 
 {% block title %} Questions {% endblock %}
 
-{% block pagetitle %} List of Questions {% endblock pagetitle %}
+{% block pagetitle %} Questions {% endblock pagetitle %}
 
 {% block script %}
 <script src="{{ URL_ROOT }}/static/yaksh/js/show_question.js"></script>
 <script src="{{ URL_ROOT }}/static/yaksh/js/question_filter.js"></script>
+<script src="{{ URL_ROOT }}/static/yaksh/js/jquery.tablesorter.min.js"></script>
 {% endblock %}
 
 {% block content %}
-
-<h4>Upload ZIP file for adding questions</h4>
+<div class="row">
+    <div class="col-sm-3 col-md-2 sidebar">
+      <ul class="nav nav-sidebar nav-stacked">
+      <li class="active"><a href="#show" data-toggle="pill" > Show all Questions</a></li>
+      <li><a href="#updown" data-toggle="pill" > Upload and Download Questions</a></li>
+      </ul>
+    </div>
+<div class="tab-content">
+<!-- Upload Questions -->
+<div id="updown" class="tab-pane fade">
+<a class="btn btn-primary" href="{{URL_ROOT}}/exam/manage/courses/download_yaml_template/"> Download Template</a>
+<br/>
+<h4> Or </h4>
 <form action="" method="post"  enctype="multipart/form-data">
 {% csrf_token %}
-{{ upload_form.as_p }}
-<button class="btn btn-primary" type="submit" name="upload" value="upload">
-Upload File <span class="glyphicon glyphicon-open"></span></button>
+    {{ upload_form.as_p }}
+<br/>
+<h4> And </h4>
+<button class="btn btn-success" type="submit" name="upload" value="upload">
+Upload File <span class="glyphicon glyphicon-open"/></button>
 </form>
+</div>
+<!-- End of upload questions -->
+
+<!-- Show questions -->
+<div id="show" class= "tab-pane fade in active">
+<form name=frm action="" method="post">
+{% csrf_token %}
 {% if message %}
-<h4>{{ message }}</h4>
+{%if message == "Questions Uploaded Successfully"%}
+<div class="alert alert-success alert-dismissable">
+<a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+  {{ message }}
+</div>
+{%else %}
+<div class="alert alert-danger alert-dismissable"> 
+  <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+  {{ message }}
+</div>
+{% endif %}
 {% endif %}
 {% if msg %}
-<h4>{{ msg }}</h4>
+<div class="alert alert-danger alert-dismissable"> 
+  <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+  {{ msg }}
+</div>
 {% endif %}
-<br><br>
-<form name=frm action="" method="post">
-{% csrf_token %}
 <div class="row" id="selectors">
     <h5 style="padding-left: 20px;">Filters</h5>
     <div class="col-md-3">
@@ -46,17 +77,46 @@ Upload File <span class="glyphicon glyphicon-open"></span></button>
 <div id="filtered-questions">
 {% if questions %}
 <h5><input id="checkall" type="checkbox"> Select All </h5>
+
+<table id="questions-table" class="tablesorter table table table-striped">
+    <thead>
+    <tr>
+    <th> Select </th>
+    <th> Summary </th>
+    <th> Language </th>
+    <th> Type </th>  
+    <th> Marks </th>
+    </tr>
+    </thead>
+    <tbody>
+    
 {% for i in questions %}
-<input type="checkbox" name="question" value="{{ i.id }}">&nbsp;&nbsp;<a href="{{URL_ROOT}}/exam/manage/addquestion/{{ i.id }}">{{ i }}</a><br>
+<tr>
+<td>
+<input type="checkbox" name="question" value="{{ i.id }}">
+</td>
+<td><a href="{{URL_ROOT}}/exam/manage/addquestion/{{ i.id }}">{{i.summary|capfirst}}</a></td>
+<td>{{i.language|capfirst}}</td>
+<td>{{i.type|capfirst}}</td>
+<td>{{i.points}}</td>
+</tr>
 {% endfor %}
+</tbody>
+</table>
 {% endif %}
 </div>
 <br>
+<center>
 <button class="btn btn-primary" type="button" onclick='location.replace("{{URL_ROOT}}/exam/manage/addquestion/");'>Add Question <span class="glyphicon glyphicon-plus"></span></button>&nbsp;&nbsp;
 {% if questions %}
 <button class="btn btn-primary" type="submit" name='download' value='download'>Download Selected <span class="glyphicon glyphicon-save"></span></button>&nbsp;&nbsp;
 <button class="btn btn-primary" type="submit" name="test" value="test">Test Selected</button>&nbsp;&nbsp;
 {% endif %}
 <button class="btn btn-danger" type="submit" onClick="return confirm_delete(frm);" name='delete' value='delete'>Delete Selected <span class="glyphicon glyphicon-minus"></span></button>
+</center>
 </form>
-{% endblock %}
+</div>
+</div>
+</div>
+<!-- End of Show questions -->
+{% endblock %}
\ No newline at end of file
diff --git a/yaksh/test_models.py b/yaksh/test_models.py
index c86d9a3..a940c0f 100644
--- a/yaksh/test_models.py
+++ b/yaksh/test_models.py
@@ -1,8 +1,9 @@
 import unittest
 from yaksh.models import User, Profile, Question, Quiz, QuestionPaper,\
     QuestionSet, AnswerPaper, Answer, Course, StandardTestCase,\
-    StdIOBasedTestCase, FileUpload, McqTestCase, AssignmentUpload
+    StdIOBasedTestCase, FileUpload, McqTestCase, AssignmentUpload 
 import json
+import ruamel.yaml as yaml
 from datetime import datetime, timedelta
 from django.utils import timezone
 import pytz
@@ -111,7 +112,7 @@ class QuestionTestCases(unittest.TestCase):
             user=self.user1
         )
 
-        self.question2 = Question.objects.create(summary='Demo Json',
+        self.question2 = Question.objects.create(summary='Yaml Json',
             language='python',
             type='code',
             active=True,
@@ -159,8 +160,10 @@ class QuestionTestCases(unittest.TestCase):
                            "language": "Python", "type": "Code",
                            "testcase": self.test_case_upload_data,
                            "files": [[file1, 0]],
-                           "summary": "Json Demo"}]
-        self.json_questions_data = json.dumps(questions_data)
+                           "summary": "Yaml Demo",
+                           "tags": ['yaml_demo']
+                           }]
+        self.yaml_questions_data = yaml.safe_dump_all(questions_data)
 
     def tearDown(self):
         shutil.rmtree(self.load_tmp_path)
@@ -191,7 +194,7 @@ class QuestionTestCases(unittest.TestCase):
             self.assertIn(tag, ['python', 'function'])
 
     def test_dump_questions(self):
-        """ Test dump questions into json """
+        """ Test dump questions into Yaml """
         question = Question()
         question_id = [self.question2.id]
         questions_zip = question.dump_questions(question_id, self.user2)
@@ -200,8 +203,8 @@ class QuestionTestCases(unittest.TestCase):
         tmp_path = tempfile.mkdtemp()
         zip_file.extractall(tmp_path)
         test_case = self.question2.get_test_cases()
-        with open("{0}/questions_dump.json".format(tmp_path), "r") as f:
-            questions = json.loads(f.read())
+        with open("{0}/questions_dump.yaml".format(tmp_path), "r") as f:
+            questions = yaml.safe_load_all(f.read())
             for q in questions:
                 self.assertEqual(self.question2.summary, q['summary'])
                 self.assertEqual(self.question2.language, q['language'])
@@ -216,13 +219,13 @@ class QuestionTestCases(unittest.TestCase):
             os.remove(os.path.join(tmp_path, file))
 
     def test_load_questions(self):
-        """ Test load questions into database from json """
+        """ Test load questions into database from Yaml """
         question = Question()
-        result = question.load_questions(self.json_questions_data, self.user1)
-        question_data = Question.objects.get(summary="Json Demo")
+        result = question.load_questions(self.yaml_questions_data, self.user1)
+        question_data = Question.objects.get(summary="Yaml Demo")
         file = FileUpload.objects.get(question=question_data)
         test_case = question_data.get_test_cases()
-        self.assertEqual(question_data.summary, 'Json Demo')
+        self.assertEqual(question_data.summary, 'Yaml Demo')
         self.assertEqual(question_data.language, 'Python')
         self.assertEqual(question_data.type, 'Code')
         self.assertEqual(question_data.description, 'factorial of a no')
diff --git a/yaksh/test_views.py b/yaksh/test_views.py
index 8025ecf..064c39d 100644
--- a/yaksh/test_views.py
+++ b/yaksh/test_views.py
@@ -3141,7 +3141,7 @@ class TestShowQuestions(TestCase):
         zip_file = string_io(response.content)
         zipped_file = zipfile.ZipFile(zip_file, 'r')
         self.assertIsNone(zipped_file.testzip())
-        self.assertIn('questions_dump.json', zipped_file.namelist())
+        self.assertIn('questions_dump.yaml', zipped_file.namelist())
         zip_file.close()
         zipped_file.close()
 
@@ -3170,12 +3170,18 @@ class TestShowQuestions(TestCase):
                                 data={'file': questions_file,
                                       'upload': 'upload'}
                                 )
+        summaries = ['Roots of quadratic equation', 'Print Output',
+                     'Adding decimals', 'For Loop over String',
+                     'Hello World in File', 'Extract columns from files',
+                     'Check Palindrome', 'Add 3 numbers', 'Reverse a string'
+                     ]
+
         uploaded_ques = Question.objects.filter(active=True,
-                                            summary="Yaksh Demo Question",
+                                            summary__in=summaries,
                                             user=self.user).count()
         self.assertEqual(response.status_code, 200)
         self.assertTemplateUsed(response, 'yaksh/showquestions.html')
-        self.assertEqual(uploaded_ques, 3)
+        self.assertEqual(uploaded_ques, 9)
         f.close()
         dummy_file = SimpleUploadedFile("test.txt", b"test")
         response = self.client.post(reverse('yaksh:show_questions'),
diff --git a/yaksh/urls.py b/yaksh/urls.py
index 5058340..4aa3276 100644
--- a/yaksh/urls.py
+++ b/yaksh/urls.py
@@ -106,5 +106,7 @@ urlpatterns = [
     url(r'^manage/download/user_assignment/(?P<question_id>\d+)/(?P<user_id>\d+)/(?P<quiz_id>\d+)/$',
         views.download_assignment_file, name="download_user_assignment"),
     url(r'^manage/download/quiz_assignments/(?P<quiz_id>\d+)/$',
-        views.download_assignment_file, name="download_quiz_assignment")
+        views.download_assignment_file, name="download_quiz_assignment"),
+    url(r'^manage/courses/download_yaml_template/',
+        views.download_yaml_template, name="download_yaml_template"),
 ]
diff --git a/yaksh/views.py b/yaksh/views.py
index 823e506..3c7df4d 100644
--- a/yaksh/views.py
+++ b/yaksh/views.py
@@ -1050,7 +1050,7 @@ def show_all_questions(request):
                 if file_name[-1] == "zip":
                     ques = Question()
                     files, extract_path = extract_files(questions_file)
-                    context['message'] = ques.read_json(extract_path, user,
+                    context['message'] = ques.read_yaml(extract_path, user,
                                                         files)
                 else:
                     message = "Please Upload a ZIP file"
@@ -1615,3 +1615,21 @@ def duplicate_course(request, course_id):
             'instructor/administrator.'
         return complete(request, msg, attempt_num=None, questionpaper_id=None)
     return my_redirect('/exam/manage/courses/')
+
+@login_required
+@email_verified
+def download_yaml_template(request):
+    user = request.user
+    if not is_moderator(user):
+        raise Http404('You are not allowed to view this page!')
+    template_path = os.path.join(os.path.dirname(__file__), "fixtures",
+                                 "demo_questions.zip"
+                                 )
+    yaml_file = zipfile.ZipFile(template_path, 'r')
+    template_yaml = yaml_file.open('questions_dump.yaml', 'r')
+    response = HttpResponse(template_yaml, content_type='text/yaml')
+    response['Content-Disposition'] = 'attachment;\
+                                      filename="questions_dump.yaml"'
+    
+    return response
+