From 66e8f39bb390b3ec2547d82512349fabf67e3d7c Mon Sep 17 00:00:00 2001 From: mahesh Date: Mon, 31 Jul 2017 17:22:54 +0530 Subject: Adds yaml serialization to download and upload questions --- yaksh/models.py | 78 +++++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 41 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 2fa34e4..f6504d7 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from datetime import datetime, timedelta import json +import yaml from random import sample from collections import Counter from django.db import models @@ -26,6 +27,7 @@ from textwrap import dedent from .file_utils import extract_files, delete_files from yaksh.xmlrpc_clients import code_server from django.conf import settings +from django.forms.models import model_to_dict languages = ( @@ -386,43 +388,38 @@ class Question(models.Model): for question in questions: test_case = question.get_test_cases() file_names = question._add_and_get_files(zip_file) - q_dict = { - 'summary': question.summary, - 'description': question.description, - 'points': question.points, 'language': question.language, - 'type': question.type, 'active': question.active, - 'snippet': question.snippet, - 'testcase': [case.get_field_value() for case in test_case], - 'files': file_names - } + q_dict = model_to_dict(question, exclude=['id', 'user','tags']) + q_dict['testcase']= [case.get_field_value() for case in test_case] + q_dict['files'] = file_names + questions_dict.append(q_dict) - question._add_json_to_zip(zip_file, questions_dict) + question._add_yaml_to_zip(zip_file, questions_dict) return zip_file_name def load_questions(self, questions_list, user, file_path=None, files_list=None): try: - questions = json.loads(questions_list) - except ValueError as exc_msg: - msg = "Error Parsing Json: {0}".format(exc_msg) - return msg - for question in questions: - question['user'] = user - file_names = question.pop('files') - test_cases = question.pop('testcase') - que, result = Question.objects.get_or_create(**question) - if file_names: - que._add_files_to_db(file_names, file_path) - for test_case in test_cases: - test_case_type = test_case.pop('test_case_type') - model_class = get_model_class(test_case_type) - new_test_case, obj_create_status = \ - model_class.objects.get_or_create( - question=que, **test_case - ) + questions = yaml.safe_load_all(questions_list) + for question in questions: + question['user'] = user + file_names = question.pop('files') + test_cases = question.pop('testcase') + que, result = Question.objects.get_or_create(**question) + if file_names: + que._add_files_to_db(file_names, file_path) + for test_case in test_cases: + test_case_type = test_case.pop('test_case_type') + model_class = get_model_class(test_case_type) + new_test_case, obj_create_status = \ + model_class.objects.get_or_create( + question=que, **test_case + ) new_test_case.type = test_case_type new_test_case.save() - return "Questions Uploaded Successfully" + msg = "Questions Uploaded Successfully" + except yaml.scanner.ScannerError as exc_msg: + msg = "Error Parsing Yaml: {0}".format(exc_msg) + return msg def get_test_cases(self, **kwargs): tc_list = [] @@ -478,25 +475,24 @@ class Question(models.Model): file_upload.extract = extract file_upload.file.save(file_name, django_file, save=True) - def _add_json_to_zip(self, zip_file, q_dict): - json_data = json.dumps(q_dict, indent=2) + def _add_yaml_to_zip(self, zip_file, q_dict): tmp_file_path = tempfile.mkdtemp() - json_path = os.path.join(tmp_file_path, "questions_dump.json") - with open(json_path, "w") as json_file: - json_file.write(json_data) - zip_file.write(json_path, os.path.basename(json_path)) + yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") + with open(yaml_path, "w") as yaml_file: + yaml.safe_dump_all(q_dict, yaml_file, default_flow_style=False) + zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) - def read_json(self, file_path, user, files=None): - json_file = os.path.join(file_path, "questions_dump.json") + def read_yaml(self, file_path, user, files=None): + yaml_file = os.path.join(file_path, "questions_dump.yaml") msg = "" - if os.path.exists(json_file): - with open(json_file, 'r') as q_file: + if os.path.exists(yaml_file): + with open(yaml_file, 'r') as q_file: questions_list = q_file.read() msg = self.load_questions(questions_list, user, file_path, files) else: - msg = "Please upload zip file with questions_dump.json in it." + msg = "Please upload zip file with questions_dump.yaml in it." if files: delete_files(files, file_path) @@ -507,7 +503,7 @@ class Question(models.Model): settings.FIXTURE_DIRS, 'demo_questions.zip' ) files, extract_path = extract_files(zip_file_path) - self.read_json(extract_path, user, files) + self.read_yaml(extract_path, user, files) def __str__(self): return self.summary -- cgit From a116370a56ccfb43cbd97e7588e0d0a6a19453d8 Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 11 Aug 2017 15:31:18 +0530 Subject: Literal scalar values for yaml dump --- yaksh/models.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index fde203c..479a0ec 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -24,6 +24,7 @@ import shutil import zipfile import tempfile from textwrap import dedent +from ast import literal_eval from .file_utils import extract_files, delete_files from yaksh.xmlrpc_clients import code_server from django.conf import settings @@ -388,15 +389,20 @@ class Question(models.Model): questions_dict = [] zip_file_name = string_io() zip_file = zipfile.ZipFile(zip_file_name, "a") + all_keys = [] for question in questions: test_case = question.get_test_cases() - file_names = question._add_and_get_files(zip_file) + file_names = str(question._add_and_get_files(zip_file)) q_dict = model_to_dict(question, exclude=['id', 'user','tags']) - q_dict['testcase']= [case.get_field_value() for case in test_case] + testcases = [] + for case in test_case: + all_keys.extend(list(case.get_field_value().keys())) + testcases.append(case.get_field_value()) + q_dict['testcase'] = testcases q_dict['files'] = file_names - + all_keys.extend(list(q_dict.keys())) questions_dict.append(q_dict) - question._add_yaml_to_zip(zip_file, questions_dict) + question._add_yaml_to_zip(zip_file, questions_dict, all_keys) return zip_file_name def load_questions(self, questions_list, user, file_path=None, @@ -405,7 +411,7 @@ class Question(models.Model): questions = yaml.safe_load_all(questions_list) for question in questions: question['user'] = user - file_names = question.pop('files') + file_names = literal_eval(question.pop('files')) test_cases = question.pop('testcase') que, result = Question.objects.get_or_create(**question) if file_names: @@ -478,11 +484,22 @@ class Question(models.Model): file_upload.extract = extract file_upload.file.save(file_name, django_file, save=True) - def _add_yaml_to_zip(self, zip_file, q_dict): + def _add_yaml_to_zip(self, zip_file, q_dict, all_keys): tmp_file_path = tempfile.mkdtemp() yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") + + def literal_representer(dumper, data): + data = data.replace("\r\n", "\n") + if not data in all_keys: + return dumper.represent_scalar('tag:yaml.org,2002:str', + data, style='|') + else: + return dumper.represent_scalar('tag:yaml.org,2002:str', data) + + yaml.add_representer(str,literal_representer) with open(yaml_path, "w") as yaml_file: - yaml.safe_dump_all(q_dict, yaml_file, default_flow_style=False) + yaml.dump_all(q_dict, yaml_file, default_flow_style=False, + allow_unicode=True) zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) -- cgit From 67f3119a2e27673694db907d21f501bc8247fce1 Mon Sep 17 00:00:00 2001 From: mahesh Date: Wed, 16 Aug 2017 18:28:26 +0530 Subject: Adds ruamel.yaml instead of pyyaml - Preserves escape characters - Adds ruamel.yaml in requirements --- yaksh/models.py | 59 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 28 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 479a0ec..7169d37 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals from datetime import datetime, timedelta import json -import yaml +import ruamel.yaml +from ruamel.yaml.scalarstring import PreservedScalarString from random import sample from collections import Counter from django.db import models @@ -383,39 +384,37 @@ class Question(models.Model): return json.dumps(question_data) def dump_questions(self, question_ids, user): - questions = Question.objects.filter( - id__in=question_ids, user_id=user.id, active=True - ) + questions = Question.objects.filter(id__in=question_ids, + user_id=user.id, active=True + ) questions_dict = [] zip_file_name = string_io() zip_file = zipfile.ZipFile(zip_file_name, "a") - all_keys = [] for question in questions: test_case = question.get_test_cases() - file_names = str(question._add_and_get_files(zip_file)) + file_names = question._add_and_get_files(zip_file) q_dict = model_to_dict(question, exclude=['id', 'user','tags']) testcases = [] for case in test_case: - all_keys.extend(list(case.get_field_value().keys())) testcases.append(case.get_field_value()) q_dict['testcase'] = testcases q_dict['files'] = file_names - all_keys.extend(list(q_dict.keys())) questions_dict.append(q_dict) - question._add_yaml_to_zip(zip_file, questions_dict, all_keys) + question._add_yaml_to_zip(zip_file, questions_dict) return zip_file_name def load_questions(self, questions_list, user, file_path=None, files_list=None): try: - questions = yaml.safe_load_all(questions_list) + questions = ruamel.yaml.safe_load_all(questions_list) for question in questions: question['user'] = user - file_names = literal_eval(question.pop('files')) + file_names = question.pop('files') test_cases = question.pop('testcase') que, result = Question.objects.get_or_create(**question) - if file_names: + if file_names!="[]": que._add_files_to_db(file_names, file_path) + for test_case in test_cases: test_case_type = test_case.pop('test_case_type') model_class = get_model_class(test_case_type) @@ -423,10 +422,10 @@ class Question(models.Model): model_class.objects.get_or_create( question=que, **test_case ) - new_test_case.type = test_case_type - new_test_case.save() + new_test_case.type = test_case_type + new_test_case.save() msg = "Questions Uploaded Successfully" - except yaml.scanner.ScannerError as exc_msg: + except ruamel.yaml.scanner.ScannerError as exc_msg: msg = "Error Parsing Yaml: {0}".format(exc_msg) return msg @@ -484,22 +483,26 @@ class Question(models.Model): file_upload.extract = extract file_upload.file.save(file_name, django_file, save=True) - def _add_yaml_to_zip(self, zip_file, q_dict, all_keys): + def _add_yaml_to_zip(self, zip_file, q_dict): tmp_file_path = tempfile.mkdtemp() yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") - def literal_representer(dumper, data): - data = data.replace("\r\n", "\n") - if not data in all_keys: - return dumper.represent_scalar('tag:yaml.org,2002:str', - data, style='|') - else: - return dumper.represent_scalar('tag:yaml.org,2002:str', data) - - yaml.add_representer(str,literal_representer) - with open(yaml_path, "w") as yaml_file: - yaml.dump_all(q_dict, yaml_file, default_flow_style=False, - allow_unicode=True) + def _dict_walk(question_dict): + for k,v in question_dict.items(): + if isinstance(v, list): + for nested_v in v: + if isinstance(nested_v, dict): + _dict_walk(nested_v) + elif v and isinstance(v,str): + question_dict[k] = PreservedScalarString(v) + for elem in q_dict: + _dict_walk(elem) + with open(yaml_path, "a") as yaml_file: + ruamel.yaml.round_trip_dump(elem, yaml_file, + default_flow_style=False, + explicit_start=True, + allow_unicode=True + ) zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) -- cgit From 98386cf63e6f574b38477564b96a5bcab25f2f8f Mon Sep 17 00:00:00 2001 From: mahesh Date: Thu, 17 Aug 2017 18:52:55 +0530 Subject: Adds yaml file containing all types of questions --- yaksh/models.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 7169d37..dfffb53 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -407,25 +407,28 @@ class Question(models.Model): files_list=None): try: questions = ruamel.yaml.safe_load_all(questions_list) + msg = "Questions Uploaded Successfully" for question in questions: question['user'] = user file_names = question.pop('files') test_cases = question.pop('testcase') que, result = Question.objects.get_or_create(**question) - if file_names!="[]": + if file_names: que._add_files_to_db(file_names, file_path) for test_case in test_cases: - test_case_type = test_case.pop('test_case_type') - model_class = get_model_class(test_case_type) - new_test_case, obj_create_status = \ - model_class.objects.get_or_create( - question=que, **test_case - ) - new_test_case.type = test_case_type - new_test_case.save() - msg = "Questions Uploaded Successfully" - except ruamel.yaml.scanner.ScannerError as exc_msg: + try: + test_case_type = test_case.pop('test_case_type') + model_class = get_model_class(test_case_type) + new_test_case, obj_create_status = \ + model_class.objects.get_or_create( + question=que, **test_case + ) + new_test_case.type = test_case_type + new_test_case.save() + except: + msg = "File not correct." + except Exception as exc_msg: msg = "Error Parsing Yaml: {0}".format(exc_msg) return msg -- cgit From 5d37ce12b0fbd01756221abe2a001eb35415bccc Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 18 Aug 2017 00:38:53 +0530 Subject: Adds a function dict_to_yaml in models --- yaksh/models.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index dfffb53..4113a62 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -106,6 +106,22 @@ def get_upload_dir(instance, filename): 'question_%s' % (instance.question.id), filename )) +def dict_to_yaml(dictionary, path_to_file=None): + for k,v in dictionary.items(): + if isinstance(v, list): + for nested_v in v: + if isinstance(nested_v, dict): + dict_to_yaml(nested_v) + elif v and isinstance(v,str): + dictionary[k] = PreservedScalarString(v) + if path_to_file: + with open(path_to_file, "a") as yaml_file: + ruamel.yaml.round_trip_dump(dictionary, yaml_file, + default_flow_style=False, + explicit_start=True, + allow_unicode=True + ) + ############################################################################### class CourseManager(models.Manager): @@ -489,23 +505,8 @@ class Question(models.Model): def _add_yaml_to_zip(self, zip_file, q_dict): tmp_file_path = tempfile.mkdtemp() yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") - - def _dict_walk(question_dict): - for k,v in question_dict.items(): - if isinstance(v, list): - for nested_v in v: - if isinstance(nested_v, dict): - _dict_walk(nested_v) - elif v and isinstance(v,str): - question_dict[k] = PreservedScalarString(v) for elem in q_dict: - _dict_walk(elem) - with open(yaml_path, "a") as yaml_file: - ruamel.yaml.round_trip_dump(elem, yaml_file, - default_flow_style=False, - explicit_start=True, - allow_unicode=True - ) + dict_to_yaml(elem, yaml_path) zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) -- cgit From a9c4aab85fb9b8edce53212548f8d0c285832dc4 Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 18 Aug 2017 02:12:37 +0530 Subject: Adds tags in yaml files --- yaksh/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 4113a62..476bc16 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -409,12 +409,13 @@ class Question(models.Model): for question in questions: test_case = question.get_test_cases() file_names = question._add_and_get_files(zip_file) - q_dict = model_to_dict(question, exclude=['id', 'user','tags']) + q_dict = model_to_dict(question, exclude=['id', 'user']) testcases = [] for case in test_case: testcases.append(case.get_field_value()) q_dict['testcase'] = testcases q_dict['files'] = file_names + q_dict['tags'] = [tags.tag.name for tags in q_dict['tags']] questions_dict.append(q_dict) question._add_yaml_to_zip(zip_file, questions_dict) return zip_file_name @@ -428,10 +429,12 @@ class Question(models.Model): question['user'] = user file_names = question.pop('files') test_cases = question.pop('testcase') + tags = question.pop('tags') que, result = Question.objects.get_or_create(**question) if file_names: que._add_files_to_db(file_names, file_path) - + if tags: + que.tags.add(*tags) for test_case in test_cases: try: test_case_type = test_case.pop('test_case_type') @@ -442,6 +445,7 @@ class Question(models.Model): ) new_test_case.type = test_case_type new_test_case.save() + except: msg = "File not correct." except Exception as exc_msg: -- cgit From 0ffc49f91dd9e21a6b9917b7841999bf853c3c9f Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 18 Aug 2017 02:13:10 +0530 Subject: Adds test cases for yaml - Fixes selenium test cases, test_models, test_views - Fixes create demo question paper --- yaksh/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 476bc16..b4c665c 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -909,8 +909,13 @@ class QuestionPaper(models.Model): total_marks=6.0, shuffle_questions=True ) + summaries = ['Roots of quadratic equation', 'Print Output', + 'Adding decimals', 'For Loop over String', + 'Hello World in File', 'Extract columns from files', + 'Check Palindrome', 'Add 3 numbers', 'Reverse a string' + ] questions = Question.objects.filter(active=True, - summary="Yaksh Demo Question", + summary__in=summaries, user=user) q_order = [str(que.id) for que in questions] question_paper.fixed_question_order = ",".join(q_order) -- cgit From ceb55baf69c2f5f7346855ee5a6e5e9f77456fcb Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 21 Jul 2017 17:07:24 +0530 Subject: Add a has_profile decorator --- yaksh/models.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 87e6260..30ecde0 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -92,11 +92,6 @@ def get_model_class(model): return model_class -def has_profile(user): - """ check if user has profile """ - return True if hasattr(user, 'profile') else False - - def get_upload_dir(instance, filename): return os.sep.join(( 'question_%s' % (instance.question.id), filename -- cgit From 3367cf1bef0b8856972f4e4125322194c699d69e Mon Sep 17 00:00:00 2001 From: maheshgudi Date: Thu, 24 Aug 2017 16:55:18 +0530 Subject: Adds alphabetical order during yaml serializing --- yaksh/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index b4c665c..cd79268 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta import json import ruamel.yaml from ruamel.yaml.scalarstring import PreservedScalarString +from ruamel.yaml.comments import CommentedMap from random import sample from collections import Counter from django.db import models @@ -119,8 +120,8 @@ def dict_to_yaml(dictionary, path_to_file=None): ruamel.yaml.round_trip_dump(dictionary, yaml_file, default_flow_style=False, explicit_start=True, - allow_unicode=True - ) + allow_unicode=True, + ) ############################################################################### @@ -510,7 +511,8 @@ class Question(models.Model): tmp_file_path = tempfile.mkdtemp() yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") for elem in q_dict: - dict_to_yaml(elem, yaml_path) + commented_map = CommentedMap(sorted(elem.items(), key=lambda x:x[0])) + dict_to_yaml(commented_map, yaml_path) zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) -- cgit From 30b48c30abebf75ed4b51fd034600e0c7d58c95b Mon Sep 17 00:00:00 2001 From: mahesh Date: Fri, 25 Aug 2017 01:33:08 +0530 Subject: Fixes order in demo_questions.zip - Template yaml is now generated on the fly. - Removes yaml_question_template files. - Fixes order for yaml file inside demo_questions.zip --- yaksh/models.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'yaksh/models.py') diff --git a/yaksh/models.py b/yaksh/models.py index 044a164..979740d 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -102,7 +102,7 @@ def get_upload_dir(instance, filename): 'question_%s' % (instance.question.id), filename )) -def dict_to_yaml(dictionary, path_to_file=None): +def dict_to_yaml(dictionary): for k,v in dictionary.items(): if isinstance(v, list): for nested_v in v: @@ -110,14 +110,10 @@ def dict_to_yaml(dictionary, path_to_file=None): dict_to_yaml(nested_v) elif v and isinstance(v,str): dictionary[k] = PreservedScalarString(v) - if path_to_file: - with open(path_to_file, "a") as yaml_file: - ruamel.yaml.round_trip_dump(dictionary, yaml_file, - default_flow_style=False, - explicit_start=True, - allow_unicode=True, - ) - + return ruamel.yaml.round_trip_dump(dictionary, explicit_start=True, + default_flow_style=False, + allow_unicode=True, + ) ############################################################################### class CourseManager(models.Manager): @@ -502,12 +498,15 @@ class Question(models.Model): file_upload.extract = extract file_upload.file.save(file_name, django_file, save=True) - def _add_yaml_to_zip(self, zip_file, q_dict): + def _add_yaml_to_zip(self, zip_file, q_dict,path_to_file=None): + tmp_file_path = tempfile.mkdtemp() yaml_path = os.path.join(tmp_file_path, "questions_dump.yaml") for elem in q_dict: - commented_map = CommentedMap(sorted(elem.items(), key=lambda x:x[0])) - dict_to_yaml(commented_map, yaml_path) + sorted_dict = CommentedMap(sorted(elem.items(), key=lambda x:x[0])) + yaml_block = dict_to_yaml(sorted_dict) + with open(yaml_path, "a") as yaml_file: + yaml_file.write(yaml_block) zip_file.write(yaml_path, os.path.basename(yaml_path)) zip_file.close() shutil.rmtree(tmp_file_path) @@ -518,7 +517,9 @@ class Question(models.Model): if os.path.exists(yaml_file): with open(yaml_file, 'r') as q_file: questions_list = q_file.read() - msg = self.load_questions(questions_list, user, file_path, files) + msg = self.load_questions(questions_list, user, + file_path, files + ) else: msg = "Please upload zip file with questions_dump.yaml in it." -- cgit