From c883413f4e7ef4642f9d18a0d383612f6abd9391 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 15 Nov 2017 16:50:24 +0530 Subject: - Add python-decouple package to requirements - Use sane defaults and python-decouple in yaksh.settings module - Replace hard coded localhost domain name referring to code server with dynamic settings variable --- online_test/settings.py | 16 +++++++++++----- requirements/requirements-codeserver.txt | 1 + yaksh/models.py | 10 +++++----- yaksh/settings.py | 11 ++++++++--- yaksh/views.py | 7 ++++--- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/online_test/settings.py b/online_test/settings.py index 790083e..3cd36a4 100644 --- a/online_test/settings.py +++ b/online_test/settings.py @@ -8,8 +8,10 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.6/ref/settings/ """ from yaksh.pipeline.settings import AUTH_PIPELINE -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +from decouple import config + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) # The directory where user data can be saved. This directory will be @@ -21,12 +23,11 @@ OUTPUT_DIR = os.path.join(BASE_DIR, "yaksh_data", "output") # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '0=fsi3g5dw*7ze1cyh441_e^5^$2ay@&z(5(n7mhir0xb267=6' +SECRET_KEY = config('SECRET_KEY', default='dUmMy_s3cR3t_k3y')#'0=fsi3g5dw*7ze1cyh441_e^5^$2ay@&z(5(n7mhir0xb267=6' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True - ALLOWED_HOSTS = [] URL_ROOT = '' @@ -67,8 +68,13 @@ WSGI_APPLICATION = 'online_test.wsgi.application' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + #'ENGINE': 'django.db.backends.sqlite3', + 'ENGINE': 'django.db.backends.{0}'.format( + config('DB_ENGINE', default='sqlite3') + ), + 'NAME': config('DB_NAME', default=os.path.join(BASE_DIR, 'db.sqlite3')) + #os.path.join(BASE_DIR, 'db.sqlite3'), + }, } diff --git a/requirements/requirements-codeserver.txt b/requirements/requirements-codeserver.txt index a4f419c..e9585fa 100644 --- a/requirements/requirements-codeserver.txt +++ b/requirements/requirements-codeserver.txt @@ -1,4 +1,5 @@ pytest +python-decouple six requests tornado diff --git a/yaksh/models.py b/yaksh/models.py index 1d24bda..bc0b4f1 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -29,10 +29,10 @@ import tempfile from textwrap import dedent from ast import literal_eval from .file_utils import extract_files, delete_files -from yaksh.code_server import(submit, - get_result as get_result_from_code_server - ) -from yaksh.settings import SERVER_POOL_PORT +from yaksh.code_server import ( + submit, get_result as get_result_from_code_server +) +from yaksh.settings import SERVER_POOL_PORT, SERVER_HOST_NAME from django.conf import settings from django.forms.models import model_to_dict @@ -1705,7 +1705,7 @@ class AnswerPaper(models.Model): elif question.type == 'code' or question.type == "upload": user_dir = self.user.profile.get_user_dir() - url = 'http://localhost:%s' % server_port + url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT) submit(url, uid, json_data, user_dir) result = {'uid': uid, 'status': 'running'} return result diff --git a/yaksh/settings.py b/yaksh/settings.py index d500d93..e414e75 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -2,16 +2,21 @@ settings for yaksh app. """ +from decouple import config + # The number of code server processes to run.. -N_CODE_SERVERS = 5 +N_CODE_SERVERS = config('N_CODE_SERVERS', default=5, cast=int) # The server pool port. This is the server which returns available server # ports so as to minimize load. This is some random number where no other # service is running. It should be > 1024 and less < 65535 though. -SERVER_POOL_PORT = 53579 +SERVER_POOL_PORT = config('EMAIL_PORT', default=55555, cast=int) + +SERVER_HOST_NAME = config('SERVER_HOST_NAME', default='http://localhost') +#'localhost' # Timeout for the code to run in seconds. This is an integer! -SERVER_TIMEOUT = 4 +SERVER_TIMEOUT = config('SERVER_TIMEOUT', default=4, cast=int) # The root of the URL, for example you might be in the situation where you # are not hosted as host.org/exam/ but as host.org/foo/exam/ for whatever diff --git a/yaksh/views.py b/yaksh/views.py index a4d9e78..a001ffd 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -35,7 +35,7 @@ except ImportError: from io import BytesIO as string_io import re # Local imports. -from yaksh.code_server import get_result as get_result_from_code_server, SERVER_POOL_PORT +from yaksh.code_server import get_result as get_result_from_code_server, from yaksh.models import ( Answer, AnswerPaper, AssignmentUpload, Course, FileUpload, FloatTestCase, HookTestCase, IntegerTestCase, McqTestCase, Profile, @@ -51,6 +51,7 @@ from yaksh.forms import ( UploadFileForm, get_object_form, FileForm, QuestionPaperForm, LessonForm, LessonFileForm, LearningModuleForm, ExerciseForm ) +from yaksh.settings import SERVER_POOL_PORT, SERVER_HOST_NAME from .settings import URL_ROOT from .file_utils import extract_files, is_csv from .send_emails import send_user_mail, generate_activation_key, send_bulk_mail @@ -782,7 +783,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None, ) if current_question.type in ['code', 'upload']: if paper.time_left() <= 0 and not paper.question_paper.quiz.is_exercise: - url = 'http://localhost:%s' % SERVER_POOL_PORT + url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT) result_details = get_result_from_code_server(url, uid, block=True) result = json.loads(result_details.get('result')) next_question, error_message, paper = _update_paper(request, uid, @@ -807,7 +808,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None, @csrf_exempt def get_result(request, uid, course_id, module_id): result = {} - url = 'http://localhost:%s' % SERVER_POOL_PORT + url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT) result_state = get_result_from_code_server(url, uid) result['status'] = result_state.get('status') if result['status'] == 'done': -- cgit From 9a871cee5d6836b82cc906ad74bc6731bb8e72f7 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 15 Nov 2017 17:11:26 +0530 Subject: Move Dockerfile to dedicated directory --- Dockerfile | 11 ----------- docker/Dockerfile | 13 +++++++++++++ 2 files changed, 13 insertions(+), 11 deletions(-) delete mode 100644 Dockerfile create mode 100644 docker/Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 6a7d894..0000000 --- a/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER FOSSEE - -# Update Packages and Install Python & net-tools -RUN apt-get update && \ -apt-get install -y software-properties-common && \ -echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ -add-apt-repository -y ppa:webupd8team/java && \ -apt-get update && \ -apt-get install -y oracle-java8-installer && \ -apt-get install -y sudo python net-tools git python3-pip vim libmysqlclient-dev scilab build-essential python3-numpy python3-scipy ipython3 ipython3-notebook python3-pandas python3-nose diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..cc0fd30 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:16.04 + +MAINTAINER FOSSEE + +# Update Packages and Install Python & net-tools +RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y + +RUN apt-get install -y software-properties-common && \ +echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ +add-apt-repository -y ppa:webupd8team/java && \ +apt-get update && \ +apt-get install -y oracle-java8-installer && \ +apt-get install -y python net-tools scilab build-essential python3-numpy python3-scipy ipython3 ipython3-notebook python3-pandas python3-nose -- cgit From 2c057133d1b131a1573c93f0d0dbb548dcc8223e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 17 Nov 2017 17:13:26 +0530 Subject: Add dynamic run-as-admin commands to the base command depending on OS --- tasks.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tasks.py b/tasks.py index 68d9967..04a2ef5 100644 --- a/tasks.py +++ b/tasks.py @@ -9,6 +9,7 @@ TARGET_CONTAINER_NAME = 'yaksh_code_server' SRC_IMAGE_NAME = 'fossee/yaksh_codeserver' CHECK_FILE = 'server_running.txt' CHECK_FILE_PATH = os.path.join(SCRIPT_DIR, 'yaksh_data', CHECK_FILE) +OS_NAME = sys.platform def create_dir(path): @@ -19,6 +20,18 @@ def remove_check_file(path): if os.path.isfile(path): os.remove(path) +def run_as(os_name): + if os_name.startswith('linux') or os_name == 'darwin': + return 'sudo' + elif os_name.startswith('Win32'): + return None + +def get_cmd(run_as_cmd, base_cmd): + if run_as_cmd: + return '{0} {1}'.format(run_as_cmd, base_cmd) + else: + return base_cmd + @task def setupdb(ctx): print("** Setting up & migrating database **") @@ -43,7 +56,10 @@ def getimage(ctx, image=SRC_IMAGE_NAME): except invoke.exceptions.Failure: print("The docker image {0} does not exist locally".format(image)) print("\n** Pulling latest image <{0}> from docker hub **".format(image)) - ctx.run("sudo docker pull {0}".format(image)) + base_cmd = "docker pull {0}".format(image) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) print("\n** Done! Successfully pulled latest image <{0}> **".format(image)) @task @@ -52,10 +68,12 @@ def start(ctx, ports=SERVER_POOL_PORT, image=SRC_IMAGE_NAME, unsafe=False, if unsafe: with ctx.cd(SCRIPT_DIR): print("** Initializing local code server **") - ctx.run("sudo python{0} -m yaksh.code_server".format( - version - ) + base_cmd = "python{0} -m yaksh.code_server".format( + version ) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) else: cmd_params = {'ports': ports, 'image': SRC_IMAGE_NAME, @@ -104,9 +122,17 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): result = ctx.run("sudo docker ps -q --filter='name={0}'".format(container)) remove_check_file(CHECK_FILE_PATH) if result.stdout: + print ("** Stopping the docker container <{0}>".format(container)) + base_cmd = "docker stop {0}".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) + print ("** Done! Stopped the docker container <{0}>".format(container)) + print ("** Discarding the docker container <{0}>".format(container)) - ctx.run("sudo docker stop {0}".format(container)) - ctx.run("sudo docker rm {0}".format(container)) + base_cmd = "docker rm {0}".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) print ("** Done! Discarded the docker container <{0}>".format(container)) else: print("** Docker container <{0}> not found **".format(container)) -- cgit From 6f2e29a067cbeaf88b8ea478bb7c24fe8772180a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 20 Nov 2017 16:07:43 +0530 Subject: Multiple Changes: - Add python-decouple package to requirements - use sane defaults and python-decouple in yaksh.settings and online_test.settings module - Replace hardcoded localhost domain name referring to code server with dynamic - Move Dockerfile to dedicated directory - Add dynamic run-as-admin commands to the base command depending on OS - Replace linux specific commands like cp with Python functions --- online_test/.env.sample | 16 +++++++++++++++ online_test/settings.py | 13 ++++++------ tasks.py | 54 ++++++++++++++++++++++++++++++------------------- yaksh/settings.py | 2 +- yaksh/views.py | 2 +- 5 files changed, 58 insertions(+), 29 deletions(-) create mode 100644 online_test/.env.sample diff --git a/online_test/.env.sample b/online_test/.env.sample new file mode 100644 index 0000000..908ec72 --- /dev/null +++ b/online_test/.env.sample @@ -0,0 +1,16 @@ +# Django settings + +SECRET_KEY=dUmMy_s3cR3t_k3y +DB_ENGINE=sqlite3 +#DB_NAME=name_of_db +#DB_USER=name_of_db_user +#DB_PASSWORD=db_password +#DB_HOST=db_hostname +#DB_PORT=8180 + +# Yaksh settings + +#N_CODE_SERVERS=5 +#SERVER_POOL_PORT=55555 +#SERVER_HOST_NAME=http://localhost +#SERVER_TIMEOUT=4 diff --git a/online_test/settings.py b/online_test/settings.py index 3cd36a4..e64a791 100644 --- a/online_test/settings.py +++ b/online_test/settings.py @@ -23,7 +23,7 @@ OUTPUT_DIR = os.path.join(BASE_DIR, "yaksh_data", "output") # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = config('SECRET_KEY', default='dUmMy_s3cR3t_k3y')#'0=fsi3g5dw*7ze1cyh441_e^5^$2ay@&z(5(n7mhir0xb267=6' +SECRET_KEY = config('SECRET_KEY', default='dUmMy_s3cR3t_k3y') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -65,16 +65,17 @@ WSGI_APPLICATION = 'online_test.wsgi.application' # Database # https://docs.djangoproject.com/en/1.6/ref/settings/#databases - DATABASES = { 'default': { - #'ENGINE': 'django.db.backends.sqlite3', 'ENGINE': 'django.db.backends.{0}'.format( config('DB_ENGINE', default='sqlite3') ), - 'NAME': config('DB_NAME', default=os.path.join(BASE_DIR, 'db.sqlite3')) - #os.path.join(BASE_DIR, 'db.sqlite3'), - + 'NAME': config('DB_NAME', default=os.path.join(BASE_DIR, 'db.sqlite3')), + # The following settings are not used with sqlite3: + 'USER': config('DB_USER', default=''), + 'PASSWORD': config('DB_PASSWORD', default=''), + 'HOST': config('DB_HOST', default='localhost'), # Empty for localhost through domain sockets or '1$ + 'PORT': config('DB_PORT', default=''), }, } diff --git a/tasks.py b/tasks.py index 04a2ef5..47edb20 100644 --- a/tasks.py +++ b/tasks.py @@ -1,7 +1,13 @@ from __future__ import print_function +import os +import sys +import shutil +from distutils.dir_util import copy_tree +from distutils.file_util import copy_file + import invoke from invoke import task -import os + from yaksh.settings import SERVER_POOL_PORT SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -52,7 +58,10 @@ def clean(ctx): @task def getimage(ctx, image=SRC_IMAGE_NAME): try: - result = ctx.run("sudo docker inspect {0}".format(image), hide=True) + base_cmd = "docker inspect {0}".format(image) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd, hide=True) except invoke.exceptions.Failure: print("The docker image {0} does not exist locally".format(image)) print("\n** Pulling latest image <{0}> from docker hub **".format(image)) @@ -92,47 +101,50 @@ def start(ctx, ports=SERVER_POOL_PORT, image=SRC_IMAGE_NAME, unsafe=False, create_dir(os.path.join(SCRIPT_DIR, 'yaksh_data', 'data')) create_dir(os.path.join(SCRIPT_DIR, 'yaksh_data', 'output')) - ctx.run('cp -r {0} {1}'.format( - os.path.join(SCRIPT_DIR, 'yaksh'), - os.path.join(SCRIPT_DIR, 'yaksh_data') - ) + copy_tree( + os.path.join(SCRIPT_DIR, 'yaksh'), + os.path.join(SCRIPT_DIR, 'yaksh_data', 'yaksh') ) - ctx.run('cp {0} {1}'.format( - os.path.join(SCRIPT_DIR, 'requirements', 'requirements-codeserver.txt'), - os.path.join(SCRIPT_DIR, 'yaksh_data') - ) + + copy_file( + os.path.join(SCRIPT_DIR, 'requirements', 'requirements-codeserver.txt'), + os.path.join(SCRIPT_DIR, 'yaksh_data') ) print("** Initializing code server within docker container **") - ctx.run( - "sudo docker run \ + base_cmd = "docker run \ -dp {ports}:{ports} --name={name} \ -v {vol_mount}:{vol_mount} \ -w {vol_mount} \ {image} {command}".format(**cmd_params) - ) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) while not os.path.isfile(CHECK_FILE_PATH): print("** Checking code server status. Press Ctrl-C to exit **\r", end="") - print("** Code server is up and running successfully **") + print("\n** Code server is up and running successfully **") @task def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): - result = ctx.run("sudo docker ps -q --filter='name={0}'".format(container)) + base_filter_cmd = "docker ps -q --filter='name={0}'".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_filter_cmd) + result = ctx.run(cmd) + remove_check_file(CHECK_FILE_PATH) if result.stdout: print ("** Stopping the docker container <{0}>".format(container)) - base_cmd = "docker stop {0}".format(container) - run_as_cmd = run_as(OS_NAME) - cmd = get_cmd(run_as_cmd, base_cmd) + base_stop_cmd = "docker stop {0}".format(container) + cmd = get_cmd(run_as_cmd, base_stop_cmd) ctx.run(cmd) print ("** Done! Stopped the docker container <{0}>".format(container)) print ("** Discarding the docker container <{0}>".format(container)) - base_cmd = "docker rm {0}".format(container) - run_as_cmd = run_as(OS_NAME) - cmd = get_cmd(run_as_cmd, base_cmd) + base_rm_cmd = "docker rm {0}".format(container) + cmd = get_cmd(run_as_cmd, base_rm_cmd) + ctx.run(cmd) print ("** Done! Discarded the docker container <{0}>".format(container)) else: print("** Docker container <{0}> not found **".format(container)) diff --git a/yaksh/settings.py b/yaksh/settings.py index e414e75..d895d19 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -10,7 +10,7 @@ N_CODE_SERVERS = config('N_CODE_SERVERS', default=5, cast=int) # The server pool port. This is the server which returns available server # ports so as to minimize load. This is some random number where no other # service is running. It should be > 1024 and less < 65535 though. -SERVER_POOL_PORT = config('EMAIL_PORT', default=55555, cast=int) +SERVER_POOL_PORT = config('SERVER_POOL_PORT', default=55555, cast=int) SERVER_HOST_NAME = config('SERVER_HOST_NAME', default='http://localhost') #'localhost' diff --git a/yaksh/views.py b/yaksh/views.py index a001ffd..8fe4523 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -35,7 +35,7 @@ except ImportError: from io import BytesIO as string_io import re # Local imports. -from yaksh.code_server import get_result as get_result_from_code_server, +from yaksh.code_server import get_result as get_result_from_code_server from yaksh.models import ( Answer, AnswerPaper, AssignmentUpload, Course, FileUpload, FloatTestCase, HookTestCase, IntegerTestCase, McqTestCase, Profile, -- cgit From 0713c60a5ad4b71d4a2050d13cf30afaf37d935b Mon Sep 17 00:00:00 2001 From: rohmhatre Date: Sun, 26 Nov 2017 00:25:02 +0530 Subject: Added docker compose configuration --- docker/Dockerfile | 13 ------- docker/Dockerfile_codeserver | 19 ++++++++++ docker/Dockerfile_django | 23 ++++++++++++ docker/Files/000-default.conf | 18 ++++++++++ docker/Files/Docker-script.sh | 5 +++ docker/Files/Start-codeserver.sh | 7 ++++ docker/Files/requirements-codeserver.txt | 6 ++++ docker/Files/requirements-common.txt | 9 +++++ docker/Files/requirements-py2.txt | 2 ++ docker/Files/requirements-py3.txt | 2 ++ docker/Makefile | 61 ++++++++++++++++++++++++++++++++ docker/README.md | 30 ++++++++++++++++ docker/docker-compose.yml | 41 +++++++++++++++++++++ manage.py | 0 online_test/settings.py | 2 ++ yaksh/code_server.py | 0 yaksh/docs/sample.sh | 0 17 files changed, 225 insertions(+), 13 deletions(-) delete mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile_codeserver create mode 100644 docker/Dockerfile_django create mode 100644 docker/Files/000-default.conf create mode 100644 docker/Files/Docker-script.sh create mode 100644 docker/Files/Start-codeserver.sh create mode 100644 docker/Files/requirements-codeserver.txt create mode 100644 docker/Files/requirements-common.txt create mode 100644 docker/Files/requirements-py2.txt create mode 100644 docker/Files/requirements-py3.txt create mode 100644 docker/Makefile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yml mode change 100755 => 100644 manage.py mode change 100755 => 100644 yaksh/code_server.py mode change 100755 => 100644 yaksh/docs/sample.sh diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index cc0fd30..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:16.04 - -MAINTAINER FOSSEE - -# Update Packages and Install Python & net-tools -RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y - -RUN apt-get install -y software-properties-common && \ -echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ -add-apt-repository -y ppa:webupd8team/java && \ -apt-get update && \ -apt-get install -y oracle-java8-installer && \ -apt-get install -y python net-tools scilab build-essential python3-numpy python3-scipy ipython3 ipython3-notebook python3-pandas python3-nose diff --git a/docker/Dockerfile_codeserver b/docker/Dockerfile_codeserver new file mode 100644 index 0000000..1bc93c2 --- /dev/null +++ b/docker/Dockerfile_codeserver @@ -0,0 +1,19 @@ +FROM ubuntu:16.04 + +MAINTAINER FOSSEE + +RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y + +VOLUME /Sites/online_test + +ADD Files/requirements-* /tmp/ + +RUN pip3 install -r /tmp/requirements-codeserver.txt && mkdir -p /Sites/online_test/yaksh_data/output /Sites/online_test/yaksh_data/data + +WORKDIR /Sites/online_test/yaksh_data + +ADD Files/Start-codeserver.sh /Sites + +EXPOSE 53579 + +CMD [ "/bin/bash" , "/Sites/Start-codeserver.sh" ] diff --git a/docker/Dockerfile_django b/docker/Dockerfile_django new file mode 100644 index 0000000..6b87afb --- /dev/null +++ b/docker/Dockerfile_django @@ -0,0 +1,23 @@ +FROM ubuntu:16.04 + +MAINTAINER FOSSEE + +RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y + +RUN apt-get install apache2 vim libapache2-mod-wsgi-py3 python3-django -y && mkdir -p /Sites/online_test + +VOLUME /Sites/online_test + +ADD Files/requirements-* /tmp/ + +RUN cd /Sites/online_test && pip3 install -r /tmp/requirements-py3.txt + +ADD Files/000-default.conf /etc/apache2/sites-enabled/ + +ADD Files/Docker-script.sh /Sites/Docker-script.sh + +EXPOSE 80 + +WORKDIR /Sites/online_test + +CMD [ "/bin/bash" , "/Sites/Docker-script.sh" ] diff --git a/docker/Files/000-default.conf b/docker/Files/000-default.conf new file mode 100644 index 0000000..6bcb382 --- /dev/null +++ b/docker/Files/000-default.conf @@ -0,0 +1,18 @@ + + ServerName localhost + ServerAdmin webmaster@localhost + DocumentRoot /Sites/online_test + WSGIDaemonProcess yaksh threads=5 python-path=/usr/local/lib/python3.5/dist-packages home=/Sites/online_test + WSGIScriptAlias / /Sites/online_test/online_test/wsgi.py + WSGIProcessGroup yaksh + Alias /static/ /Sites/online_test/yaksh/static/ + + WSGIProcessGroup yaksh + WSGIApplicationGroup %{GLOBAL} + Order deny,allow + Allow from all + Require all granted + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + diff --git a/docker/Files/Docker-script.sh b/docker/Files/Docker-script.sh new file mode 100644 index 0000000..752a638 --- /dev/null +++ b/docker/Files/Docker-script.sh @@ -0,0 +1,5 @@ +#!/bin/bash +chown -R www-data /Sites/online_test +chmod -R 644 /Sites/online_test +chmod -R +X /Sites +/usr/sbin/apache2ctl -D FOREGROUND diff --git a/docker/Files/Start-codeserver.sh b/docker/Files/Start-codeserver.sh new file mode 100644 index 0000000..8b6ea96 --- /dev/null +++ b/docker/Files/Start-codeserver.sh @@ -0,0 +1,7 @@ +#!/bin/bash +chown -R nobody /Sites/online_test +chmod -R a+rwX yaksh_data/output +chmod -R a+rX yaksh_data/data yaksh_data/yaksh +chmod -R o-w yaksh_data/data yaksh +chmod -R +X /Sites/online_test/ +/usr/bin/sudo -su nobody python3 -m yaksh.code_server diff --git a/docker/Files/requirements-codeserver.txt b/docker/Files/requirements-codeserver.txt new file mode 100644 index 0000000..e9585fa --- /dev/null +++ b/docker/Files/requirements-codeserver.txt @@ -0,0 +1,6 @@ +pytest +python-decouple +six +requests +tornado +psutil diff --git a/docker/Files/requirements-common.txt b/docker/Files/requirements-common.txt new file mode 100644 index 0000000..b4d2e5b --- /dev/null +++ b/docker/Files/requirements-common.txt @@ -0,0 +1,9 @@ +-r requirements-codeserver.txt +invoke==0.21.0 +django==1.9.5 +django-taggit==0.18.1 +pytz==2016.4 +python-social-auth==0.2.19 +selenium==2.53.6 +coverage +ruamel.yaml==0.15.23 \ No newline at end of file diff --git a/docker/Files/requirements-py2.txt b/docker/Files/requirements-py2.txt new file mode 100644 index 0000000..38777a1 --- /dev/null +++ b/docker/Files/requirements-py2.txt @@ -0,0 +1,2 @@ +-r requirements-common.txt +mysql-python==1.2.5 diff --git a/docker/Files/requirements-py3.txt b/docker/Files/requirements-py3.txt new file mode 100644 index 0000000..3d13335 --- /dev/null +++ b/docker/Files/requirements-py3.txt @@ -0,0 +1,2 @@ +-r requirements-common.txt +mysqlclient==1.3.9 diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 0000000..9119601 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,61 @@ +help: + @echo "Docker Compose Help for Deploying Yaksh interface" + @echo "-----------------------\n" + @echo "Clone yaksh online interface from github :" + @echo " make clone" + @echo "" + @echo "Build docker images:" + @echo " make build" + @echo "" + @echo "To run containers:" + @echo " make start" + @echo "" + @echo "You need to create super a user to work with yaksh:" + @echo " make createsuperuser" + @echo "" + @echo "Now Your interface is ready. You can access it using browser just go to http://localhost:8000" + @echo "" + @echo "Really, really start over:" + @echo " make clean" + @echo "" + @echo "other utilities:restart,tail,status" + +begin: migrate fixtures start + +clone: + @/bin/bash clone-yaksh.sh + +start: + @docker-compose up -d + +stop: + @docker-compose stop + +status: + @docker-compose ps + +restart: stop start + +clean: stop + @docker-compose rm --force + +build: + @docker-compose build --no-cache + @docker pull mariadb:10.2 + +createsuperuser: migrate superuser + +migrate: + @docker exec -it yaksh_django python3 manage.py makemigrations + @docker exec -it yaksh_django python3 manage.py migrate + +superuser: + @docker exec -it yaksh_django python3 manage.py createsuperuser + @docker exec -it yaksh_django python3 manage.py add_group + @docker exec -it yaksh_django python3 manage.py collectstatic + @echo "You can access it using browser, just go to http://localhost:8000" + +tail: + @docker-compose logs -f + +.PHONY: start stop status restart clean build migrate tail clone superuser createsuperuser diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..34e30da --- /dev/null +++ b/docker/README.md @@ -0,0 +1,30 @@ +#### Pre-Requisites + + 1. Install Docker https://docs.docker.com/engine/installation/ + 2. Install Docker Compose https://docs.docker.com/compose/install/#install-compose + 3. install git + + +#### Help for deploying Yaksh interface. + + - To get help while deployment + $ make help + + - Clone yaksh form github: + $ make clone + + - Build docker images: + $ make build + + - To run containers: + $ make start + + - You need to create super a user to work with yaksh: + $ make createsuperuser + + - Now Your interface is ready. You can access it using browser just go to http://localhost:8000 + + - Clean your docker containers: + $ make clean + + - other utilities like restart, tail, status diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..747cbda --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,41 @@ +version: '2' +services: + yaksh-django: + build: + context: . + dockerfile: Dockerfile_django + image: yaksh.django + container_name: yaksh_django + volumes: + - ../.:/Sites/online_test + depends_on: + - yaksh-db + - yaksh-codeserver + links: + - yaksh-db + - yaksh-codeserver + ports: + - 8000:80 + restart: always + + + yaksh-codeserver: + build: + context: . + dockerfile: Dockerfile_codeserver + image: yaksh.codeserver + container_name: yaksh_codeserver + volumes: + - ../yaksh_data:/Sites/online_test/yaksh_data + restart: always + + + yaksh-db: + image: mariadb:10.2 + container_name: yaksh_db + volumes: + - ./mysql:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: yaksh + restart: always diff --git a/manage.py b/manage.py old mode 100755 new mode 100644 diff --git a/online_test/settings.py b/online_test/settings.py index e64a791..c55a056 100644 --- a/online_test/settings.py +++ b/online_test/settings.py @@ -107,6 +107,8 @@ MEDIA_URL = "/data/" MEDIA_ROOT = os.path.join(BASE_DIR, "yaksh_data", "data") +STATIC_ROOT='yaksh/static/' + # Set this varable to if smtp-server is not allowing to send email. EMAIL_USE_TLS = False diff --git a/yaksh/code_server.py b/yaksh/code_server.py old mode 100755 new mode 100644 diff --git a/yaksh/docs/sample.sh b/yaksh/docs/sample.sh old mode 100755 new mode 100644 -- cgit From 8c9c7fdf227c7a107345c70fea4cf77cc550a10f Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 7 Dec 2017 16:25:52 +0530 Subject: Multiple changes: - Add deployment commands to tasks.py - Add deployment commands to README_production.md --- README_production.md | 25 ++++++++++++- tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/README_production.md b/README_production.md index 8b79785..1251ce0 100644 --- a/README_production.md +++ b/README_production.md @@ -169,7 +169,7 @@ To install this app follow the steps below: 1. Install [Docker](https://github.com/FOSSEE/online_test/blob/master/README.md) - 1. Got to the directory where the project is located + 1. Go to the directory where the project is located cd /path/to/online_test 1. Create a docker image. This may take a few minutes @@ -181,6 +181,29 @@ To install this app follow the steps below: 1. Run the invoke script using the command ```invoke start``` The command will create and run a new docker container (that is running the code_server.py within it), it will also bind the ports of the host with those of the container +#### Deploying Multiple Dockers + + 1. Install [Docker](https://github.com/FOSSEE/online_test/blob/master/README.md) + + 1. Go to the ```docker``` directory + + 1. Build the docker images + invoke build + + 1. Run the containers and scripts necessary to deploy the web application + invoke deploy + + Use ```invoke deploy --fixtures``` to load the fixtures + + 1. Create the superuser and moderator group + invoke createsuperuser + + 1. Stop the containers + invoke halt + + 1. Remove the containers + invoke clean + #### Additional commands available We provide several convenient commands for you to use: diff --git a/tasks.py b/tasks.py index 47edb20..d518374 100644 --- a/tasks.py +++ b/tasks.py @@ -43,9 +43,14 @@ def setupdb(ctx): print("** Setting up & migrating database **") ctx.run("python manage.py makemigrations") ctx.run("python manage.py migrate") + print("** Done! Migrations complete **") + +def loadfixtures(ctx): + print("** Loading fixtures into database **") ctx.run("python manage.py loaddata demo_fixtures.json") + print("** Done! Loaded fixtures into database **") -@task(setupdb) +@task(setupdb, loadfixtures) def serve(ctx): print("** Running the Django web server. Press Ctrl-C to Exit **") ctx.run("python manage.py runserver") @@ -135,16 +140,105 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): remove_check_file(CHECK_FILE_PATH) if result.stdout: - print ("** Stopping the docker container <{0}>".format(container)) + print ("** Stopping the docker container <{0}> **".format(container)) base_stop_cmd = "docker stop {0}".format(container) cmd = get_cmd(run_as_cmd, base_stop_cmd) ctx.run(cmd) - print ("** Done! Stopped the docker container <{0}>".format(container)) + print ("** Done! Stopped the docker container <{0}> **".format(container)) - print ("** Discarding the docker container <{0}>".format(container)) + print ("** Discarding the docker container <{0}> **".format(container)) base_rm_cmd = "docker rm {0}".format(container) cmd = get_cmd(run_as_cmd, base_rm_cmd) ctx.run(cmd) - print ("** Done! Discarded the docker container <{0}>".format(container)) + print ("** Done! Discarded the docker container <{0}> **".format(container)) else: print("** Docker container <{0}> not found **".format(container)) + +@task +def build(ctx): + run_as_cmd = run_as(OS_NAME) + + base_build_cmd = "docker-compose build --no-cache" + cmd = get_cmd(run_as_cmd, base_build_cmd) + print ("** Building docker images **") + ctx.run(cmd) + print ("** Done! Built the docker images **") + + base_build_cmd = "docker pull mariadb:10.2 " + cmd = get_cmd(run_as_cmd, base_build_cmd) + print ("** Pulling maria-db base image **") + ctx.run(cmd) + print ("** Done! Pulled maria-db base image **") + +@task +def begin(ctx): + print("** Initializing docker containers **") + base_cmd = "docker-compose up -d" + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) + print ("** Done! Initialized the docker containers **") + +@task(begin) +def deploy(ctx, fixtures=False): + run_as_cmd = run_as(OS_NAME) + + print("** Setting up & migrating database **") + base_make_migrate_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py makemigrations" + cmd = get_cmd(run_as_cmd, base_make_migrate_cmd) + ctx.run(cmd) + + base_migrate_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py migrate" + cmd = get_cmd(run_as_cmd, base_migrate_cmd) + ctx.run(cmd) + print("** Done! Migrations complete **") + + if fixtures: + base_fixture_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py loaddata demo_fixtures.json" + cmd = get_cmd(run_as_cmd, base_fixture_cmd) + print("** Loading fixtures into database **") + ctx.run(cmd) + print("** Done! Loaded fixtures into database **") + + base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" + cmd = get_cmd(run_as_cmd, base_static_cmd) + print ("** Setting up static assets **") + ctx.run(cmd) + print ("** Done! Set up static assets **") + +@task +def createsuperuser(ctx): + run_as_cmd = run_as(OS_NAME) + + base_su_cmd = "docker exec -it yaksh_django python3 manage.py createsuperuser" + cmd = get_cmd(run_as_cmd, base_su_cmd) + print ("** Creating Superuser **") + ctx.run(cmd) + print ("** Done! Created Superuser **") + + base_mod_cmd = "docker exec -it yaksh_django python3 manage.py add_group" + cmd = get_cmd(run_as_cmd, base_mod_cmd) + print ("** Creating Moderator group **") + ctx.run(cmd) + print ("** Done! Created Moderator group **") + +@task +def halt(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose stop" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Stopping containers **") + ctx.run(cmd) + print ("** Done! Stopped containers **") + +@task(halt) +def remove(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose rm --force" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Removing containers **") + ctx.run(cmd) + print ("** Done! Removed containers **") -- cgit From 268371d956dbe5cf349f1f6d8b3e4594a5c8c479 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 11 Dec 2017 12:14:33 +0530 Subject: Multiple changes: - Remove requirement files placed in docker/Files - Modify tasks to dynamically copy requirement files into docker directory - Add java installation to Docker files --- docker/Dockerfile_codeserver | 8 +++++++- docker/Files/requirements-codeserver.txt | 6 ------ docker/Files/requirements-common.txt | 9 --------- docker/Files/requirements-py2.txt | 2 -- docker/Files/requirements-py3.txt | 2 -- online_test/.env.sample | 2 +- tasks.py | 6 ++++++ 7 files changed, 14 insertions(+), 21 deletions(-) delete mode 100644 docker/Files/requirements-codeserver.txt delete mode 100644 docker/Files/requirements-common.txt delete mode 100644 docker/Files/requirements-py2.txt delete mode 100644 docker/Files/requirements-py3.txt diff --git a/docker/Dockerfile_codeserver b/docker/Dockerfile_codeserver index 1bc93c2..d1cb1cc 100644 --- a/docker/Dockerfile_codeserver +++ b/docker/Dockerfile_codeserver @@ -2,7 +2,13 @@ FROM ubuntu:16.04 MAINTAINER FOSSEE -RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y +RUN apt-get update && \ +apt-get install -y software-properties-common && \ +echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ +add-apt-repository -y ppa:webupd8team/java && \ +apt-get update && \ +apt-get install -y oracle-java8-installer && \ +apt-get install git python3-pip vim libmysqlclient-dev sudo -y VOLUME /Sites/online_test diff --git a/docker/Files/requirements-codeserver.txt b/docker/Files/requirements-codeserver.txt deleted file mode 100644 index e9585fa..0000000 --- a/docker/Files/requirements-codeserver.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest -python-decouple -six -requests -tornado -psutil diff --git a/docker/Files/requirements-common.txt b/docker/Files/requirements-common.txt deleted file mode 100644 index b4d2e5b..0000000 --- a/docker/Files/requirements-common.txt +++ /dev/null @@ -1,9 +0,0 @@ --r requirements-codeserver.txt -invoke==0.21.0 -django==1.9.5 -django-taggit==0.18.1 -pytz==2016.4 -python-social-auth==0.2.19 -selenium==2.53.6 -coverage -ruamel.yaml==0.15.23 \ No newline at end of file diff --git a/docker/Files/requirements-py2.txt b/docker/Files/requirements-py2.txt deleted file mode 100644 index 38777a1..0000000 --- a/docker/Files/requirements-py2.txt +++ /dev/null @@ -1,2 +0,0 @@ --r requirements-common.txt -mysql-python==1.2.5 diff --git a/docker/Files/requirements-py3.txt b/docker/Files/requirements-py3.txt deleted file mode 100644 index 3d13335..0000000 --- a/docker/Files/requirements-py3.txt +++ /dev/null @@ -1,2 +0,0 @@ --r requirements-common.txt -mysqlclient==1.3.9 diff --git a/online_test/.env.sample b/online_test/.env.sample index 908ec72..14fcf89 100644 --- a/online_test/.env.sample +++ b/online_test/.env.sample @@ -1,7 +1,7 @@ # Django settings SECRET_KEY=dUmMy_s3cR3t_k3y -DB_ENGINE=sqlite3 +#DB_ENGINE=mysql #DB_NAME=name_of_db #DB_USER=name_of_db_user #DB_PASSWORD=db_password diff --git a/tasks.py b/tasks.py index d518374..6cbdecc 100644 --- a/tasks.py +++ b/tasks.py @@ -154,10 +154,16 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): else: print("** Docker container <{0}> not found **".format(container)) +# Docker compose based deployment @task def build(ctx): run_as_cmd = run_as(OS_NAME) + copy_tree( + os.path.join(SCRIPT_DIR, 'requirements'), + os.path.join(SCRIPT_DIR, 'docker', 'Files') + ) + base_build_cmd = "docker-compose build --no-cache" cmd = get_cmd(run_as_cmd, base_build_cmd) print ("** Building docker images **") -- cgit From e18e49376ce867720151adfc77ae5846917bbc87 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 12 Dec 2017 15:28:00 +0530 Subject: Fix bug to allow tests to run successfully --- yaksh/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yaksh/models.py b/yaksh/models.py index bc0b4f1..e733fc0 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1705,7 +1705,7 @@ class AnswerPaper(models.Model): elif question.type == 'code' or question.type == "upload": user_dir = self.user.profile.get_user_dir() - url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT) + url = '{0}:{1}'.format(SERVER_HOST_NAME, server_port) submit(url, uid, json_data, user_dir) result = {'uid': uid, 'status': 'running'} return result -- cgit From 37600557c3d098916ee6f7808d7ef3c96163d780 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Dec 2017 15:43:34 +0530 Subject: Modify README_production and reformat as RST --- README_production.md | 228 ------------------------------------------ README_production.rst | 271 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 228 deletions(-) delete mode 100644 README_production.md create mode 100644 README_production.rst diff --git a/README_production.md b/README_production.md deleted file mode 100644 index 1251ce0..0000000 --- a/README_production.md +++ /dev/null @@ -1,228 +0,0 @@ -Production Deployment -====================== - -This README provides documentation to help deploy Yaksh in a production environment. If you wish to take Yaksh on a trial run, here is a [Quickstart Guide] (https://github.com/FOSSEE/online_test/blob/master/README.md) - -#### Pre-Requisite - - 1. Ensure [pip](https://pip.pypa.io/en/latest/installing.html) is installed - 1. Install dependencies using - pip install -r requirements.txt - 1. Install MySql Server - 1. Install Python MySql support - 1. Install Apache Server for deployment - -#### Configure MySql server - - 1. Create a database named ``yaksh`` by following the steps below - - $> mysql -u root -p - mysql> create database yaksh - - 1. Add a user named ```yaksh_user``` and give access to it on the database ```yaksh``` by following the steps below - - 1. mysql> grant usage on yaksh.* to yaksh_user@localhost identified by 'mysecretpassword'; - - 1. mysql> grant all privileges on yaksh.* to yaksh_user@localhost; - - 1. Add `DATABASE_PASSWORD = 'mysecretpassword'` and `DATABASE_USER = 'yaksh_user'` to online_test/settings.py - -To deploy this app follow the steps below: - - 1. Clone this repository and cd to the cloned repo. - $ git clone https://github.com/FOSSEE/online_test.git - - 1. Run: - python manage.py syncdb - - 1. Add questions by editing the "docs/sample_questions.py" or any other file in the same format and then run the following: - - python manage.py load_exam docs/sample_questions.py - - Note that you can supply multiple Python files as arguments and all of - those will be added to the database. - - 1. First run the python server provided. This ensures that the code is executed in a safe environment. Do this like so: - - $ sudo python yaksh/code_server.py - - Put this in the background once it has started since this will not - return back the prompt. It is important that the server be running - *before* students start attempting the exam. Using sudo is - necessary since the server is run as the user "nobody". This runs - on the ports configured in the settings.py file in the variable - "SERVER_PORTS". The "SERVER_TIMEOUT" also can be changed there. - This is the maximum time allowed to execute the submitted code. - Note that this will likely spawn multiple processes as "nobody" - depending on the number of server ports specified. - - 1. The ```wsgi.py``` script should make it - easy to deploy this using mod_wsgi. You will need to add a line of the form: - - WSGIScriptAlias / "/online_test/wsgi.py" - - to your apache.conf. For more details see the Django docs here: - - https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ - - 1. Go to http://desired_host_or_ip:desired_port/admin - - 1. Login with your credentials and look at the questions and modify if - needed. Create a new Quiz, set the date and duration or - activate/deactivate the quiz. - - 1. Now ask users to login at: - - http://host:port/exam - - And you should be all set. - - 1. Note that the directory "output" will contain directories, one for each - user. Users can potentially write output into these that can be used - for checking later. - - 1. As Moderator user you can visit http://host/exam/monitor to view - results and user data interactively. You could also "grade" the - papers manually if needed. - - 1. You may dump the results and user data using the results2csv and - dump_user_data commands. - - 1. The file docs/sample_questions.py is a template that you can use for your own questions. - - 1. Sometimes you might be in the situation where you are not hosted as - "host.org/exam/" but as "host.org/foo/exam/" for whatever reason. In - this case edit "settings.py" and set the "URL_ROOT" to the root you - have to serve at. In the above example for "host.org/foo/exam" set - URL_ROOT='/foo'. - -#### Installation & Usage - -To install this app follow the steps below: - - 1. Clone this repository and cd to the cloned repo. - ```$ git clone https://github.com/FOSSEE/online_test.git``` - - 1. Run: - - python manage.py syncdb - - 1. Add questions by editing the "docs/sample_questions.py" or any other file in the same format and then run the following: - - python manage.py load_exam docs/sample_questions.py - - Note that you can supply multiple Python files as arguments and all of - those will be added to the database. - - 1. First run the python server provided. This ensures that the code is executed in a safe environment. Do this like so: - - $ sudo python yaksh/code_server.py - - Put this in the background once it has started since this will not - return back the prompt. It is important that the server be running - *before* students start attempting the exam. Using sudo is - necessary since the server is run as the user "nobody". This runs - on the ports configured in the settings.py file in the variable - "SERVER_PORTS". The "SERVER_TIMEOUT" also can be changed there. - This is the maximum time allowed to execute the submitted code. - Note that this will likely spawn multiple processes as "nobody" - depending on the number of server ports specified. - - You can also use a Dockerized code server (see below) - - 1. Now, Run: - - python manage.py runserver : - - 1. Go to http://desired_host_or_ip:desired_port/admin - - 1. Login with your credentials and look at the questions and modify if - needed. Create a new Quiz, set the date and duration or - activate/deactivate the quiz. - - 1. Now ask users to login at: - - http://host:port/exam - - And you should be all set. - - 1. Note that the directory "output" will contain directories, one for each - user. Users can potentially write output into these that can be used - for checking later. - - 1. As admin user you can visit http://host/exam/monitor to view - results and user data interactively. You could also "grade" the - papers manually if needed. - - 1. You may dump the results and user data using the results2csv and - dump_user_data commands. - - 1. The file docs/sample_questions.py is a template that you can use for your own questions. - - 1. Sometimes you might be in the situation where you are not hosted as - "host.org/exam/" but as "host.org/foo/exam/" for whatever reason. In - this case edit "settings.py" and set the "URL_ROOT" to the root you - have to serve at. In the above example for "host.org/foo/exam" set - URL_ROOT='/foo'. - -#### Using Dockerized Code Server - - 1. Install [Docker](https://github.com/FOSSEE/online_test/blob/master/README.md) - - 1. Go to the directory where the project is located - cd /path/to/online_test - - 1. Create a docker image. This may take a few minutes - docker build -t yaksh_code_server . - - 1. Check if the image has been created using the output of, - docker images - - 1. Run the invoke script using the command ```invoke start``` - The command will create and run a new docker container (that is running the code_server.py within it), it will also bind the ports of the host with those of the container - -#### Deploying Multiple Dockers - - 1. Install [Docker](https://github.com/FOSSEE/online_test/blob/master/README.md) - - 1. Go to the ```docker``` directory - - 1. Build the docker images - invoke build - - 1. Run the containers and scripts necessary to deploy the web application - invoke deploy - - Use ```invoke deploy --fixtures``` to load the fixtures - - 1. Create the superuser and moderator group - invoke createsuperuser - - 1. Stop the containers - invoke halt - - 1. Remove the containers - invoke clean - -#### Additional commands available - -We provide several convenient commands for you to use: - - - load_exam : load questions and a quiz from a python file. See - docs/sample_questions.py - - - load_questions_xml : load questions from XML file, see - docs/sample_questions.xml use of this is deprecated in favor of - load_exam. - - - results2csv : Dump the quiz results into a CSV file for further - processing. - - - dump_user_data : Dump out relevalt user data for either all users or - specified users. - -For more information on these do this: - - $ python manage.py help [command] - -where [command] is one of the above. diff --git a/README_production.rst b/README_production.rst new file mode 100644 index 0000000..dec2034 --- /dev/null +++ b/README_production.rst @@ -0,0 +1,271 @@ +Production Deployment +===================== + +This README provides documentation to help deploy Yaksh in a production +environment. If you wish to take Yaksh on a trial run, here is a +[Quickstart Guide] +(https://github.com/FOSSEE/online\_test/blob/master/README.md) + +Pre-Requisite +^^^^^^^^^^^^^ + +1. Ensure `pip `__ is + installed +2. Install dependencies using pip install -r requirements.txt +3. Install MySql Server +4. Install Python MySql support +5. Install Apache Server for deployment + +Configure MySql server +^^^^^^^^^^^^^^^^^^^^^^ + +1. Create a database named ``yaksh`` by following the steps below + + :: + + $> mysql -u root -p + $> mysql> create database yaksh + +2. Add a user named ``yaksh_user`` and give access to it on the database + ``yaksh`` by following the steps below + + :: + + mysql> grant usage on yaksh to yaksh_user@localhost identified + by 'mysecretpassword'; + + mysql> grant all privileges on yaksh to yaksh_user@localhost; + +3. Add ``DATABASE_PASSWORD = 'mysecretpassword'`` and + ``DATABASE_USER = 'yaksh_user'`` to online\_test/settings.py + +To deploy this app follow the steps below: + +1. Clone this repository and cd to the cloned repo. + + :: + + $ git clone https://github.com/FOSSEE/online\_test.git + +2. Run: + + :: + + python manage.py makemigrations yaksh + + python manage.py migrate yaksh + +4. First run the python server provided. This ensures that the code is + executed in a safe environment. Do this like so: + + :: + + $ sudo python yaksh/code_server.py + + Put this in the background once it has started since this will not + return back the prompt. It is important that the server be running + *before* students start attempting the exam. Using sudo is necessary + since the server is run as the user "nobody". This runs on the ports + configured in the settings.py file in the variable "SERVER\_PORTS". + The "SERVER\_TIMEOUT" also can be changed there. This is the maximum + time allowed to execute the submitted code. Note that this will + likely spawn multiple processes as "nobody" depending on the number + of server ports specified. + +5. The ``wsgi.py`` script should make it easy to deploy this using + mod\_wsgi. You will need to add a line of the form: + + :: + + WSGIScriptAlias / "/online_test/wsgi.py" + + to your apache.conf. For more details see the Django docs here: + + https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ + +6. Go to http://desired\_host\_or\_ip:desired\_port/admin + +7. Login with your credentials and look at the questions and modify if + needed. Create a new Quiz, set the date and duration or + activate/deactivate the quiz. + +8. Now ask users to login at: + + :: + + http://host:port/exam + + And you should be all set. + +9. Note that the directory "output" will contain directories, one for + each user. Users can potentially write output into these that can be + used for checking later. + +10. As Moderator user you can visit http://host/exam/monitor to view + results and user data interactively. You could also "grade" the + papers manually if needed. + +11. You may dump the results and user data using the results2csv and + dump\_user\_data commands. + +12. The file docs/sample\_questions.py is a template that you can use + for your own questions. + +13. Sometimes you might be in the situation where you are not hosted as + "host.org/exam/" but as "host.org/foo/exam/" for whatever reason. In + this case edit "settings.py" and set the "URL\_ROOT" to the root you + have to serve at. In the above example for "host.org/foo/exam" set + URL\_ROOT='/foo'. + +Installation & Usage +^^^^^^^^^^^^^^^^^^^^ + +To install this app follow the steps below: + +1. Clone this repository and cd to the cloned repo. + ``$ git clone https://github.com/FOSSEE/online_test.git`` + +2. Run: + + :: + + $ python manage.py makemigrations yaksh + + $ python manage.py migrate yaksh + +3. Run the python server provided. This ensures that the code is + executed in a safe environment. Do this like so: + + :: + + $ sudo python -m yaksh.code_server # For Python 2.x + + + $ sudo python3 -m yaksh.code_server # For Python 3.x + + Put this in the background once it has started since this will not + return back the prompt. It is important that the server be running + *before* students start attempting the exam. Using sudo is necessary + since the server is run as the user "nobody". This runs the number + ports configured in the settings.py file in the variable + "N\_CODE\_SERVERS". The "SERVER\_TIMEOUT" also can be changed there. + This is the maximum time allowed to execute the submitted code. Note + that this will likely spawn multiple processes as "nobody" depending + on the number of server ports specified. + + You can also use a Dockerized code server (see below) + +4. Now, Run: + + :: + + python manage.py runserver : + +5. Create a Superuser/Administrator: + + :: + + python manage.py createsuperuser + +6. Go to http://desired\_host\_or\_ip:desired\_port/exam + + And you should be all set. + +7. Note that the directory "output" will contain directories, one for + each user. Users can potentially write output into these that can be + used for checking later. + +8. As admin user you can visit http://desired\_host\_or\_ip/exam/monitor to view results + and user data interactively. You could also "grade" the papers + manually if needed. + +Using Dockerized Code Server +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Install + `Docker `__ + +2. Go to the directory where the project is located cd + /path/to/online\_test + +3. Create a docker image. This may take a few minutes docker build -t + yaksh\_code\_server ./docker/Dockerfile\_codeserver + +4. Check if the image has been created using the output of, docker + images + +5. Run the invoke script using the command ``invoke start`` The command + will create and run a new docker container (that is running the + code\_server.py within it), it will also bind the ports of the host + with those of the container + +Deploying Multiple Dockers +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Install + `Docker `__ + +2. Go to the directory where the project is located at: + + :: + + cd /path/to/online_test + +3. Build the docker images + + :: + + invoke build + +4. Run the containers and scripts necessary to deploy the web + application + + :: + + invoke deploy + +5. Use ``invoke deploy --fixtures`` to load the fixtures + +6. Create the superuser and moderator group + :: + + invoke createsuperuser + +7. Stop the containers + + :: + + invoke halt + +8. Remove the containers + + :: + + invoke clean + + +Additional commands available +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We provide several convenient commands for you to use: + +- load\_exam : load questions and a quiz from a python file. See + docs/sample\_questions.py + +- load\_questions\_xml : load questions from XML file, see + docs/sample\_questions.xml use of this is deprecated in favor of + load\_exam. + +- results2csv : Dump the quiz results into a CSV file for further + processing. + +- dump\_user\_data : Dump out relevalt user data for either all users + or specified users. + +For more information on these do this: + +:: + + $ python manage.py help [command] + +where [command] is one of the above. -- cgit From 692c452323d39665191b3934f712e774258e31a1 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Dec 2017 16:32:08 +0530 Subject: Multiple changes: - Modify JDK & JRE installation command - Remove vim installation command --- docker/Dockerfile_codeserver | 2 +- docker/Dockerfile_django | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile_codeserver b/docker/Dockerfile_codeserver index d1cb1cc..0112c88 100644 --- a/docker/Dockerfile_codeserver +++ b/docker/Dockerfile_codeserver @@ -8,7 +8,7 @@ echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | de add-apt-repository -y ppa:webupd8team/java && \ apt-get update && \ apt-get install -y oracle-java8-installer && \ -apt-get install git python3-pip vim libmysqlclient-dev sudo -y +apt-get install git python3-pip libmysqlclient-dev sudo default-jre default-jdk -y VOLUME /Sites/online_test diff --git a/docker/Dockerfile_django b/docker/Dockerfile_django index 6b87afb..daddc45 100644 --- a/docker/Dockerfile_django +++ b/docker/Dockerfile_django @@ -4,7 +4,7 @@ MAINTAINER FOSSEE RUN apt-get update -y && apt-get install git python3-pip vim libmysqlclient-dev sudo -y -RUN apt-get install apache2 vim libapache2-mod-wsgi-py3 python3-django -y && mkdir -p /Sites/online_test +RUN apt-get install apache2 libapache2-mod-wsgi-py3 python3-django -y && mkdir -p /Sites/online_test VOLUME /Sites/online_test -- cgit From 280216008e6906c8418e29610e3b5b593908e915 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Dec 2017 16:36:01 +0530 Subject: Remove makefile --- docker/Makefile | 61 --------------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 docker/Makefile diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index 9119601..0000000 --- a/docker/Makefile +++ /dev/null @@ -1,61 +0,0 @@ -help: - @echo "Docker Compose Help for Deploying Yaksh interface" - @echo "-----------------------\n" - @echo "Clone yaksh online interface from github :" - @echo " make clone" - @echo "" - @echo "Build docker images:" - @echo " make build" - @echo "" - @echo "To run containers:" - @echo " make start" - @echo "" - @echo "You need to create super a user to work with yaksh:" - @echo " make createsuperuser" - @echo "" - @echo "Now Your interface is ready. You can access it using browser just go to http://localhost:8000" - @echo "" - @echo "Really, really start over:" - @echo " make clean" - @echo "" - @echo "other utilities:restart,tail,status" - -begin: migrate fixtures start - -clone: - @/bin/bash clone-yaksh.sh - -start: - @docker-compose up -d - -stop: - @docker-compose stop - -status: - @docker-compose ps - -restart: stop start - -clean: stop - @docker-compose rm --force - -build: - @docker-compose build --no-cache - @docker pull mariadb:10.2 - -createsuperuser: migrate superuser - -migrate: - @docker exec -it yaksh_django python3 manage.py makemigrations - @docker exec -it yaksh_django python3 manage.py migrate - -superuser: - @docker exec -it yaksh_django python3 manage.py createsuperuser - @docker exec -it yaksh_django python3 manage.py add_group - @docker exec -it yaksh_django python3 manage.py collectstatic - @echo "You can access it using browser, just go to http://localhost:8000" - -tail: - @docker-compose logs -f - -.PHONY: start stop status restart clean build migrate tail clone superuser createsuperuser -- cgit From 260e36f6ef7ad9c208501ab8948a9579992d7a02 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Dec 2017 16:46:25 +0530 Subject: Change as per review suggestions --- tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index 6cbdecc..8c78838 100644 --- a/tasks.py +++ b/tasks.py @@ -27,9 +27,9 @@ def remove_check_file(path): os.remove(path) def run_as(os_name): - if os_name.startswith('linux') or os_name == 'darwin': + if os_name.startswith('linux') or os_name == 'darwin' or os_name.startswith('freebsd'): return 'sudo' - elif os_name.startswith('Win32'): + else: # For os_name = 'Win32' return None def get_cmd(run_as_cmd, base_cmd): @@ -225,7 +225,7 @@ def createsuperuser(ctx): ctx.run(cmd) print ("** Done! Created Superuser **") - base_mod_cmd = "docker exec -it yaksh_django python3 manage.py add_group" + base_mod_cmd = "docker exec -i yaksh_django python3 manage.py add_group" cmd = get_cmd(run_as_cmd, base_mod_cmd) print ("** Creating Moderator group **") ctx.run(cmd) -- cgit From 04eeb95a8f218944632dee0547babdeef2e42c53 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Wed, 27 Dec 2017 14:10:20 +0530 Subject: Modify README for production --- README_production.rst | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/README_production.rst b/README_production.rst index dec2034..02b2008 100644 --- a/README_production.rst +++ b/README_production.rst @@ -6,6 +6,12 @@ environment. If you wish to take Yaksh on a trial run, here is a [Quickstart Guide] (https://github.com/FOSSEE/online\_test/blob/master/README.md) +################### +Deploying Locally +################### + +Follow these steps to deploy locally on the server. For deployment instructions using Docker see :ref:`deploying-multiple-dockers`. + Pre-Requisite ^^^^^^^^^^^^^ @@ -16,9 +22,6 @@ Pre-Requisite 4. Install Python MySql support 5. Install Apache Server for deployment -Configure MySql server -^^^^^^^^^^^^^^^^^^^^^^ - 1. Create a database named ``yaksh`` by following the steps below :: @@ -153,7 +156,7 @@ To install this app follow the steps below: that this will likely spawn multiple processes as "nobody" depending on the number of server ports specified. - You can also use a Dockerized code server (see below) + You can also use a Dockerized code server, see :ref:`dockerized-code-server`. 4. Now, Run: @@ -179,6 +182,8 @@ To install this app follow the steps below: and user data interactively. You could also "grade" the papers manually if needed. +.. _dockerized-code-server: + Using Dockerized Code Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -199,11 +204,17 @@ Using Dockerized Code Server code\_server.py within it), it will also bind the ports of the host with those of the container +.. _deploying-multiple-dockers: + +###################################### Deploying Multiple Dockers -^^^^^^^^^^^^^^^^^^^^^^^^^^ +###################################### -1. Install - `Docker `__ +Follow these steps to deploy and run the Django Server, MySQL instance and Code Server in seperate Docker instances. + +1. Install `Docker `__ + +2. Install `Docker Compose `__ 2. Go to the directory where the project is located at: @@ -244,8 +255,11 @@ Deploying Multiple Dockers invoke clean +.. _add-commands: + +###################################### Additional commands available -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +###################################### We provide several convenient commands for you to use: -- cgit From ca9e214a28dd0546855863041efe7e6d8de6710e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 28 Dec 2017 14:48:10 +0530 Subject: - Modify code server docker file to remove Oracle JDK, JRE - Add decorator to tasks --- README_production.rst | 14 ++++++++++---- docker/Dockerfile_codeserver | 5 ----- tasks.py | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README_production.rst b/README_production.rst index 02b2008..28b756e 100644 --- a/README_production.rst +++ b/README_production.rst @@ -63,7 +63,9 @@ To deploy this app follow the steps below: :: - $ sudo python yaksh/code_server.py + $ sudo python -m yaksh.code_server # For Python 2 + + $ sudo python3 -m yaksh.code_server # For Python 3 Put this in the background once it has started since this will not return back the prompt. It is important that the server be running @@ -194,7 +196,7 @@ Using Dockerized Code Server /path/to/online\_test 3. Create a docker image. This may take a few minutes docker build -t - yaksh\_code\_server ./docker/Dockerfile\_codeserver + yaksh\_code\_server -f ./docker/Dockerfile\_codeserver 4. Check if the image has been created using the output of, docker images @@ -204,6 +206,8 @@ Using Dockerized Code Server code\_server.py within it), it will also bind the ports of the host with those of the container +6. You can use ``invoke --list`` to get a list of all the available commands + .. _deploying-multiple-dockers: ###################################### @@ -216,11 +220,11 @@ Follow these steps to deploy and run the Django Server, MySQL instance and Code 2. Install `Docker Compose `__ -2. Go to the directory where the project is located at: +2. Go to the ``docker`` directory where the project is located: :: - cd /path/to/online_test + cd /path/to/online_test/docker 3. Build the docker images @@ -254,6 +258,8 @@ Follow these steps to deploy and run the Django Server, MySQL instance and Code invoke clean +9. You can use ``invoke --list`` to get a list of all the available commands + .. _add-commands: diff --git a/docker/Dockerfile_codeserver b/docker/Dockerfile_codeserver index 0112c88..955e681 100644 --- a/docker/Dockerfile_codeserver +++ b/docker/Dockerfile_codeserver @@ -3,11 +3,6 @@ FROM ubuntu:16.04 MAINTAINER FOSSEE RUN apt-get update && \ -apt-get install -y software-properties-common && \ -echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ -add-apt-repository -y ppa:webupd8team/java && \ -apt-get update && \ -apt-get install -y oracle-java8-installer && \ apt-get install git python3-pip libmysqlclient-dev sudo default-jre default-jdk -y VOLUME /Sites/online_test diff --git a/tasks.py b/tasks.py index 8c78838..ac2721a 100644 --- a/tasks.py +++ b/tasks.py @@ -45,6 +45,7 @@ def setupdb(ctx): ctx.run("python manage.py migrate") print("** Done! Migrations complete **") +@task def loadfixtures(ctx): print("** Loading fixtures into database **") ctx.run("python manage.py loaddata demo_fixtures.json") -- cgit From 9a67cce12061fe63b526fdfbe03a03f6e859a044 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 28 Dec 2017 19:25:51 +0530 Subject: Add docker related changes --- docker/Dockerfile_codeserver | 2 +- docker/Files/Docker-script.sh | 6 ++++-- docker/Files/Start-codeserver.sh | 7 +++---- docker/docker-compose.yml | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile_codeserver b/docker/Dockerfile_codeserver index 955e681..1f030d5 100644 --- a/docker/Dockerfile_codeserver +++ b/docker/Dockerfile_codeserver @@ -11,7 +11,7 @@ ADD Files/requirements-* /tmp/ RUN pip3 install -r /tmp/requirements-codeserver.txt && mkdir -p /Sites/online_test/yaksh_data/output /Sites/online_test/yaksh_data/data -WORKDIR /Sites/online_test/yaksh_data +WORKDIR /Sites/online_test ADD Files/Start-codeserver.sh /Sites diff --git a/docker/Files/Docker-script.sh b/docker/Files/Docker-script.sh index 752a638..dc35a3f 100644 --- a/docker/Files/Docker-script.sh +++ b/docker/Files/Docker-script.sh @@ -1,5 +1,7 @@ #!/bin/bash -chown -R www-data /Sites/online_test -chmod -R 644 /Sites/online_test +chown -R www-data /Sites/online_test +chown -R www-data /Sites/online_test/yaksh +chown -R nobody /Sites/online_test/yaksh_data +chmod -R 664 /Sites/online_test chmod -R +X /Sites /usr/sbin/apache2ctl -D FOREGROUND diff --git a/docker/Files/Start-codeserver.sh b/docker/Files/Start-codeserver.sh index 8b6ea96..019a31d 100644 --- a/docker/Files/Start-codeserver.sh +++ b/docker/Files/Start-codeserver.sh @@ -1,7 +1,6 @@ #!/bin/bash -chown -R nobody /Sites/online_test +chown -R nobody /Sites/online_test/yaksh_data chmod -R a+rwX yaksh_data/output -chmod -R a+rX yaksh_data/data yaksh_data/yaksh -chmod -R o-w yaksh_data/data yaksh -chmod -R +X /Sites/online_test/ +chmod -R a+rX yaksh_data/data +chmod -R o-w yaksh_data/data /usr/bin/sudo -su nobody python3 -m yaksh.code_server diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 747cbda..7ee3860 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,7 +26,7 @@ services: image: yaksh.codeserver container_name: yaksh_codeserver volumes: - - ../yaksh_data:/Sites/online_test/yaksh_data + - ..:/Sites/online_test restart: always -- cgit From c873adbf3bf94564dbc64ff3bced722dabc6fdc3 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 29 Dec 2017 16:49:21 +0530 Subject: Change README and tasks.py --- README_production.rst | 14 ++++++++------ tasks.py | 16 ---------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/README_production.rst b/README_production.rst index 28b756e..1438dcd 100644 --- a/README_production.rst +++ b/README_production.rst @@ -17,7 +17,14 @@ Pre-Requisite 1. Ensure `pip `__ is installed -2. Install dependencies using pip install -r requirements.txt +2. Install dependencies, Run; + + :: + + pip install -r requirements/requirements-py2.txt # For Python 2 + + pip3 install -r requirements/requirements-py3.txt # For Python 3 + 3. Install MySql Server 4. Install Python MySql support 5. Install Apache Server for deployment @@ -241,11 +248,6 @@ Follow these steps to deploy and run the Django Server, MySQL instance and Code 5. Use ``invoke deploy --fixtures`` to load the fixtures -6. Create the superuser and moderator group - :: - - invoke createsuperuser - 7. Stop the containers :: diff --git a/tasks.py b/tasks.py index ac2721a..5e7295a 100644 --- a/tasks.py +++ b/tasks.py @@ -216,22 +216,6 @@ def deploy(ctx, fixtures=False): ctx.run(cmd) print ("** Done! Set up static assets **") -@task -def createsuperuser(ctx): - run_as_cmd = run_as(OS_NAME) - - base_su_cmd = "docker exec -it yaksh_django python3 manage.py createsuperuser" - cmd = get_cmd(run_as_cmd, base_su_cmd) - print ("** Creating Superuser **") - ctx.run(cmd) - print ("** Done! Created Superuser **") - - base_mod_cmd = "docker exec -i yaksh_django python3 manage.py add_group" - cmd = get_cmd(run_as_cmd, base_mod_cmd) - print ("** Creating Moderator group **") - ctx.run(cmd) - print ("** Done! Created Moderator group **") - @task def halt(ctx): run_as_cmd = run_as(OS_NAME) -- cgit From f5bc35996373251d718aa83b3abe1241efc41922 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 29 Dec 2017 16:56:53 +0530 Subject: Add sample environment file --- .sampleenv | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .sampleenv diff --git a/.sampleenv b/.sampleenv new file mode 100644 index 0000000..a31ec1f --- /dev/null +++ b/.sampleenv @@ -0,0 +1,13 @@ +# Django settings +SECRET_KEY=dUmMy_s3cR3t_k3y +#DB_ENGINE=mysql +#DB_NAME=yaksh +#DB_USER=root +#DB_PASSWORD=root +#DB_HOST=yaksh-db +#DB_PORT=3306 +# Yaksh settings +N_CODE_SERVERS=5 +#SERVER_POOL_PORT=53579 +#SERVER_HOST_NAME=http://yaksh-codeserver +#SERVER_TIMEOUT=4 -- cgit From 7d7cc537f908db40ca6f3b10cc2c10d597b2877a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 2 Jan 2018 14:31:09 +0530 Subject: Add changes as requested in review --- README_production.rst | 151 +++++++++++++++++--------------------------------- tasks.py | 36 ++++++++++-- 2 files changed, 82 insertions(+), 105 deletions(-) diff --git a/README_production.rst b/README_production.rst index 1438dcd..29d79e5 100644 --- a/README_production.rst +++ b/README_production.rst @@ -29,14 +29,14 @@ Pre-Requisite 4. Install Python MySql support 5. Install Apache Server for deployment -1. Create a database named ``yaksh`` by following the steps below +6. Create a database named ``yaksh`` by following the steps below :: $> mysql -u root -p $> mysql> create database yaksh -2. Add a user named ``yaksh_user`` and give access to it on the database +7. Add a user named ``yaksh_user`` and give access to it on the database ``yaksh`` by following the steps below :: @@ -46,88 +46,9 @@ Pre-Requisite mysql> grant all privileges on yaksh to yaksh_user@localhost; -3. Add ``DATABASE_PASSWORD = 'mysecretpassword'`` and +8. Add ``DATABASE_PASSWORD = 'mysecretpassword'`` and ``DATABASE_USER = 'yaksh_user'`` to online\_test/settings.py -To deploy this app follow the steps below: - -1. Clone this repository and cd to the cloned repo. - - :: - - $ git clone https://github.com/FOSSEE/online\_test.git - -2. Run: - - :: - - python manage.py makemigrations yaksh - - python manage.py migrate yaksh - -4. First run the python server provided. This ensures that the code is - executed in a safe environment. Do this like so: - - :: - - $ sudo python -m yaksh.code_server # For Python 2 - - $ sudo python3 -m yaksh.code_server # For Python 3 - - Put this in the background once it has started since this will not - return back the prompt. It is important that the server be running - *before* students start attempting the exam. Using sudo is necessary - since the server is run as the user "nobody". This runs on the ports - configured in the settings.py file in the variable "SERVER\_PORTS". - The "SERVER\_TIMEOUT" also can be changed there. This is the maximum - time allowed to execute the submitted code. Note that this will - likely spawn multiple processes as "nobody" depending on the number - of server ports specified. - -5. The ``wsgi.py`` script should make it easy to deploy this using - mod\_wsgi. You will need to add a line of the form: - - :: - - WSGIScriptAlias / "/online_test/wsgi.py" - - to your apache.conf. For more details see the Django docs here: - - https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ - -6. Go to http://desired\_host\_or\_ip:desired\_port/admin - -7. Login with your credentials and look at the questions and modify if - needed. Create a new Quiz, set the date and duration or - activate/deactivate the quiz. - -8. Now ask users to login at: - - :: - - http://host:port/exam - - And you should be all set. - -9. Note that the directory "output" will contain directories, one for - each user. Users can potentially write output into these that can be - used for checking later. - -10. As Moderator user you can visit http://host/exam/monitor to view - results and user data interactively. You could also "grade" the - papers manually if needed. - -11. You may dump the results and user data using the results2csv and - dump\_user\_data commands. - -12. The file docs/sample\_questions.py is a template that you can use - for your own questions. - -13. Sometimes you might be in the situation where you are not hosted as - "host.org/exam/" but as "host.org/foo/exam/" for whatever reason. In - this case edit "settings.py" and set the "URL\_ROOT" to the root you - have to serve at. In the above example for "host.org/foo/exam" set - URL\_ROOT='/foo'. Installation & Usage ^^^^^^^^^^^^^^^^^^^^ @@ -137,7 +58,19 @@ To install this app follow the steps below: 1. Clone this repository and cd to the cloned repo. ``$ git clone https://github.com/FOSSEE/online_test.git`` -2. Run: +2. Rename the ``.sampleenv`` to ``.env`` + +3. In the ``.env`` file, uncomment the following and replace the values (please keep the remaining settings as is); + + :: + + DB_ENGINE=mysql # Or psycopg (postgresql), sqlite3 (SQLite) + DB_NAME=yaksh + DB_USER=root + DB_PASSWORD=mypassword # Or the password used while creating a Database + DB_PORT=3306 + +4. Run: :: @@ -145,7 +78,7 @@ To install this app follow the steps below: $ python manage.py migrate yaksh -3. Run the python server provided. This ensures that the code is +5. Run the python server provided. This ensures that the code is executed in a safe environment. Do this like so: :: @@ -167,27 +100,32 @@ To install this app follow the steps below: You can also use a Dockerized code server, see :ref:`dockerized-code-server`. -4. Now, Run: +6. The ``wsgi.py`` script should make it easy to deploy this using + mod\_wsgi. You will need to add a line of the form: - :: + :: - python manage.py runserver : + WSGIScriptAlias / "/online_test/wsgi.py" + + to your apache.conf. For more details see the Django docs here: -5. Create a Superuser/Administrator: + https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ + +7. Create a Superuser/Administrator: :: python manage.py createsuperuser -6. Go to http://desired\_host\_or\_ip:desired\_port/exam +8. Go to http://desired\_host\_or\_ip:desired\_port/exam And you should be all set. -7. Note that the directory "output" will contain directories, one for +9. Note that the directory "output" will contain directories, one for each user. Users can potentially write output into these that can be used for checking later. -8. As admin user you can visit http://desired\_host\_or\_ip/exam/monitor to view results +10. As admin user you can visit http://desired\_host\_or\_ip/exam/monitor to view results and user data interactively. You could also "grade" the papers manually if needed. @@ -227,40 +165,55 @@ Follow these steps to deploy and run the Django Server, MySQL instance and Code 2. Install `Docker Compose `__ -2. Go to the ``docker`` directory where the project is located: +3. Rename the ``.sampleenv`` to ``.env`` + +4. In the ``.env`` file, uncomment all the values and keep the default values as is. + +5. Go to the ``docker`` directory where the project is located: :: cd /path/to/online_test/docker -3. Build the docker images +6. Build the docker images :: invoke build -4. Run the containers and scripts necessary to deploy the web +7. Run the containers and scripts necessary to deploy the web application :: - invoke deploy + invoke begin + +8. Make sure that all the containers are ``Up`` and stable + + :: + + invoke status + +8. Run the containers and scripts necessary to deploy the web + application, ``--fixtures`` allows you to load fixtures. + + :: -5. Use ``invoke deploy --fixtures`` to load the fixtures + invoke deploy --fixtures -7. Stop the containers +10. Stop the containers, you can use ``invoke restart`` to restart the containers without removing them :: invoke halt -8. Remove the containers +11. Remove the containers :: invoke clean -9. You can use ``invoke --list`` to get a list of all the available commands +12. You can use ``invoke --list`` to get a list of all the available commands .. _add-commands: diff --git a/tasks.py b/tasks.py index 5e7295a..4a29c4d 100644 --- a/tasks.py +++ b/tasks.py @@ -26,6 +26,10 @@ def remove_check_file(path): if os.path.isfile(path): os.remove(path) +def remove_dir(path): + if os.path.isdir(path): + shutil.rmtree(path) + def run_as(os_name): if os_name.startswith('linux') or os_name == 'darwin' or os_name.startswith('freebsd'): return 'sudo' @@ -186,8 +190,8 @@ def begin(ctx): ctx.run(cmd) print ("** Done! Initialized the docker containers **") -@task(begin) -def deploy(ctx, fixtures=False): +@task +def deploy(ctx, fixtures=False, static=True): run_as_cmd = run_as(OS_NAME) print("** Setting up & migrating database **") @@ -210,11 +214,20 @@ def deploy(ctx, fixtures=False): ctx.run(cmd) print("** Done! Loaded fixtures into database **") - base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" - cmd = get_cmd(run_as_cmd, base_static_cmd) - print ("** Setting up static assets **") + if static: + base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" + cmd = get_cmd(run_as_cmd, base_static_cmd) + print ("** Setting up static assets **") + ctx.run(cmd) + print ("** Done! Set up static assets **") + +@task +def status(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose ps" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Fetching container status **") ctx.run(cmd) - print ("** Done! Set up static assets **") @task def halt(ctx): @@ -225,11 +238,22 @@ def halt(ctx): ctx.run(cmd) print ("** Done! Stopped containers **") +@task +def restart(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose restart" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Restarting containers **") + ctx.run(cmd) + print ("** Done! Restarted containers **") + @task(halt) def remove(ctx): run_as_cmd = run_as(OS_NAME) base_cmd = "docker-compose rm --force" cmd = get_cmd(run_as_cmd, base_cmd) + sql_dir = os.path.join(SCRIPT_DIR, 'docker', 'mysql') print ("** Removing containers **") + remove_dir(sql_dir) ctx.run(cmd) print ("** Done! Removed containers **") -- cgit From 2846e4372bb5d255c9b4405456b133f174916893 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 2 Jan 2018 18:24:27 +0530 Subject: - Update README_production.rst to fix incorrect commands, links and formatting - Remove README inside docker folder --- README_production.rst | 40 ++++++++++++++++++++++++---------------- docker/README.md | 30 ------------------------------ online_test/.env.sample | 16 ---------------- tasks.py | 2 +- 4 files changed, 25 insertions(+), 63 deletions(-) delete mode 100644 docker/README.md delete mode 100644 online_test/.env.sample diff --git a/README_production.rst b/README_production.rst index 29d79e5..a9bd55b 100644 --- a/README_production.rst +++ b/README_production.rst @@ -3,14 +3,13 @@ Production Deployment This README provides documentation to help deploy Yaksh in a production environment. If you wish to take Yaksh on a trial run, here is a -[Quickstart Guide] -(https://github.com/FOSSEE/online\_test/blob/master/README.md) +`Quickstart Guide `__ ################### Deploying Locally ################### -Follow these steps to deploy locally on the server. For deployment instructions using Docker see :ref:`deploying-multiple-dockers`. +Follow these steps to deploy locally on the server. For deployment instructions using Docker see `Deploying Multiple Dockers `__ Pre-Requisite ^^^^^^^^^^^^^ @@ -56,7 +55,10 @@ Installation & Usage To install this app follow the steps below: 1. Clone this repository and cd to the cloned repo. - ``$ git clone https://github.com/FOSSEE/online_test.git`` + + :: + + $ git clone https://github.com/FOSSEE/online_test.git 2. Rename the ``.sampleenv`` to ``.env`` @@ -98,7 +100,8 @@ To install this app follow the steps below: that this will likely spawn multiple processes as "nobody" depending on the number of server ports specified. - You can also use a Dockerized code server, see :ref:`dockerized-code-server`. + You can also use a Dockerized code server, see `Dockerized Code Server `__ + 6. The ``wsgi.py`` script should make it easy to deploy this using mod\_wsgi. You will need to add a line of the form: @@ -109,7 +112,7 @@ To install this app follow the steps below: to your apache.conf. For more details see the Django docs here: - https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ + https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ 7. Create a Superuser/Administrator: @@ -125,9 +128,7 @@ To install this app follow the steps below: each user. Users can potentially write output into these that can be used for checking later. -10. As admin user you can visit http://desired\_host\_or\_ip/exam/monitor to view results - and user data interactively. You could also "grade" the papers - manually if needed. +10. As admin user you can visit http://desired\_host\_or\_ip/exam/monitor to view results and user data interactively. You could also "grade" the papers manually if needed. .. _dockerized-code-server: @@ -137,14 +138,20 @@ Using Dockerized Code Server 1. Install `Docker `__ -2. Go to the directory where the project is located cd - /path/to/online\_test +2. Go to the directory where the project is located + + :: -3. Create a docker image. This may take a few minutes docker build -t - yaksh\_code\_server -f ./docker/Dockerfile\_codeserver + cd /path/to/online_test -4. Check if the image has been created using the output of, docker - images +3. Create a docker image. This may take a few minutes, + + :: + + docker build -t yaksh_code_server -f ./docker/Dockerfile_codeserver + +4. Check if the image has been created using the output of ``docker + images`` 5. Run the invoke script using the command ``invoke start`` The command will create and run a new docker container (that is running the @@ -153,6 +160,7 @@ Using Dockerized Code Server 6. You can use ``invoke --list`` to get a list of all the available commands + .. _deploying-multiple-dockers: ###################################### @@ -211,7 +219,7 @@ Follow these steps to deploy and run the Django Server, MySQL instance and Code :: - invoke clean + invoke remove 12. You can use ``invoke --list`` to get a list of all the available commands diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 34e30da..0000000 --- a/docker/README.md +++ /dev/null @@ -1,30 +0,0 @@ -#### Pre-Requisites - - 1. Install Docker https://docs.docker.com/engine/installation/ - 2. Install Docker Compose https://docs.docker.com/compose/install/#install-compose - 3. install git - - -#### Help for deploying Yaksh interface. - - - To get help while deployment - $ make help - - - Clone yaksh form github: - $ make clone - - - Build docker images: - $ make build - - - To run containers: - $ make start - - - You need to create super a user to work with yaksh: - $ make createsuperuser - - - Now Your interface is ready. You can access it using browser just go to http://localhost:8000 - - - Clean your docker containers: - $ make clean - - - other utilities like restart, tail, status diff --git a/online_test/.env.sample b/online_test/.env.sample deleted file mode 100644 index 14fcf89..0000000 --- a/online_test/.env.sample +++ /dev/null @@ -1,16 +0,0 @@ -# Django settings - -SECRET_KEY=dUmMy_s3cR3t_k3y -#DB_ENGINE=mysql -#DB_NAME=name_of_db -#DB_USER=name_of_db_user -#DB_PASSWORD=db_password -#DB_HOST=db_hostname -#DB_PORT=8180 - -# Yaksh settings - -#N_CODE_SERVERS=5 -#SERVER_POOL_PORT=55555 -#SERVER_HOST_NAME=http://localhost -#SERVER_TIMEOUT=4 diff --git a/tasks.py b/tasks.py index 4a29c4d..8396723 100644 --- a/tasks.py +++ b/tasks.py @@ -63,7 +63,7 @@ def serve(ctx): @task def clean(ctx): print("** Discarding database **") - ctx.run("rm -rf {0}".format(os.path.join(SCRIPT_DIR, 'db.sqlite3'))) + remove_check_file(os.path.join(SCRIPT_DIR, 'db.sqlite3')) @task def getimage(ctx, image=SRC_IMAGE_NAME): -- cgit From bbf844c6ebf9b29c436930b5d5129ebf77a3aad1 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 4 Jan 2018 16:20:00 +0530 Subject: Replace hardcoded codeserver URL in regrade method within models --- yaksh/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yaksh/models.py b/yaksh/models.py index e733fc0..2f616cf 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -1740,7 +1740,7 @@ class AnswerPaper(models.Model): server_port=server_port ) if question.type == "code": - url = 'http://localhost:%s' % server_port + url = '{0}:{1}'.format(SERVER_HOST_NAME, server_port) check_result = get_result_from_code_server(url, result['uid'], block=True ) -- cgit