From 2c057133d1b131a1573c93f0d0dbb548dcc8223e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 17 Nov 2017 17:13:26 +0530 Subject: Add dynamic run-as-admin commands to the base command depending on OS --- tasks.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 68d9967..04a2ef5 100644 --- a/tasks.py +++ b/tasks.py @@ -9,6 +9,7 @@ TARGET_CONTAINER_NAME = 'yaksh_code_server' SRC_IMAGE_NAME = 'fossee/yaksh_codeserver' CHECK_FILE = 'server_running.txt' CHECK_FILE_PATH = os.path.join(SCRIPT_DIR, 'yaksh_data', CHECK_FILE) +OS_NAME = sys.platform def create_dir(path): @@ -19,6 +20,18 @@ def remove_check_file(path): if os.path.isfile(path): os.remove(path) +def run_as(os_name): + if os_name.startswith('linux') or os_name == 'darwin': + return 'sudo' + elif os_name.startswith('Win32'): + return None + +def get_cmd(run_as_cmd, base_cmd): + if run_as_cmd: + return '{0} {1}'.format(run_as_cmd, base_cmd) + else: + return base_cmd + @task def setupdb(ctx): print("** Setting up & migrating database **") @@ -43,7 +56,10 @@ def getimage(ctx, image=SRC_IMAGE_NAME): except invoke.exceptions.Failure: print("The docker image {0} does not exist locally".format(image)) print("\n** Pulling latest image <{0}> from docker hub **".format(image)) - ctx.run("sudo docker pull {0}".format(image)) + base_cmd = "docker pull {0}".format(image) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) print("\n** Done! Successfully pulled latest image <{0}> **".format(image)) @task @@ -52,10 +68,12 @@ def start(ctx, ports=SERVER_POOL_PORT, image=SRC_IMAGE_NAME, unsafe=False, if unsafe: with ctx.cd(SCRIPT_DIR): print("** Initializing local code server **") - ctx.run("sudo python{0} -m yaksh.code_server".format( - version - ) + base_cmd = "python{0} -m yaksh.code_server".format( + version ) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) else: cmd_params = {'ports': ports, 'image': SRC_IMAGE_NAME, @@ -104,9 +122,17 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): result = ctx.run("sudo docker ps -q --filter='name={0}'".format(container)) remove_check_file(CHECK_FILE_PATH) if result.stdout: + print ("** Stopping the docker container <{0}>".format(container)) + base_cmd = "docker stop {0}".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) + print ("** Done! Stopped the docker container <{0}>".format(container)) + print ("** Discarding the docker container <{0}>".format(container)) - ctx.run("sudo docker stop {0}".format(container)) - ctx.run("sudo docker rm {0}".format(container)) + base_cmd = "docker rm {0}".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) print ("** Done! Discarded the docker container <{0}>".format(container)) else: print("** Docker container <{0}> not found **".format(container)) -- cgit From 6f2e29a067cbeaf88b8ea478bb7c24fe8772180a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 20 Nov 2017 16:07:43 +0530 Subject: Multiple Changes: - Add python-decouple package to requirements - use sane defaults and python-decouple in yaksh.settings and online_test.settings module - Replace hardcoded localhost domain name referring to code server with dynamic - Move Dockerfile to dedicated directory - Add dynamic run-as-admin commands to the base command depending on OS - Replace linux specific commands like cp with Python functions --- tasks.py | 54 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 21 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 04a2ef5..47edb20 100644 --- a/tasks.py +++ b/tasks.py @@ -1,7 +1,13 @@ from __future__ import print_function +import os +import sys +import shutil +from distutils.dir_util import copy_tree +from distutils.file_util import copy_file + import invoke from invoke import task -import os + from yaksh.settings import SERVER_POOL_PORT SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -52,7 +58,10 @@ def clean(ctx): @task def getimage(ctx, image=SRC_IMAGE_NAME): try: - result = ctx.run("sudo docker inspect {0}".format(image), hide=True) + base_cmd = "docker inspect {0}".format(image) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd, hide=True) except invoke.exceptions.Failure: print("The docker image {0} does not exist locally".format(image)) print("\n** Pulling latest image <{0}> from docker hub **".format(image)) @@ -92,47 +101,50 @@ def start(ctx, ports=SERVER_POOL_PORT, image=SRC_IMAGE_NAME, unsafe=False, create_dir(os.path.join(SCRIPT_DIR, 'yaksh_data', 'data')) create_dir(os.path.join(SCRIPT_DIR, 'yaksh_data', 'output')) - ctx.run('cp -r {0} {1}'.format( - os.path.join(SCRIPT_DIR, 'yaksh'), - os.path.join(SCRIPT_DIR, 'yaksh_data') - ) + copy_tree( + os.path.join(SCRIPT_DIR, 'yaksh'), + os.path.join(SCRIPT_DIR, 'yaksh_data', 'yaksh') ) - ctx.run('cp {0} {1}'.format( - os.path.join(SCRIPT_DIR, 'requirements', 'requirements-codeserver.txt'), - os.path.join(SCRIPT_DIR, 'yaksh_data') - ) + + copy_file( + os.path.join(SCRIPT_DIR, 'requirements', 'requirements-codeserver.txt'), + os.path.join(SCRIPT_DIR, 'yaksh_data') ) print("** Initializing code server within docker container **") - ctx.run( - "sudo docker run \ + base_cmd = "docker run \ -dp {ports}:{ports} --name={name} \ -v {vol_mount}:{vol_mount} \ -w {vol_mount} \ {image} {command}".format(**cmd_params) - ) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) while not os.path.isfile(CHECK_FILE_PATH): print("** Checking code server status. Press Ctrl-C to exit **\r", end="") - print("** Code server is up and running successfully **") + print("\n** Code server is up and running successfully **") @task def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): - result = ctx.run("sudo docker ps -q --filter='name={0}'".format(container)) + base_filter_cmd = "docker ps -q --filter='name={0}'".format(container) + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_filter_cmd) + result = ctx.run(cmd) + remove_check_file(CHECK_FILE_PATH) if result.stdout: print ("** Stopping the docker container <{0}>".format(container)) - base_cmd = "docker stop {0}".format(container) - run_as_cmd = run_as(OS_NAME) - cmd = get_cmd(run_as_cmd, base_cmd) + base_stop_cmd = "docker stop {0}".format(container) + cmd = get_cmd(run_as_cmd, base_stop_cmd) ctx.run(cmd) print ("** Done! Stopped the docker container <{0}>".format(container)) print ("** Discarding the docker container <{0}>".format(container)) - base_cmd = "docker rm {0}".format(container) - run_as_cmd = run_as(OS_NAME) - cmd = get_cmd(run_as_cmd, base_cmd) + base_rm_cmd = "docker rm {0}".format(container) + cmd = get_cmd(run_as_cmd, base_rm_cmd) + ctx.run(cmd) print ("** Done! Discarded the docker container <{0}>".format(container)) else: print("** Docker container <{0}> not found **".format(container)) -- cgit From 8c9c7fdf227c7a107345c70fea4cf77cc550a10f Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 7 Dec 2017 16:25:52 +0530 Subject: Multiple changes: - Add deployment commands to tasks.py - Add deployment commands to README_production.md --- tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 5 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 47edb20..d518374 100644 --- a/tasks.py +++ b/tasks.py @@ -43,9 +43,14 @@ def setupdb(ctx): print("** Setting up & migrating database **") ctx.run("python manage.py makemigrations") ctx.run("python manage.py migrate") + print("** Done! Migrations complete **") + +def loadfixtures(ctx): + print("** Loading fixtures into database **") ctx.run("python manage.py loaddata demo_fixtures.json") + print("** Done! Loaded fixtures into database **") -@task(setupdb) +@task(setupdb, loadfixtures) def serve(ctx): print("** Running the Django web server. Press Ctrl-C to Exit **") ctx.run("python manage.py runserver") @@ -135,16 +140,105 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): remove_check_file(CHECK_FILE_PATH) if result.stdout: - print ("** Stopping the docker container <{0}>".format(container)) + print ("** Stopping the docker container <{0}> **".format(container)) base_stop_cmd = "docker stop {0}".format(container) cmd = get_cmd(run_as_cmd, base_stop_cmd) ctx.run(cmd) - print ("** Done! Stopped the docker container <{0}>".format(container)) + print ("** Done! Stopped the docker container <{0}> **".format(container)) - print ("** Discarding the docker container <{0}>".format(container)) + print ("** Discarding the docker container <{0}> **".format(container)) base_rm_cmd = "docker rm {0}".format(container) cmd = get_cmd(run_as_cmd, base_rm_cmd) ctx.run(cmd) - print ("** Done! Discarded the docker container <{0}>".format(container)) + print ("** Done! Discarded the docker container <{0}> **".format(container)) else: print("** Docker container <{0}> not found **".format(container)) + +@task +def build(ctx): + run_as_cmd = run_as(OS_NAME) + + base_build_cmd = "docker-compose build --no-cache" + cmd = get_cmd(run_as_cmd, base_build_cmd) + print ("** Building docker images **") + ctx.run(cmd) + print ("** Done! Built the docker images **") + + base_build_cmd = "docker pull mariadb:10.2 " + cmd = get_cmd(run_as_cmd, base_build_cmd) + print ("** Pulling maria-db base image **") + ctx.run(cmd) + print ("** Done! Pulled maria-db base image **") + +@task +def begin(ctx): + print("** Initializing docker containers **") + base_cmd = "docker-compose up -d" + run_as_cmd = run_as(OS_NAME) + cmd = get_cmd(run_as_cmd, base_cmd) + ctx.run(cmd) + print ("** Done! Initialized the docker containers **") + +@task(begin) +def deploy(ctx, fixtures=False): + run_as_cmd = run_as(OS_NAME) + + print("** Setting up & migrating database **") + base_make_migrate_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py makemigrations" + cmd = get_cmd(run_as_cmd, base_make_migrate_cmd) + ctx.run(cmd) + + base_migrate_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py migrate" + cmd = get_cmd(run_as_cmd, base_migrate_cmd) + ctx.run(cmd) + print("** Done! Migrations complete **") + + if fixtures: + base_fixture_cmd = "docker exec -i yaksh_django" \ + " python3 manage.py loaddata demo_fixtures.json" + cmd = get_cmd(run_as_cmd, base_fixture_cmd) + print("** Loading fixtures into database **") + ctx.run(cmd) + print("** Done! Loaded fixtures into database **") + + base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" + cmd = get_cmd(run_as_cmd, base_static_cmd) + print ("** Setting up static assets **") + ctx.run(cmd) + print ("** Done! Set up static assets **") + +@task +def createsuperuser(ctx): + run_as_cmd = run_as(OS_NAME) + + base_su_cmd = "docker exec -it yaksh_django python3 manage.py createsuperuser" + cmd = get_cmd(run_as_cmd, base_su_cmd) + print ("** Creating Superuser **") + ctx.run(cmd) + print ("** Done! Created Superuser **") + + base_mod_cmd = "docker exec -it yaksh_django python3 manage.py add_group" + cmd = get_cmd(run_as_cmd, base_mod_cmd) + print ("** Creating Moderator group **") + ctx.run(cmd) + print ("** Done! Created Moderator group **") + +@task +def halt(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose stop" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Stopping containers **") + ctx.run(cmd) + print ("** Done! Stopped containers **") + +@task(halt) +def remove(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose rm --force" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Removing containers **") + ctx.run(cmd) + print ("** Done! Removed containers **") -- cgit From 268371d956dbe5cf349f1f6d8b3e4594a5c8c479 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Mon, 11 Dec 2017 12:14:33 +0530 Subject: Multiple changes: - Remove requirement files placed in docker/Files - Modify tasks to dynamically copy requirement files into docker directory - Add java installation to Docker files --- tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index d518374..6cbdecc 100644 --- a/tasks.py +++ b/tasks.py @@ -154,10 +154,16 @@ def stop(ctx, container=TARGET_CONTAINER_NAME, hide=True): else: print("** Docker container <{0}> not found **".format(container)) +# Docker compose based deployment @task def build(ctx): run_as_cmd = run_as(OS_NAME) + copy_tree( + os.path.join(SCRIPT_DIR, 'requirements'), + os.path.join(SCRIPT_DIR, 'docker', 'Files') + ) + base_build_cmd = "docker-compose build --no-cache" cmd = get_cmd(run_as_cmd, base_build_cmd) print ("** Building docker images **") -- cgit From 260e36f6ef7ad9c208501ab8948a9579992d7a02 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 26 Dec 2017 16:46:25 +0530 Subject: Change as per review suggestions --- tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 6cbdecc..8c78838 100644 --- a/tasks.py +++ b/tasks.py @@ -27,9 +27,9 @@ def remove_check_file(path): os.remove(path) def run_as(os_name): - if os_name.startswith('linux') or os_name == 'darwin': + if os_name.startswith('linux') or os_name == 'darwin' or os_name.startswith('freebsd'): return 'sudo' - elif os_name.startswith('Win32'): + else: # For os_name = 'Win32' return None def get_cmd(run_as_cmd, base_cmd): @@ -225,7 +225,7 @@ def createsuperuser(ctx): ctx.run(cmd) print ("** Done! Created Superuser **") - base_mod_cmd = "docker exec -it yaksh_django python3 manage.py add_group" + base_mod_cmd = "docker exec -i yaksh_django python3 manage.py add_group" cmd = get_cmd(run_as_cmd, base_mod_cmd) print ("** Creating Moderator group **") ctx.run(cmd) -- cgit From ca9e214a28dd0546855863041efe7e6d8de6710e Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Thu, 28 Dec 2017 14:48:10 +0530 Subject: - Modify code server docker file to remove Oracle JDK, JRE - Add decorator to tasks --- tasks.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 8c78838..ac2721a 100644 --- a/tasks.py +++ b/tasks.py @@ -45,6 +45,7 @@ def setupdb(ctx): ctx.run("python manage.py migrate") print("** Done! Migrations complete **") +@task def loadfixtures(ctx): print("** Loading fixtures into database **") ctx.run("python manage.py loaddata demo_fixtures.json") -- cgit From c873adbf3bf94564dbc64ff3bced722dabc6fdc3 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Fri, 29 Dec 2017 16:49:21 +0530 Subject: Change README and tasks.py --- tasks.py | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index ac2721a..5e7295a 100644 --- a/tasks.py +++ b/tasks.py @@ -216,22 +216,6 @@ def deploy(ctx, fixtures=False): ctx.run(cmd) print ("** Done! Set up static assets **") -@task -def createsuperuser(ctx): - run_as_cmd = run_as(OS_NAME) - - base_su_cmd = "docker exec -it yaksh_django python3 manage.py createsuperuser" - cmd = get_cmd(run_as_cmd, base_su_cmd) - print ("** Creating Superuser **") - ctx.run(cmd) - print ("** Done! Created Superuser **") - - base_mod_cmd = "docker exec -i yaksh_django python3 manage.py add_group" - cmd = get_cmd(run_as_cmd, base_mod_cmd) - print ("** Creating Moderator group **") - ctx.run(cmd) - print ("** Done! Created Moderator group **") - @task def halt(ctx): run_as_cmd = run_as(OS_NAME) -- cgit From 7d7cc537f908db40ca6f3b10cc2c10d597b2877a Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 2 Jan 2018 14:31:09 +0530 Subject: Add changes as requested in review --- tasks.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 5e7295a..4a29c4d 100644 --- a/tasks.py +++ b/tasks.py @@ -26,6 +26,10 @@ def remove_check_file(path): if os.path.isfile(path): os.remove(path) +def remove_dir(path): + if os.path.isdir(path): + shutil.rmtree(path) + def run_as(os_name): if os_name.startswith('linux') or os_name == 'darwin' or os_name.startswith('freebsd'): return 'sudo' @@ -186,8 +190,8 @@ def begin(ctx): ctx.run(cmd) print ("** Done! Initialized the docker containers **") -@task(begin) -def deploy(ctx, fixtures=False): +@task +def deploy(ctx, fixtures=False, static=True): run_as_cmd = run_as(OS_NAME) print("** Setting up & migrating database **") @@ -210,11 +214,20 @@ def deploy(ctx, fixtures=False): ctx.run(cmd) print("** Done! Loaded fixtures into database **") - base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" - cmd = get_cmd(run_as_cmd, base_static_cmd) - print ("** Setting up static assets **") + if static: + base_static_cmd = "docker exec -i yaksh_django python3 manage.py collectstatic" + cmd = get_cmd(run_as_cmd, base_static_cmd) + print ("** Setting up static assets **") + ctx.run(cmd) + print ("** Done! Set up static assets **") + +@task +def status(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose ps" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Fetching container status **") ctx.run(cmd) - print ("** Done! Set up static assets **") @task def halt(ctx): @@ -225,11 +238,22 @@ def halt(ctx): ctx.run(cmd) print ("** Done! Stopped containers **") +@task +def restart(ctx): + run_as_cmd = run_as(OS_NAME) + base_cmd = "docker-compose restart" + cmd = get_cmd(run_as_cmd, base_cmd) + print ("** Restarting containers **") + ctx.run(cmd) + print ("** Done! Restarted containers **") + @task(halt) def remove(ctx): run_as_cmd = run_as(OS_NAME) base_cmd = "docker-compose rm --force" cmd = get_cmd(run_as_cmd, base_cmd) + sql_dir = os.path.join(SCRIPT_DIR, 'docker', 'mysql') print ("** Removing containers **") + remove_dir(sql_dir) ctx.run(cmd) print ("** Done! Removed containers **") -- cgit From 2846e4372bb5d255c9b4405456b133f174916893 Mon Sep 17 00:00:00 2001 From: ankitjavalkar Date: Tue, 2 Jan 2018 18:24:27 +0530 Subject: - Update README_production.rst to fix incorrect commands, links and formatting - Remove README inside docker folder --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tasks.py') diff --git a/tasks.py b/tasks.py index 4a29c4d..8396723 100644 --- a/tasks.py +++ b/tasks.py @@ -63,7 +63,7 @@ def serve(ctx): @task def clean(ctx): print("** Discarding database **") - ctx.run("rm -rf {0}".format(os.path.join(SCRIPT_DIR, 'db.sqlite3'))) + remove_check_file(os.path.join(SCRIPT_DIR, 'db.sqlite3')) @task def getimage(ctx, image=SRC_IMAGE_NAME): -- cgit