optimisation of code execution.

7 files changed, 243 insertions, 30 deletions

diff --git a/instances.py b/instances.py
new file mode 100644
index 0000000..d7059be
--- /dev/null
+++ b/instances.py
@@ -0,0 +1,153 @@
+# importing the global modules
+import pexpect
+import os
+import re
+import time
+import sys
+
+#importing the local variables
+from soc.settings import PROJECT_DIR
+from soc.config import SCILAB_FLAGS, SCIMAX_LOADER, UPLOADS_PATH
+
+''' An object of class ScilabInstance handles spawning and maintaining of multiple
+scilab instances. 
+
+maxsize is the upper bound of number of Scilab instances that can be alive at the 
+same time.
+
+instances list maintains a pool of free Scilab instances.
+
+count is the number of Scilab instances alive currently. 
+
+spawn_instance method is used to create Pexpect objects, which in turn spawn Scilab
+instance. A new instance is spawned only if the count is not exceeding the value of 
+maxsize. 
+
+kill_instances method is used to kill the free Scilab instances in the list instances,
+based on the parameter count passed while invoking the method.
+
+get_available_instance method is used to fetch a non-busy Scilab instance. 
+It receives one from the list instances or else invokes spawn_instances method to
+fetch a new instance to execute the Scilab code. If there are no Scilab instances
+available, the request has to wait until an instance is available.
+
+execute_code method executes the code passed as one of its parameter. It invokes
+get_available_instance method, fetches a Scilab instance and executes the code.
+After the execution of the code, the Pexpect object containing Scilab instance is 
+put back into the instances list. 
+'''
+class ScilabInstance(object):
+	
+	#defining instance variables
+	def __init__(self):
+		self.maxsize = 5
+		self.instances = []
+		self.count = 0
+
+	# spawning an instance
+	def spawn_instance(self):	
+		if (self.count < self.maxsize):
+			new_instance = pexpect.spawn('scilab-adv-cli')
+			self.count += 1
+			try:
+				new_instance.expect('-->', timeout = 30)
+				self.instances.append(new_instance)
+			except:
+				new_instance.close()
+				self.count -= 1
+	
+	# killing some spawned instances
+	def kill_instances(self, count):
+		for i in range(count):
+			instance = self.instances.pop(0)
+			instance.close()
+			self.count -= 1
+	
+	# returns an active_instancescilab instance. This will block till it gets an active_instance.
+	def get_available_instance(self):
+		if not self.instances and self.count < self.maxsize:
+			self.spawn_instance()
+		while not self.instances:
+			pass
+		return self.instances.pop(0)
+
+
+	def execute_code(self, code, token, book_id, dependency_exists):
+		#Check for system commands
+		system_commands = re.compile(
+			'unix\(.*\)|unix_g\(.*\)|unix_w\(.*\)|unix_x\(.*\)|unix_s\(.*\)|host|newfun|execstr|ascii|mputl|dir\(\)'
+		)
+		if system_commands.search(code):
+			return { 
+				'output': 'System Commands not allowed',
+			}
+
+		#Remove all clear;
+		code = re.sub(r'clear.*all|clear|clc\(\)|clc', '', code)
+
+		plot_exists = False
+
+		#Finding the plot and appending xs2jpg function
+		#p = re.compile(r'.*plot.*\(.*\).*\n|bode\(.*\)|evans\(.*\)')
+		p = re.compile(r'plot*|.*plot.*\(.*\).*\n|bode\(.*\)|evans\(.*\)')
+
+		plot_path = ''
+		if p.search(code):
+			plot_exists = True
+			code = code + '\n'
+			current_time = time.time()
+			plot_path = PROJECT_DIR + '/static/tmp/{0}.png'.format(str(current_time))
+			#code += 'xs2jpg(gcf(), "{0}");\n'.format(plot_path)
+
+		#Check whether to load scimax / maxima
+		if 'syms' in code or 'Syms' in code:
+			code = code.replace('syms', 'Syms')
+			code = 'exec(\'{0}\');\nmaxinit\n'.format(SCIMAX_LOADER) + code
+
+		file_path = PROJECT_DIR + '/static/tmp/' + token + '.sci'
+
+		#traps even syntax errors eg: endfunton
+		f = open(file_path, "w")
+		f.write("clear;");
+		f.write('driver("PNG");\n')
+		f.write('xinit("{0}");\n'.format(plot_path))
+		f.write('mode(2);\n')
+		if dependency_exists:
+			f.write(
+				'getd("{0}/{1}/DEPENDENCIES/");'.format(UPLOADS_PATH, book_id)
+			)
+		f.write('lines(0);\n')
+		f.write(code)
+		f.write('\nxend();')
+		f.close()
+		
+		cmd = 'exec("' + file_path + '", 2);'
+
+		active_instance= self.get_available_instance()
+		active_instance.sendline(cmd)
+
+		try:
+			active_instance.expect('\[0m ', timeout = 30)
+			active_instance.expect('', timeout = 30)
+			output = self.trim(active_instance.before)
+			self.instances.append(active_instance)
+			
+		except:
+			active_instance.before += "Exception Occured: It seems that you are running \
+											an infinite code"
+			output = self.trim(active_instance.before)
+			active_instance.close()
+			self.count -= 1		
+			if(self.count == 0):
+				self.spawn_instance()												
+		
+		data = {
+			'output': output,
+			'plot_path': plot_path.replace(PROJECT_DIR, '')
+		}
+		return data
+
+	def trim(self, output):
+		output = [line for line in output.split('\n') if line.strip() != '']
+		output = '\n'.join(output)
+		return output
\ No newline at end of file
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..9a515ac
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export DJANGO_SETTINGS_MODULE=soc.settings &&  
+python tornado_main.py
diff --git a/static/dajaxice/dajaxice.core.js b/static/dajaxice/dajaxice.core.js
index 917ed79..f252e7f 100644
--- a/static/dajaxice/dajaxice.core.js
+++ b/static/dajaxice/dajaxice.core.js
@@ -13,10 +13,6 @@ var Dajaxice = {
         return Dajaxice.call('website.node', 'POST', callback_function, argv, custom_settings);
     },
 
-    execute: function(callback_function, argv, custom_settings){
-        return Dajaxice.call('website.execute', 'POST', callback_function, argv, custom_settings);
-    },
-
     code: function(callback_function, argv, custom_settings){
         return Dajaxice.call('website.code', 'POST', callback_function, argv, custom_settings);
     },
diff --git a/static/website/js/cloud.js b/static/website/js/cloud.js
index 33dd209..e0ee41d 100644
--- a/static/website/js/cloud.js
+++ b/static/website/js/cloud.js
@@ -110,15 +110,24 @@ $(document).ready(function() {
         }, {example_id: $(this).val()});
     });
 
+    
     /* Execute the code */
     $plotbox_wrapper  = $("#plotbox-wrapper");
     $plotbox = $("#plotbox");
     $(document).on("click", "#execute", function() {
         $("#execute-inner").html("Executing...");
-        Dajaxice.website.execute(function(data) {
+        var send_data = {
+            token: $("[name='csrfmiddlewaretoken']").val(),
+            code: editor.getValue(),
+            book_id: $("#books").val() || 0,
+            chapter_id: $("#chapters").val() || 0,
+            example_id: $("#examples").val() || 0
+        };   
+        $.post("/execute-code", send_data,
+        function(data){
             $("#execute-inner").html("Execute");
             result.setValue(data.output);
-            if(data.plot_path) {
+            if(data.plot_path){
                 $plot = $("<img>");
                 $plot.attr({
                     src: data.plot_path,
@@ -127,12 +136,6 @@ $(document).ready(function() {
                 $plotbox.html($plot);
                 $plotbox_wrapper.lightbox_me({centered: true});
             }
-        }, {
-            token: $("[name='csrfmiddlewaretoken']").val(),
-            code: editor.getValue(),
-            book_id: $("#books").val() || 0,
-            chapter_id: $("#chapters").val() || 0,
-            example_id: $("#examples").val() || 0
         });
     });
 
diff --git a/tornado_main.py b/tornado_main.py
index 534a5a6..98ac671 100644
--- a/tornado_main.py
+++ b/tornado_main.py
@@ -2,9 +2,6 @@
 
 # Run this with
 # PYTHONPATH=. DJANGO_SETTINGS_MODULE=testsite.settings testsite/tornado_main.py
-# Serves by default at
-# http://localhost:8080/hello-tornado and
-# http://localhost:8080/hello-django
 
 from tornado.options import options, define, parse_command_line
 import django.core.handlers.wsgi
@@ -13,25 +10,82 @@ import tornado.ioloop
 import tornado.web
 import tornado.wsgi
 import os
+from django.utils import simplejson
+from website.models import TextbookCompanionExampleDependency, TextbookCompanionDependencyFiles
+
+from concurrent.futures import ThreadPoolExecutor
+from tornado import gen
+from instances import ScilabInstance
+import threading
 
 define('port', type=int, default=8080)
 
 # Custom settings
 from soc.settings import PROJECT_DIR
+from django.core.wsgi import get_wsgi_application
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "soc.settings")
 
+# request_count keeps track of the number of requests at hand, it is incremented 
+# when post method is invoked and decremented before exiting post method in 
+# class ExecutionHandler.
+DEFAULT_WORKERS = 5
+request_count = 0
+
+# ThreadPoolExecutor is an Executor subclass that uses a pool of threads to execute
+# function calls asynchronously.
+# It runs numbers of threads equal to DEFAULT_WORKERS in the background.
+executor = ThreadPoolExecutor(max_workers = DEFAULT_WORKERS)
+
+# scilab_executor is an object of class ScilabInstance used to manage(spawn, kill)
+# the Scilab instances and execute the code using those instances.
+scilab_executor = ScilabInstance()
+scilab_executor.spawn_instance()
+
+# instance_manager function is run at a fixed interval to kill the Scilab instances 
+# not in use. If the number of user requests is more than the count of active Scilab
+# instances, maximum instances defined will be in process. Instances will be killed
+# only when their number is more than the user requests.
+def instance_manager():
+  if(scilab_executor.count > request_count):
+    scilab_executor.kill_instances(scilab_executor.count-request_count-1)
+  threading.Timer(300, instance_manager).start()
+  
+instance_manager()
+
+# Whenever django server sends an ajax request,
+# the request is handled by the  ExecutionHandler
+# post method passes all the parameters received from the ajax call and
+# passes it to the submit method of ThreadPoolExecutor class through its object.
+# yield is used to gather the output asynchronously in the variable data
+class ExecutionHandler(tornado.web.RequestHandler): 
+	@gen.coroutine
+	def post(self):
+		global request_count
+		request_count += 1
+
+		token = buffer(self.request.arguments['token'][0])
+		token = str(token)
+		code =  buffer(self.request.arguments['code'][0])
+		code = str(code)
+		book_id = int(self.request.arguments['book_id'][0])
+		chapter_id = int(self.request.arguments['chapter_id'][0])
+		example_id = int(self.request.arguments['example_id'][0])
+		
+		dependency_exists = TextbookCompanionExampleDependency.objects.using('scilab')\
+			.filter(example_id=example_id).exists()
+		data  = yield executor.submit(scilab_executor.execute_code, code, token, 
+			book_id, dependency_exists)
+		self.write(data)
+		request_count -= 1
 
-class HelloHandler(tornado.web.RequestHandler):
-  def get(self):
-    self.write('Hello from tornado')
 
 def main():
   parse_command_line()
   wsgi_app = tornado.wsgi.WSGIContainer(
-    django.core.handlers.wsgi.WSGIHandler())
+    get_wsgi_application())
   tornado_app = tornado.web.Application(
     [
-      ('/hello-tornado', HelloHandler),
+      ('/execute-code', ExecutionHandler),
       ('/static/(.*)', tornado.web.StaticFileHandler, {'path': PROJECT_DIR + '/static/'}),
       ('.*', tornado.web.FallbackHandler, dict(fallback=wsgi_app)),
       ], debug=False)
@@ -39,5 +93,6 @@ def main():
   server.listen(options.port)
   tornado.ioloop.IOLoop.instance().start()
 
+
 if __name__ == '__main__':
-  main()
+  main()
\ No newline at end of file
diff --git a/website/ajax.py b/website/ajax.py
index 6d8d074..e69a07f 100644
--- a/website/ajax.py
+++ b/website/ajax.py
@@ -10,7 +10,6 @@ from django.core.context_processors import csrf
 from django.views.decorators.csrf import csrf_exempt, csrf_protect
 from django.db.models import Q
 
-from website.helpers import scilab_run
 from website.models import TextbookCompanionPreference,\
     TextbookCompanionProposal, TextbookCompanionChapter,\
     TextbookCompanionExample, TextbookCompanionExampleFiles,\
diff --git a/website/js/cloud.js b/website/js/cloud.js
index 33dd209..e0ee41d 100644
--- a/website/js/cloud.js
+++ b/website/js/cloud.js
@@ -110,15 +110,24 @@ $(document).ready(function() {
         }, {example_id: $(this).val()});
     });
 
+    
     /* Execute the code */
     $plotbox_wrapper  = $("#plotbox-wrapper");
     $plotbox = $("#plotbox");
     $(document).on("click", "#execute", function() {
         $("#execute-inner").html("Executing...");
-        Dajaxice.website.execute(function(data) {
+        var send_data = {
+            token: $("[name='csrfmiddlewaretoken']").val(),
+            code: editor.getValue(),
+            book_id: $("#books").val() || 0,
+            chapter_id: $("#chapters").val() || 0,
+            example_id: $("#examples").val() || 0
+        };   
+        $.post("/execute-code", send_data,
+        function(data){
             $("#execute-inner").html("Execute");
             result.setValue(data.output);
-            if(data.plot_path) {
+            if(data.plot_path){
                 $plot = $("<img>");
                 $plot.attr({
                     src: data.plot_path,
@@ -127,12 +136,6 @@ $(document).ready(function() {
                 $plotbox.html($plot);
                 $plotbox_wrapper.lightbox_me({centered: true});
             }
-        }, {
-            token: $("[name='csrfmiddlewaretoken']").val(),
-            code: editor.getValue(),
-            book_id: $("#books").val() || 0,
-            chapter_id: $("#chapters").val() || 0,
-            example_id: $("#examples").val() || 0
         });
     });