path: root/blocks/simulationAPI/helpers/scilab_manager.py

from celery.utils.log import get_task_logger
from datetime import datetime
from django.conf import settings
from django.http import FileResponse, Http404, JsonResponse
import gevent
from gevent.event import Event
from gevent.lock import RLock
import glob
import json
import fileinput
import os
from os.path import abspath, exists, isdir, isfile, join, splitext
import re
import signal
import subprocess
from tempfile import mkdtemp, mkstemp
from threading import current_thread
from time import time
import unicodedata
import uuid
import logging
from xml.dom import minidom
import shutil

from simulationAPI.helpers import config


os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'blocks.settings')

# Scilab dir
SCILAB_DIR = abspath(settings.SCILAB_DIR)
READCONTENTFILE = abspath("resources/Read_Content.txt")
SCILAB = join(SCILAB_DIR, 'bin', 'scilab-adv-cli')
BASEDIR = abspath('src/static')
IMAGEDIR = join(BASEDIR, config.IMAGEDIR)
IMAGEURLDIR = '/' + config.IMAGEDIR + '/'
SESSIONDIR = abspath(config.SESSIONDIR)
SYSTEM_COMMANDS = re.compile(config.SYSTEM_COMMANDS)
SPECIAL_CHARACTERS = re.compile(config.SPECIAL_CHARACTERS)

# This is the path to the upload directory and values directory
UPLOAD_FOLDER = 'uploads'  # to store xcos file
VALUES_FOLDER = 'values'  # to store files related to tkscale block
# to store uploaded sci files for sci-func block
SCRIPT_FILES_FOLDER = 'script_files'
# to store workspace files for TOWS_c block
WORKSPACE_FILES_FOLDER = 'workspace_files'

# Delay time to look for new line (in s)
LOOK_DELAY = 0.1

# display limit for long strings
DISPLAY_LIMIT = 10
# handle scilab startup
SCILAB_START = (
    "try;funcprot(0);lines(0,120);"
    "clearfun('messagebox');"
    "function messagebox(msg,title,icon,buttons,modal),disp(msg),endfunction;"
    "function xinfo(msg),disp(msg),endfunction;"
    "funcprot(1);"
    "catch;[error_message,error_number,error_line,error_func]=lasterror();"
    "disp(error_message,error_number,error_line,error_func);exit(3);end;"
)
SCILAB_END = (
    "catch;[error_message,error_number,error_line,error_func]=lasterror();"
    "disp(error_message,error_number,error_line,error_func);exit(2);end;exit;"
)

SCILAB_CMD = [SCILAB,
              "-noatomsautoload",
              "-nogui",
              "-nouserstartup",
              "-nb",
              "-nw",
              "-e", SCILAB_START
              ]

USER_DATA = {}


def secure_filename(filename: str) -> str:
    filename = unicodedata.normalize("NFKD", filename)
    filename = filename.encode("ascii", "ignore").decode("ascii")  # Remove accents
    filename = re.sub(r"[^a-zA-Z0-9_.-]", "_", filename)  # Replace invalid characters
    return filename.strip("._")  # Prevent filenames like ".." or "."


def makedirs(dirname, dirtype=None):
    if not exists(dirname):
        os.makedirs(dirname)


def rmdir(dirname, dirtype=None):
    if not isdir(dirname):
        logger.error('dir %s does not exist', dirname)
        return False
    if not config.REMOVEFILE:
        logger.debug('not removing dir %s', dirname)
        return True
    try:
        os.rmdir(dirname)
        return True
    except Exception as e:
        logger.warning('could not remove dir %s: %s', dirname, str(e))
        return False


def remove(filename):
    if not isfile(filename):
        logger.error('file %s does not exist', filename)
        return False
    if not config.REMOVEFILE:
        logger.debug('not removing file %s', filename)
        return True
    try:
        os.remove(filename)
        return True
    except Exception as e:
        logger.error('could not remove file %s: %s', filename, str(e))
        return False


logger = get_task_logger(__name__)

makedirs(SESSIONDIR, 'top session')


class ScilabInstance:
    proc = None
    log_name = None
    base = None
    starttime = None
    endtime = None

    def __init__(self):
        (self.proc, self.log_name) = prestart_scilab()

    def __str__(self):
        return "{pid: %s, log_name: %s}" % (self.proc.pid, self.log_name)


class Diagram:
    diagram_id = None
    # session dir
    sessiondir = None
    # store uploaded filename
    xcos_file_name = None
    # type of uploaded file
    workspace_counter = 0
    save_variables = set()
    # workspace from script
    workspace_file = None
    # tk count
    tk_count = 0
    # store log name
    instance = None
    # is thread running?
    tkbool = False
    tk_starttime = None
    # in memory values
    tk_deltatimes = None
    tk_values = None
    tk_times = None
    # List to store figure IDs from log_name
    figure_list = None
    file_image = ''

    def __init__(self):
        self.figure_list = []

    def __str__(self):
        return "{instance: %s, tkbool: %s, figure_list: %s}" % (
            self.instance, self.tkbool, self.figure_list)

    def clean(self):
        if self.instance is not None:
            kill_scilab(self)
            self.instance = None
        if self.xcos_file_name is not None:
            remove(self.xcos_file_name)
            self.xcos_file_name = None
        if self.workspace_file is not None:
            remove(self.workspace_file)
            self.workspace_file = None
        if self.file_image != '':
            remove(join(IMAGEDIR, self.file_image))
            self.file_image = ''


class Script:
    script_id = None
    sessiondir = None
    filename = None
    status = 0
    instance = None
    workspace_file = None

    def __str__(self):
        return (
            "{script_id: %s, filename: %s, status: %d, instance: %s, "
            "workspace_file: %s}") % (
                self.script_id, self.filename, self.status, self.instance,
                self.workspace_file)

    def clean(self):
        if self.instance is not None:
            kill_script(self)
            self.instance = None
        if self.filename is not None:
            remove(self.filename)
            self.filename = None
        if self.workspace_file is not None:
            remove(self.workspace_file)
            self.workspace_file = None


class SciFile:
    '''Variables used in sci-func block'''
    instance = None

    def __str__(self):
        return "{instance: %s}" % self.instance

    def clean(self):
        if self.instance is not None:
            kill_scifile(self)
            self.instance = None


class UserData:
    sessiondir = None
    diagrams = None
    scripts = None
    datafiles = None
    scifile = None
    timestamp = None

    def __init__(self):
        self.sessiondir = mkdtemp(
            prefix=datetime.now().strftime('%Y%m%d.'), dir=SESSIONDIR)
        self.diagrams = {}
        self.datafiles = []
        self.scripts = {}
        self.scifile = SciFile()
        self.timestamp = time()

    def __str__(self):
        return (f"UserData(sessiondir={self.sessiondir}, "
                f"diagrams={list(self.diagrams.keys())}, "
                f"datafiles={len(self.datafiles)}, "
                f"scripts={list(self.scripts.keys())}, "
                f"scifile={self.scifile}, "
                f"timestamp={datetime.fromtimestamp(self.timestamp).strftime('%Y-%m-%dT%H:%M:%S')})")

    def __repr__(self):
        return self.__str__()

    def clean(self):
        for diagram in self.diagrams.values():
            diagram.clean()
        self.diagrams = None
        for script in self.scripts:
            self.scripts[script].clean()
        self.scripts = None
        for datafile in self.datafiles:
            datafile.clean()
        self.datafiles = None
        self.scifile.clean()
        self.scifile = None
        # name of workspace file
        workspace = join(self.sessiondir, WORKSPACE_FILES_FOLDER,
                         "workspace.dat")
        if exists(workspace):
            remove(workspace)

        sessiondir = self.sessiondir

        rmdir(join(sessiondir, WORKSPACE_FILES_FOLDER), 'workspace files')
        rmdir(join(sessiondir, SCRIPT_FILES_FOLDER), 'script files')
        rmdir(join(sessiondir, VALUES_FOLDER), 'values')
        rmdir(join(sessiondir, UPLOAD_FOLDER), 'upload')
        rmdir(sessiondir, 'session')


INSTANCES_1 = []
INSTANCES_2 = []
evt = Event()


def no_free_scilab_instance():
    l1 = len(INSTANCES_1)
    return l1 == 0


def too_many_scilab_instances():
    l1 = len(INSTANCES_1)
    l2 = len(INSTANCES_2)
    return l1 >= config.SCILAB_MIN_INSTANCES or \
        l1 + l2 >= config.SCILAB_MAX_INSTANCES


def start_scilab_instances():
    l1 = len(INSTANCES_1)
    l2 = len(INSTANCES_2)
    lssi = min(config.SCILAB_START_INSTANCES,
               config.SCILAB_MAX_INSTANCES - l2) - l1
    if lssi > 0:
        logger.info('can start %s instances', lssi)
    return lssi


def print_scilab_instances():
    l1 = len(INSTANCES_1)
    l2 = len(INSTANCES_2)
    msg = ''
    if l1 > 0:
        msg += ', free=' + str(l1)
    if l2 > 0:
        msg += ', in use=' + str(l2)
    logger.info('instance count: %s', msg[2:])


FIRST_INSTANCE = True


def prestart_scilab_instances():
    global FIRST_INSTANCE

    current_thread().name = 'PreStart'

    attempt = 1

    while True:
        while too_many_scilab_instances():
            evt.wait()

        for i in range(start_scilab_instances()):
            instance = ScilabInstance()
            proc = instance.proc
            if proc is None:
                gevent.thread.interrupt_main()
                return

            if FIRST_INSTANCE:
                gevent.sleep(1)
                for i in range(2, 4):
                    if proc.poll() is not None:
                        break
                    gevent.sleep(i)

            if proc.poll() is not None:
                (out, err) = proc.communicate()
                out = re.sub(r'^[ !\\-]*\n', r'', out, flags=re.MULTILINE)
                if out:
                    logger.info('=== Output from scilab console ===\n%s',
                                out)
                if err:
                    logger.info('=== Error from scilab console ===\n%s',
                                err)

                # Check for errors in Scilab
                if 'Cannot find scilab-bin' in out:
                    logger.critical('scilab has not been built. '
                                    'Follow the installation instructions')
                    gevent.thread.interrupt_main()
                    return

                returncode = proc.returncode
                msg = 'attempts' if attempt != 1 else 'attempt'

                if attempt >= 4:
                    logger.critical('aborting after %s %s: rc = %s',
                                    attempt, msg, returncode)
                    gevent.thread.interrupt_main()
                    return

                logger.error('retrying after %s %s: rc = %s',
                             attempt, msg, returncode)
                gevent.sleep(config.SCILAB_INSTANCE_RETRY_INTERVAL * attempt)
                attempt += 1
                FIRST_INSTANCE = True
                continue

            INSTANCES_1.append(instance)
            attempt = 1
            FIRST_INSTANCE = False

        print_scilab_instances()

        if too_many_scilab_instances():
            evt.clear()


def get_scilab_instance():
    global FIRST_INSTANCE

    try:
        while True:
            instance = INSTANCES_1.pop(0)
            proc = instance.proc
            if proc.poll() is not None:
                logger.warning('scilab instance exited: return code is %s',
                               proc.returncode)
                FIRST_INSTANCE = True
                if not too_many_scilab_instances():
                    evt.set()
                    if no_free_scilab_instance():
                        gevent.sleep(4)
                continue

            INSTANCES_2.append(instance)
            print_scilab_instances()
            if not too_many_scilab_instances():
                evt.set()

            return instance
    except IndexError:
        logger.error('No free instance')
        return None


def remove_scilab_instance(instance):
    try:
        INSTANCES_2.remove(instance)
        print_scilab_instances()
        if not too_many_scilab_instances():
            evt.set()
    except ValueError:
        logger.error('could not find instance %s', instance)


def stop_scilab_instance(base, createlogfile=False):
    stop_instance(base.instance, createlogfile)

    base.instance = None


def kill_scilab_with(proc, sig):
    '''
    function to kill a process group with a signal. wait for maximum 2 seconds
    for process to exit. return True on exit, False otherwise
    '''

    if proc.poll() is not None:
        return True

    try:
        os.killpg(proc.pid, sig)
    except OSError:
        logger.warning('could not kill %s with signal %s', proc.pid, sig)
        return False
    except TypeError:
        logger.warning('could not kill invalid process %s with signal %s', proc.pid, sig)
        return True
    except ProcessLookupError:
        logger.warning('could not find process %s to kill with signal %s', proc.pid, sig)
        return True
    except Exception as e:
        logger.warning('Error killing process %s with signal %s', proc.pid, sig, e)

    for i in range(0, 20):
        gevent.sleep(LOOK_DELAY)
        if proc.poll() is not None:
            return True
    return False


def stop_instance(instance, createlogfile=False, removeinstance=True):
    if instance is None:
        logger.warning('no instance')
        return

    if not kill_scilab_with(instance.proc, signal.SIGTERM):
        kill_scilab_with(instance.proc, signal.SIGKILL)

    if removeinstance:
        remove_scilab_instance(instance)

    if instance.log_name is None:
        if createlogfile:
            logger.warning('empty diagram')
    else:
        # remove(instance.log_name)
        instance.log_name = None

    instance.base = None


def stop_scilab_instances():
    if len(INSTANCES_1) > 0:
        logger.info('stopping %s idle instances', len(INSTANCES_1))
        while len(INSTANCES_1) > 0:
            instance = INSTANCES_1.pop()
            stop_instance(instance, removeinstance=False)

    if len(INSTANCES_2) > 0:
        logger.info('stopping %s busy instances', len(INSTANCES_2))
        while len(INSTANCES_2) > 0:
            instance = INSTANCES_2.pop()
            stop_instance(instance, removeinstance=False)


def reap_scilab_instances():
    current_thread().name = 'Reaper'
    while True:
        gevent.sleep(100)

        remove_instances = []

        for instance in INSTANCES_2:
            if instance.endtime < time():
                remove_instances.append(instance)

        count = len(remove_instances)
        if count == 0:
            continue

        logger.info('removing %s stale instances', count)
        for instance in remove_instances:
            base = instance.base
            if base is None:
                logger.warning('cannot stop instance %s', instance)
                stop_instance(instance)
            elif isinstance(base, Diagram):
                kill_scilab(base)
            elif isinstance(base, Script):
                kill_script(base)
            elif isinstance(base, SciFile):
                kill_scifile(base)
            else:
                logger.warning('cannot stop instance %s', instance)
                stop_instance(instance)


class DataFile:
    sessiondir = None
    data_filename = None

    def clean(self):
        if self.data_filename is not None:
            remove(self.data_filename)
            self.data_filename = None


def clean_sessions(final=False):
    current_thread().name = 'Clean'
    totalcount = 0
    cleanuds = []
    for session_id, ud in USER_DATA.items():
        totalcount += 1
        if final or time() - ud.timestamp > config.SESSIONTIMEOUT:
            cleanuds.append(session_id)

    logger.info('cleaning %s/%s sessions', len(cleanuds), totalcount)
    for session_id in cleanuds:
        current_thread().name = 'Clean-%s' % session_id[:6]
        try:
            logger.info('cleaning')
            ud = USER_DATA.pop(session_id)
            ud.clean()
        except Exception as e:
            logger.warning('could not clean: %s', str(e))


def clean_sessions_thread():
    current_thread().name = 'Clean'
    while True:
        gevent.sleep(config.SESSIONTIMEOUT / 2)
        try:
            clean_sessions()
        except Exception as e:
            logger.warning('Exception in clean_sessions: %s', str(e))


logfilefdrlock = RLock()
LOGFILEFD = 123


def get_session(session):
    session_id = session.session_id
    if not hasattr(current_thread(), 's_name'):
        current_thread().s_name = current_thread().name
    current_thread().name = 'S-%s-%s' % (current_thread().s_name[12:], session_id[:6])
    return session_id


def init_session(session):
    session_id = get_session(session)

    if session_id not in USER_DATA:
        USER_DATA[session_id] = UserData()

    ud = USER_DATA[session_id]
    ud.timestamp = time()

    sessiondir = ud.sessiondir

    makedirs(sessiondir, 'session')
    makedirs(join(sessiondir, UPLOAD_FOLDER), 'upload')
    makedirs(join(sessiondir, VALUES_FOLDER), 'values')
    makedirs(join(sessiondir, SCRIPT_FILES_FOLDER), 'script files')
    makedirs(join(sessiondir, WORKSPACE_FILES_FOLDER), 'workspace files')

    return (ud.diagrams, ud.scripts, ud.scifile, ud.datafiles, sessiondir)


def prestart_scilab():
    logfilefd, log_name = mkstemp(prefix=datetime.now().strftime(
        'scilab-log-%Y%m%d-'), suffix='.txt', dir=SESSIONDIR)

    with logfilefdrlock:
        if logfilefd != LOGFILEFD:
            os.dup2(logfilefd, LOGFILEFD)
            os.close(logfilefd)

        try:
            proc = subprocess.Popen(
                SCILAB_CMD,
                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE, start_new_session=True,
                universal_newlines=True, pass_fds=(LOGFILEFD, ))
        except FileNotFoundError:
            logger.critical('scilab has not been built. '
                            'Follow the installation instructions')
            proc = None
            remove(log_name)
            log_name = None

        os.close(LOGFILEFD)

    return (proc, log_name)


def run_scilab(command, base, createlogfile=False, timeout=1800):
    instance = get_scilab_instance()
    if instance is None:
        logger.error('cannot run command %s', command)
        return None

    logger.info('Scilab instance log file: %s', instance.log_name)
    cmd = 'try;' + command + SCILAB_END + '\n'
    logger.info('running command %s', cmd)
    instance.proc.stdin.write(cmd)
    instance.proc.stdin.flush()

    # output, error = instance.proc.communicate(timeout=timeout)
    # with open(instance.log_name, 'a') as log:
    #     log.write(output if output else '')
    #     log.write(error if error else '')

    if not createlogfile:
        remove(instance.log_name)
        instance.log_name = None

    instance.base = base
    instance.starttime = time()
    instance.endtime = time() + timeout
    return instance


def is_unsafe_script(filename):
    '''
    Read file and check for system commands and return error if file contains
    system commands
    '''
    with open(filename, 'r') as f:
        if not re.search(SYSTEM_COMMANDS, f.read()):
            return False

    # Delete saved file if system commands are encountered in that file
    remove(filename)
    return True


def uploaddatafile(session, task):
    '''
    Below route is called for uploading audio/other file.
    '''
    # Get the au/other data file
    file = task.file.name
    # Check if the data file is not null
    if not file:
        msg = "Error occured while uploading file. Please try again\n"
        rv = {'msg': msg}
        return JsonResponse(rv)

    (datafile, sessiondir, currlen) = add_datafile(session)
    fname = join(sessiondir, UPLOAD_FOLDER, currlen + '@@' + secure_filename(file.filename))
    file.save(fname)
    datafile.data_filename = fname
    rv = {'filepath': datafile.data_filename}
    return JsonResponse(rv)


def handle_line(current_line, line):
    line = line.rstrip()
    if line.endswith('...'):
        # Remove continuation and add to buffer
        current_line += line.rstrip('.').rstrip() + ' '
        logger.info('Current line: %s#', current_line.rstrip())
        complete_line = False
    else:
        if current_line:
            current_line += line
            logger.info('Line: %s$', current_line.rstrip())
        else:
            current_line = line
        complete_line = True
    return current_line, complete_line


def handle_uploaded_sce_file(file, fname):
    with open(fname, 'w') as f:
        current_line = ''
        partial_line = ''

        for chunk in file.chunks():
            text = chunk.decode('ascii')
            lines = text.splitlines()

            if not lines:
                continue

            # Add partial_line to the first line
            if partial_line:
                lines[0] = partial_line + lines[0]
                partial_line = ''
                logger.info('Complete line: %s$', lines[0].rstrip())

            # If the last line doesn't end with a newline, it is partial
            if not lines[-1].endswith('\n'):
                partial_line = lines.pop()  # Save the partial line
                logger.info('Partial line: %s#', partial_line)

            for line in lines:
                current_line, complete_line = handle_line(current_line, line)
                if complete_line:
                    f.write(current_line.rstrip() + '\n')
                    current_line = ''

        # Add partial_line to the remaining line
        if partial_line:
            current_line, complete_line = handle_line(current_line, partial_line)

        # Write remaining line if any
        if current_line:
            logger.info('Last line: %s$', current_line.rstrip())
            f.write(current_line.rstrip() + '\n')


def uploadscript(session, task):
    '''
    Below route is called for uploading script file.
    '''
    (script, sessiondir) = add_script(session, str(task.task_id))

    file = task.file
    if not file:
        msg = "Upload Error\n"
        rv = {'msg': msg}
        return rv

    fname = join(sessiondir, SCRIPT_FILES_FOLDER,
                 f"{script.script_id}_script.sce")
    handle_uploaded_sce_file(file, fname)
    script.filename = fname

    if is_unsafe_script(fname):
        msg = ("System calls are not allowed in script.\n"
               "Please edit the script again.\n")
        script.status = -1
        rv = {'status': script.status, 'msg': msg}
        return rv

    wfname = join(sessiondir, WORKSPACE_FILES_FOLDER,
                  f"{script.script_id}_script_workspace.dat")
    script.workspace_file = wfname
    command = "exec('%s');save('%s');" % (fname, wfname)

    script.instance = run_scilab(command, script)

    if script.instance is None:
        msg = "Resource not available"
        script.status = -2
        rv = {'status': script.status, 'msg': msg}
        return rv

    # Save workspace file in task model
    task.workspace_file = wfname
    task.save()

    msg = ''
    script.status = 1
    rv = {'task_id': str(task.task_id), 'script_id': script.script_id, 'status': script.status, 'msg': msg}
    return rv


def clean_output(s):
    '''handle whitespace and sequences in output'''
    s = re.sub(r'[\a\b\f\r\v]', r'', s)
    # https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
    s = re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', r'', s)
    s = re.sub(r'\t', r'    ', s)
    s = re.sub(r' +(\n|$)', r'\n', s)
    s = re.sub(r'\n+', r'\n', s)
    s = re.sub(r'^\n', r'', s)
    return s


def getscriptoutput(session, task):
    '''
    Below route is called for uploading script file.
    '''
    script = get_script(session, str(task.task_id))
    if script is None:
        # when called with same script_id again or with incorrect script_id
        logger.warning('no script')
        msg = "no script"
        rv = {'msg': msg}
        return rv

    instance = script.instance
    if instance is None:
        logger.warning('no instance')
        msg = "no instance"
        rv = {'msg': msg}
        return rv

    proc = instance.proc

    try:
        # output from scilab terminal is saved for checking error msg
        output = proc.communicate(timeout=30)[0]
        output = clean_output(output)
        remove_scilab_instance(script.instance)
        script.instance = None

        returncode = proc.returncode
        if returncode < 0 or returncode == 2:
            logger.warning('return code is %s', returncode)
            msg = 'Script stopped'
            script.status = -5
            rv = {'status': script.status, 'msg': msg, 'output': output}
            return rv
        if returncode > 0:
            logger.info('return code is %s', returncode)
            if output:
                logger.info('=== Output from scilab console ===\n%s', output)

        # if error is encountered while execution of script file, then error
        # message is returned to the user
        if '!--error' in output:
            msg = ("Check result window for details.\n"
                   "Please edit the script and execute again.\n")
            script.status = -3
            rv = {'status': script.status, 'msg': msg, 'output': output}
            return rv

        logger.info('workspace for %s saved in %s',
                    script.script_id, script.workspace_file)
        msg = ''
        script.status = 0

        cmd = list_variables(script.workspace_file)
        script.instance = run_scilab(cmd, script)
        instance = script.instance

        if instance is None:
            msg = "Resource not available"
            script.status = -2
            rv = {'status': script.status, 'msg': msg}
            return rv

        proc = instance.proc
        listoutput = proc.communicate(timeout=10)[0]
        remove_scilab_instance(script.instance)
        script.instance = None

        returncode = proc.returncode
        if returncode < 0 or returncode == 2:
            logger.warning('return code is %s', returncode)
            msg = 'Script stopped'
            script.status = -5
            rv = {'status': script.status, 'msg': msg, 'output': listoutput}
            return rv
        if returncode > 0:
            logger.info('return code is %s', returncode)
            if listoutput:
                logger.info('=== List output from scilab console ===\n%s',
                            listoutput)
        try:
            listoutput = listoutput.strip()
            variables = json.loads(listoutput)
        except Exception as e:
            logger.warning('error while loading: %s: %s', listoutput, str(e))
            variables = []

        rv = {'script_id': script.script_id, 'status': script.status,
              'msg': msg, 'output': output, 'returncode': returncode,
              'variables': variables}
        return rv
    except subprocess.TimeoutExpired:
        kill_script(script)
        msg = 'Timeout'
        script.status = -4
        rv = {'status': script.status, 'msg': msg}
        return rv
    except UnicodeDecodeError:
        kill_script(script)
        msg = 'Unicode Decode Error'
        script.status = -6
        rv = {'status': script.status, 'msg': msg}
        return rv


def sendfile(session, task):
    '''
    This route is used in chart.js for sending image filename
    '''
    diagram = get_diagram(session, task)
    if diagram is None:
        logger.warning('no diagram')
        return ''
    if diagram.file_image == '':
        logger.warning('no diagram image')
        return ''

    return IMAGEURLDIR + diagram.file_image


def list_variables(filename):
    '''
    add scilab commands to list only user defined variables
    '''

    command = "[__V1,__V2,__V3]=listvarinfile('%s');" % filename
    command += "__V5=grep(string(__V2),'/^([124568]|1[0])$/','r');"
    command += "__V1=__V1(__V5);"
    command += "__V2=__V2(__V5);"
    command += "__V3=list(__V3(__V5));"
    command += "__V5=setdiff(grep(__V1,'/^[^%]+$/','r'),grep(__V1,'/^PWD$/','r'));"
    command += "if ~isempty(__V5) then;"
    command += "__V1=__V1(__V5);"
    command += "__V2=__V2(__V5);"
    command += "__V3=list(__V3(__V5));"
    command += "__V6=''''+strcat(__V1,''',''')+'''';"
    command += "__V7='load(''%s'','+__V6+');';" % filename
    command += "execstr(__V7);"
    command += "__V9='[';"
    command += "for __V8=1:size(__V5,2);"
    command += "__V18=__V1(__V8);"
    command += "__V28=__V2(__V8);"
    command += "__V38=__V3(__V8);"
    command += "__V9=__V9+'{\"\"name\"\":\"\"'+__V18+'\"\",'+"
    command += "'\"\"type\"\":\"\"'+string(__V28)+'\"\",'+"
    command += "'\"\"size\"\":\"\"'+sci2exp(__V38)+'\"\",'+"
    command += "'\"\"value\"\":';"
    command += "if size(__V38,2)>1 then;"
    command += "__V10=__V38(1)*__V38(2);"
    command += "else;"
    command += "__V10=__V38;"
    command += "end;"
    command += "if __V10<=100 then;"
    command += "if __V28<>10 then;"
    command += "__V9=__V9+'\"\"'+sci2exp(evstr(__V18))+'\"\"';"
    command += "else;"
    command += "__V9=__V9+sci2exp(evstr(__V18));"
    command += "end;"
    command += "else;"
    command += "__V9=__V9+'\"\"'+sci2exp(evstr(__V18))+'\"\"';"
    command += "end;"
    command += "__V9=__V9+'}';"
    command += "if __V8<size(__V5,2) then;"
    command += "__V9=__V9+',';"
    command += "end;"
    command += "end;"
    command += "__V9=__V9+']';"
    command += "printf('%s',__V9);"
    command += "end;"
    return command


def load_variables(filename):
    '''
    add scilab commands to load only user defined variables
    '''

    command = "[__V1,__V2]=listvarinfile('%s');" % filename
    command += "__V5=grep(string(__V2),'/^([124568]|1[07])$/','r');"
    command += "__V1=__V1(__V5);"
    command += "__V2=__V2(__V5);"
    command += "__V5=grep(__V1,'/^[^%]+$/','r');"
    command += "if ~isempty(__V5) then;"
    command += "__V1=__V1(__V5);"
    command += "__V2=__V2(__V5);"
    command += "__V6=''''+strcat(__V1,''',''')+'''';"
    command += "__V7='load(''%s'','+__V6+');';" % filename
    command += "execstr(__V7);"
    command += "end;"
    command += "clear __V1 __V2 __V5 __V6 __V7;"
    return command


def start_scilab(session, task, xcosfile):
    '''
    function to execute xcos file using scilab (scilab-adv-cli), access log
    file written by scilab
    '''
    diagram = get_diagram(session, task)
    if diagram is None:
        logger.warning('no diagram')
        return "error"

    # name of primary workspace file
    workspace_file = diagram.workspace_file
    # name of workspace file
    workspace = join(diagram.sessiondir, WORKSPACE_FILES_FOLDER,
                     "workspace.dat")

    if diagram.workspace_counter in (2, 3) and not exists(workspace):
        logger.warning('no workspace')
        return ("Workspace does not exist. "
                "Please simulate a diagram with TOWS_c block first. "
                "Do not use any FROMWSB block in that diagram.")

    loadfile = workspace_file is not None or \
        diagram.workspace_counter in (2, 3)

    command = ""

    if loadfile:
        # ignore import errors
        command += "try;"

        if workspace_file is not None:
            command += load_variables(workspace_file)

        if diagram.workspace_counter in (2, 3):
            # 3 - for both TOWS_c and FROMWSB and also workspace dat file exist
            # In this case workspace is saved in format of dat file (Scilab way
            # of saying workpsace)
            # For FROMWSB block and also workspace dat file exist
            command += load_variables(workspace)

        command += "catch;disp('Error: ' + lasterror());end;"

    # Scilab Commands for running of scilab based on existence of different
    # blocks in same diagram from workspace_counter's value
    #    1: Indicate TOWS_c exist
    #    2: Indicate FROMWSB exist
    #    3: Both TOWS_c and FROMWSB exist
    #    4: Indicate AFFICH_m exist (We dont want graphic window to open so
    #    xs2jpg() command is removed)
    #    5: Indicate Sci-func block as it some time return image as output
    #    rather than Sinks's log file.
    #    0/No-condition : For all other blocks

    command += "loadXcosLibs();"
    command += "importXcosDiagram('%s');" % diagram.xcos_file_name
    command += "xcos_simulate(scs_m,4);"

    if diagram.workspace_counter == 4:
        # For AFFICH-m block
        pass
    elif diagram.workspace_counter == 5:
        # For Sci-Func block (Image are return as output in some cases)
        diagram.file_image = 'img-%s-%s.jpg' % (
            datetime.now().strftime('%Y%m%d'), str(uuid.uuid4()))
        command += "xs2jpg(gcf(),'%s/%s');" % (IMAGEDIR, diagram.file_image)
    elif config.CREATEIMAGE:
        # For all other block
        command += "xs2jpg(gcf(),'%s/%s');" % (IMAGEDIR, 'img_test.jpg')

    if diagram.workspace_counter in (1, 3):
        if diagram.save_variables:
            command += "save('%s','%s');" % (
                workspace, "','".join(diagram.save_variables))
        else:
            command += "save('%s');" % workspace

    diagram.instance = run_scilab(command, diagram, True,
                                  config.SCILAB_INSTANCE_TIMEOUT_INTERVAL + 60)

    if diagram.instance is None:
        return "Resource not available"

    instance = diagram.instance
    logger.info('log_name=%s', instance.log_name)
    task.log_name = instance.log_name
    task.save()

    # Start sending log to chart function for creating chart
    try:
        # For processes taking less than 10 seconds
        scilab_out = instance.proc.communicate(timeout=4)[0]
        scilab_out = re.sub(r'^[ !\\-]*\n', r'',
                            scilab_out, flags=re.MULTILINE)
        if scilab_out:
            logger.info('=== Output from scilab console ===\n%s', scilab_out)
        # Check for errors in Scilab
        if "Empty diagram" in scilab_out:
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return "Empty diagram"

        m = re.search(r'Fatal error: exception Failure\("([^"]*)"\)',
                      scilab_out)
        if m:
            msg = 'modelica error: ' + m.group(1)
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return msg

        if ("xcos_simulate: "
                "Error during block parameters update.") in scilab_out:
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return "Error in block parameter. Please check block parameters"

        if "xcosDiagramToScilab:" in scilab_out:
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return "Error in xcos diagram. Please check diagram"

        if "Simulation problem:" in scilab_out:
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return "Error in simulation. Please check script uploaded/executed"

        if "Cannot find scilab-bin" in scilab_out:
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return ("scilab has not been built. "
                    "Follow the installation instructions")

        if os.stat(instance.log_name).st_size == 0 and \
                diagram.workspace_counter not in (1, 5):
            remove_scilab_instance(diagram.instance)
            diagram.instance = None
            return "log file is empty"

    # For processes taking more than 10 seconds
    except subprocess.TimeoutExpired:
        pass

    gevent.spawn(wait_for_instance, diagram)
    return ""


def wait_for_instance(diagram):
    diagram.instance.proc.wait()
    kill_scilab(diagram)


def stopDetailsThread(diagram):
    diagram.tkbool = False  # stops the thread
    gevent.sleep(LOOK_DELAY)
    fname = join(diagram.sessiondir, VALUES_FOLDER,
                 diagram.diagram_id + "_*")
    for fn in glob.glob(fname):
        # deletes all files created under the 'diagram_id' name
        remove(fn)


def upload(session, task, xcosfile):
    '''Route that will process the file upload'''
    # Get the file
    file = xcosfile
    # Check if the file is not null
    if not file:
        return "error"
    # flags to check if both TOWS_c and FROMWSB are present
    flag1 = 0
    flag2 = 0
    list1 = []
    list2 = []
    # Make the filename safe, remove unsupported chars
    (diagram, scripts, sessiondir) = add_diagram(session, task)

    script = get_script(session, task.script_task_id, scripts=scripts)
    if script is None and task.workspace_file is not None:
        logger.info('adding script')
        (script, __) = add_script(session, task.script_task_id)
        script.workspace_file = task.workspace_file
    if script is not None:
        diagram.workspace_file = script.workspace_file
    # Save the file in xml extension and using it for further modification
    # by using xml parser
    temp_file_xml_name = diagram.diagram_id + ".xml"
    shutil.copy(xcosfile, temp_file_xml_name)
    # file.save(temp_file_xml_name)
    new_xml = minidom.parse(temp_file_xml_name)

    # to identify if we have to load or save to workspace or neither #0 if
    # neither TOWS_c or FROMWSB found
    blocks = new_xml.getElementsByTagName("BasicBlock")
    tk_is_present = False
    pattern = re.compile(r"<SplitBlock")
    for i, line in enumerate(open(temp_file_xml_name)):
        for match in re.finditer(pattern, line):
            list1.append(i + 1)
    pattern1 = re.compile(r"<ControlPort")
    for i, line in enumerate(open(temp_file_xml_name)):
        for match in re.finditer(pattern1, line):
            list2.append(i + 1)
    pattern2 = re.compile(r"<ImplicitInputPort")
    count1 = 0

    for i, line in enumerate(open(temp_file_xml_name)):
        for match in re.finditer(pattern2, line):
            count1 += 1
    if count1 >= 1:
        splitline = []
        count = 0
        for i in range(len(list1)):
            for j in range(len(list2)):
                if list2[j] == list1[i] + 3:
                    count += 1
                    splitline.append(list1[i])
        blocksplit = new_xml.getElementsByTagName("SplitBlock")
        block_ids = []  # this stores the id of split blocks
        for block in blocksplit:
            if block.getAttribute("style") == "SPLIT_f":
                # block_ids.append(int(block.getAttribute("id")))
                block_ids.append(block.getAttribute("id"))
        compsplit = []
        for i in range(len(splitline)):
            for j in range(len(list1)):
                if splitline[i] == list1[j]:
                    compsplit.append(j)

        finalsplit = []
        for i in range(len(compsplit)):
            finalsplit.append(block_ids[compsplit[i]])

        blockcontrol = new_xml.getElementsByTagName("ControlPort")
        for block in blockcontrol:
            for i in range(len(finalsplit)):
                # match the lines with the parent of our spliblocks which
                # we need to change
                if block.getAttribute("parent") == str(finalsplit[i]):
                    block.setAttribute('id', '-1')

        blockcommand = new_xml.getElementsByTagName("CommandPort")
        for block in blockcommand:
            for i in range(len(finalsplit)):
                if block.getAttribute("parent") == str(finalsplit[i]):
                    block.setAttribute('id', '-1')

        # here we take the ids of command controllink which we will search
        # and change
        finalchangeid = []
        for i in range(len(finalsplit)):
            finalchangeid.append(finalsplit[i] + 4)
            finalchangeid.append(finalsplit[i] + 5)

        # here we save the contents
        with open(temp_file_xml_name, 'w') as f:
            f.write(new_xml.toxml())

        with open(temp_file_xml_name, "r") as f:
            newline = []
            i = 0
            for word in f.readlines():

                if "<CommandControlLink id=" in word:
                    temp_word = ""
                    for i in range(len(finalchangeid)):
                        fcid = str(finalchangeid[i])
                        srch = '<CommandControlLink id="' + fcid + '"'
                        if srch in word:
                            rplc = '<ImplicitLink id="' + fcid + '"'
                            temp_word = word.replace(srch, rplc)
                            i += 1
                    if temp_word != "":
                        newline.append(temp_word)
                    else:
                        newline.append(word)
                else:
                    newline.append(word)
        with open(temp_file_xml_name, "w") as f:
            for line in newline:
                f.writelines(line)
        with open(temp_file_xml_name, "r") as in_file:
            buf = in_file.readlines()
        # length=len(finalsplit)
        # return finalsplit
        with open(temp_file_xml_name, "w") as out_file:
            for line in buf:
                for i in range(len(finalsplit)):
                    fs = str(finalsplit[i])
                    srch = ('<ControlPort connectable="0" '
                            'dataType="UNKNOW_TYPE" id="-1" ordering="1" '
                            'parent="' + fs + '"')
                    if srch in line:
                        line = (
                            '\t    <ImplicitInputPort connectable="0" '
                            'dataType="UNKNOW_TYPE" '
                            'id="' + str(finalsplit[i] + 1) + '" '
                            'ordering="1" parent="' + fs + '" '
                            'style="ImplicitInputPort">\n'
                            '\t\t<mxGeometry as="geometry" height="10" '
                            'relative="1" width="10" y="0.5000">\n'
                            '\t\t</mxGeometry>\n'
                            '\t    </ImplicitInputPort>\n'
                            '\t    <ImplicitOutputPort connectable="0" '
                            'dataType="UNKNOW_TYPE" '
                            'id="' + str(finalsplit[i] + 2) + '" '
                            'ordering="1" parent="' + fs + '" '
                            'style="ImplicitOutputPort">\n'
                            '\t\t<mxGeometry as="geometry" height="10" '
                            'relative="1" width="10" y="0.5000">\n'
                            '\t\t</mxGeometry>\n'
                            '\t    </ImplicitOutputPort>\n'
                            '\t    <ImplicitOutputPort connectable="0" '
                            'dataType="UNKNOW_TYPE" '
                            'id="' + str(finalsplit[i] + 3) + '" '
                            'ordering="1" parent="' + fs + '" '
                            'style="ImplicitOutputPort">\n'
                            '\t\t<mxGeometry as="geometry" height="10" '
                            'relative="1" width="10" y="0.5000">\n'
                            '\t\t</mxGeometry>\n'
                            '\t    </ImplicitOutputPort>\n' + line)

                out_file.write(line)
        list3 = []
        implitdetect = []
        # return temp_file_xml_name
        for i in range(len(finalsplit)):
            implitdetect.append(finalsplit[i] + 5)
            implitdetect.append(finalsplit[i] + 6)
        for i in range(len(implitdetect)):
            pattern3 = re.compile(
                "<ImplicitLink id=\"" + str(implitdetect[i]) + "\"")
            for i, line in enumerate(open(temp_file_xml_name)):
                for match in re.finditer(pattern3, line):
                    list3.append(i - 1)
        with open(temp_file_xml_name, 'r+') as f:
            data = f.read().splitlines()
            replace = list3
            for i in replace:
                data[i] = '\t    </ImplicitLink>'
            f.seek(0)
            f.write('\n'.join(data))
            f.truncate()
        fname = join(sessiondir, UPLOAD_FOLDER,
                     splitext(temp_file_xml_name)[0] + ".xcos")
        shutil.move(temp_file_xml_name, fname)
        diagram.xcos_file_name = fname
        return diagram.diagram_id

    # List to contain all affich blocks
    blockaffich = new_xml.getElementsByTagName("AfficheBlock")
    for block in blockaffich:
        interfaceFunctionName = block.getAttribute("interfaceFunctionName")
        if interfaceFunctionName == "AFFICH_m":
            diagram.workspace_counter = 4

    # List to contain all the block IDs of tkscales so that we can create
    # read blocks with these IDs
    block_id = []
    for block in blocks:
        interfaceFunctionName = block.getAttribute("interfaceFunctionName")
        if interfaceFunctionName == "TKSCALE":
            block_id.append(block.getAttribute("id"))
            block.setAttribute('id', '-1')
            tk_is_present = True
            # Changed the ID of tkscales to -1 so that virtually the
            # tkscale blocks get disconnected from diagram at the backend
        # Taking workspace_counter 1 for TOWS_c and 2 for FROMWSB
        elif interfaceFunctionName == "scifunc_block_m":
            diagram.workspace_counter = 5
        elif interfaceFunctionName == "TOWS_c":
            if block.childNodes:
                for node in block.childNodes:
                    if not isinstance(node, minidom.Element):
                        continue
                    if node.getAttribute("as") != "exprs":
                        continue
                    if node.childNodes is None:
                        continue
                    childCount = 0
                    for childChildNode in node.childNodes:
                        if not isinstance(childChildNode, minidom.Element):
                            continue
                        childCount += 1
                        if childCount != 2:
                            continue
                        value = childChildNode.getAttribute("value")
                        if value is not None:
                            diagram.save_variables.add(value)
                        break
            diagram.workspace_counter = 1
            flag1 = 1
        elif interfaceFunctionName == "FROMWSB":
            diagram.workspace_counter = 2
            flag2 = 1
    if diagram.save_variables:
        logger.info("save variables = %s", diagram.save_variables)
    if flag1 and flag2:
        # Both TOWS_c and FROMWSB are present
        diagram.workspace_counter = 3
    # Hardcoded the real time scaling to 1.0 (i.e., no scaling of time
    # occurs) only if tkscale is present
    if tk_is_present:
        for dia in new_xml.getElementsByTagName("XcosDiagram"):
            dia.setAttribute('realTimeScaling', '1.0')

    # Save the changes made by parser
    with open(temp_file_xml_name, 'w') as f:
        f.write(new_xml.toxml())

    # In front of block tkscale printing the block corresponding to read
    # function and assigning corresponding values
    skipblock = False
    for line in fileinput.input(temp_file_xml_name, inplace=1):

        if 'interfaceFunctionName=\"TKSCALE\"' in line:
            # change the block ID
            i = diagram.tk_count
            print('<BasicBlock blockType="d" id="', block_id[i], '" '
                  'interfaceFunctionName="RFILE_f" parent="1" '
                  'simulationFunctionName="readf" '
                  'simulationFunctionType="DEFAULT" style="RFILE_f">',
                  sep='')
            print('<ScilabString as="exprs" height="5" width="1">')
            print('<data column="0" line="0" value="1"/>')
            # Value equal to 1 implies take readings from first column in
            # the file
            print('<data column="0" line="1" value="2"/>')
            # Path to the file from which read block obtains the values
            fname = join(diagram.sessiondir, VALUES_FOLDER,
                         diagram.diagram_id + "_tk" + str(i + 1) + ".txt")
            print('<data column="0" line="2" value="', fname, '"/>',
                  sep='')
            print('<data column="0" line="3" value="(2(e10.3,1x))"/>')
            # (2(e10.3,1x)) The format in which numbers are written
            # Two columns with base 10 and 3 digits after decimal and 1x
            # represents 1 unit space between two columns.
            print('<data column="0" line="4" value="2"/>')
            print('</ScilabString>')
            print('<ScilabDouble as="realParameters" '
                  'height="0" width="0"/>')
            print('<ScilabDouble as="integerParameters" '
                  'height="105" width="1">')
            diagram.tk_count += 1
            # The remaining part of the block is read from the
            # Read_Content.txt file and written to the xml file
            with open(READCONTENTFILE, "r") as read_file:
                for line_content in read_file:
                    print(line_content, end='')
            skipblock = True
        elif skipblock:
            if '</BasicBlock>' in line:
                skipblock = False
        else:
            print(line, end='')

    # Changing the file extension from xml to xcos
    fname = join(sessiondir, UPLOAD_FOLDER,
                 splitext(temp_file_xml_name)[0] + ".xcos")
    # Move the xcos file to uploads directory
    shutil.move(temp_file_xml_name, fname)
    diagram.xcos_file_name = fname
    return diagram.diagram_id


def get_diagram(session, task, remove=False):
    if not task:
        logger.warning('no id')
        return None
    task_id = str(task.task_id)

    (diagrams, __, __, __, __) = init_session(session)

    if task_id not in diagrams:
        logger.warning('id %s not in diagrams', task_id)
        return None

    diagram = diagrams[task_id]

    if remove:
        diagrams[task_id] = Diagram()

    return diagram


def add_diagram(session, task):
    task_id = str(task.task_id)

    (diagrams, scripts, __, __, sessiondir) = init_session(session)

    diagram = Diagram()
    diagram.diagram_id = task_id
    diagram.sessiondir = sessiondir
    diagrams[task_id] = diagram

    return (diagram, scripts, sessiondir)


def get_script(session, task_id, scripts=None, remove=False):
    if task_id is None:
        return None

    if scripts is None:
        (__, scripts, __, __, __) = init_session(session)

    if task_id not in scripts:
        logger.warning('id %s not in scripts', task_id)
        return None

    script = scripts[task_id]

    if remove:
        del scripts[task_id]

    return script


def add_script(session, task_id):
    (__, scripts, __, __, sessiondir) = init_session(session)

    script = Script()
    script.script_id = task_id
    script.sessiondir = sessiondir
    scripts[task_id] = script

    return (script, sessiondir)


def add_datafile(session):
    (__, __, __, datafiles, sessiondir) = init_session(session)

    datafile = DataFile()
    datafile.sessiondir = sessiondir
    datafiles.append(datafile)

    return (datafile, sessiondir, str(len(datafiles)))


def DownloadFile(request):
    '''route for download of binary and audio'''
    fn = request.form['path']
    if fn == '' or fn[0] == '.' or '/' in fn:
        logger.warning('downloadfile=%s', fn)
        return "error"
    # check if audio file or binary file
    if "audio" in fn:
        mime_type = 'audio/basic'
    else:
        mime_type = 'application/octet-stream'
    file_path = os.path.join(SESSIONDIR, fn)
    if not os.path.exists(file_path):
        raise Http404("File not found")
    return FileResponse(open(file_path, 'r'),
                        as_attachment=True, content_type=mime_type)


def DeleteFile(request):
    '''route for deletion of binary and audio file'''
    fn = request.form['path']
    if fn == '' or fn[0] == '.' or '/' in fn:
        logger.warning('deletefile=%s', fn)
        return "error"
    remove(fn)  # deleting the file
    return "0"


def get_request_id(request, key='id'):
    args = request.args
    if args is None:
        logger.warning('No args in request')
        return ''
    if key not in args:
        logger.warning('No %s in request.args', key)
        return ''
    value = args[key]
    if re.fullmatch(r'[0-9]+', value):
        return value
    displayvalue = value if len(
        value) <= DISPLAY_LIMIT + 3 else value[:DISPLAY_LIMIT] + '...'
    logger.warning('Invalid value %s for %s in request.args',
                   displayvalue, key)
    return ''


def internal_fun(session, task, internal_key):
    (__, __, scifile, __, sessiondir) = init_session(session)

    if internal_key not in config.INTERNAL:
        msg = internal_key + ' not found'
        logger.warning(msg)
        return JsonResponse({'msg': msg})
    internal_data = config.INTERNAL[internal_key]

    cmd = ""
    for scriptfile in internal_data['scriptfiles']:
        cmd += "exec('%s');" % scriptfile
    file_name = join(sessiondir, internal_key + ".txt")
    function = internal_data['function']
    parameters = internal_data['parameters']
    if 'num' in parameters:
        p = 's'
        cmd += "%s=poly(0,'%s');" % (p, p)
        p = 'z'
        cmd += "%s=poly(0,'%s');" % (p, p)
    cmd += "%s('%s'" % (function, file_name)
    task_parameters = json.loads(task.parameters)
    for parameter in parameters:
        if parameter not in task_parameters:
            msg = parameter + ' parameter is missing'
            logger.warning(msg)
            return JsonResponse({'msg': msg})
        value = task_parameters[parameter]
        try:
            value.encode('ascii')
        except UnicodeEncodeError:
            msg = parameter + ' parameter has non-ascii value'
            logger.warning(msg)
            return JsonResponse({'msg': msg})
        if re.search(SYSTEM_COMMANDS, value):
            msg = parameter + ' parameter has unsafe value'
            logger.warning(msg)
            return JsonResponse({'msg': msg})
        if re.search(SPECIAL_CHARACTERS, value):
            msg = parameter + ' parameter has value with special characters'
            logger.warning(msg)
            return JsonResponse({'msg': msg})
        if 'num' in parameters:
            cmd += ",%s" % value
        else:
            cmd += ",'%s'" % value
    cmd += ");"

    if scifile.instance is not None:
        msg = 'Cannot execute more than one script at the same time.'
        return JsonResponse({'msg': msg})

    scifile.instance = run_scilab(cmd, scifile)
    if scifile.instance is None:
        msg = "Resource not available"
        return JsonResponse({'msg': msg})

    proc = scifile.instance.proc
    (out, err) = proc.communicate()
    out = re.sub(r'^[ !\\-]*\n', r'', out, flags=re.MULTILINE)
    if out:
        logger.info('=== Output from scilab console ===\n%s', out)
    if err:
        logger.info('=== Error from scilab console ===\n%s', err)
    remove_scilab_instance(scifile.instance)
    scifile.instance = None

    if not isfile(file_name):
        msg = "Output file not available"
        logger.warning(msg)
        return JsonResponse({'msg': msg})

    with open(file_name) as f:
        data = f.read()  # Read the data into a variable

    remove(file_name)

    return data


def clean_text(s):
    return re.sub(r'[ \t]*[\r\n]+[ \t]*', r'', s)


def clean_text_2(s, forindex):
    '''handle whitespace'''
    s = re.sub(r'[\a\b\f\r\v]', r'', s)
    s = re.sub(r'\t', r'    ', s)
    s = re.sub(r' +(\n|$)', r'\n', s)
    if forindex:
        s = re.sub(r'\n+$', r'', s)
        # double each backslash
        s = re.sub(r'\\', r'\\\\', s)
        # replace each newline with '\n'
        s = re.sub(r'\n', r'\\n', s)
    else:
        s = re.sub(r'\n{2,}$', r'\n', s)
    return s


def kill_scilab(diagram=None, session=None, task=None):
    '''Define function to kill scilab(if still running) and remove files'''
    if diagram is None:
        diagram = get_diagram(session, task, True)

    if diagram is None:
        logger.warning('no diagram')
        return
    logger.info('kill_scilab: diagram=%s', diagram)

    stop_scilab_instance(diagram, True)

    if diagram.xcos_file_name is None:
        logger.warning('empty diagram')
    else:
        # Remove xcos file
        remove(diagram.xcos_file_name)
        diagram.xcos_file_name = None

    if diagram.file_image != '':
        logger.warning('not removing %s', diagram.file_image)

    stopDetailsThread(diagram)


def kill_script(script=None, session=None, task=None):
    '''Below route is called for stopping a running script file.'''
    if script is None:
        script = get_script(session, str(task.task_id), remove=True)
        if script is None:
            # when called with same script_id again or with incorrect script_id
            logger.warning('no script')
            return "error"

    logger.info('kill_script: script=%s', script)

    stop_scilab_instance(script)

    if script.filename is None:
        logger.warning('empty script')
    else:
        remove(script.filename)
        script.filename = None

    if script.workspace_file is None:
        logger.warning('empty workspace')
    else:
        remove(script.workspace_file)
        script.workspace_file = None

    return "ok"


def kill_scifile(scifile=None, session=None):
    '''Below route is called for stopping a running sci file.'''
    if scifile is None:
        (__, __, scifile, __, __) = init_session(session)

    logger.info('kill_scifile: scifile=%s', scifile)

    stop_scilab_instance(scifile)

    return "ok"


worker = None
reaper = None
cleaner = None


def start_threads():
    global worker, reaper, cleaner
    worker = gevent.spawn(prestart_scilab_instances)
    worker.name = 'PreStart'
    reaper = gevent.spawn(reap_scilab_instances)
    reaper.name = 'Reaper'
    cleaner = gevent.spawn(clean_sessions_thread)
    cleaner.name = 'Clean'


def stop_threads():
    global worker, reaper, cleaner
    gevent.kill(worker)
    worker = None
    gevent.kill(reaper)
    reaper = None
    gevent.kill(cleaner)
    cleaner = None
    clean_sessions(True)
    stop_scilab_instances()
    logger.info('exiting')