use gevent pool

6 files changed, 338 insertions, 85 deletions

diff --git a/blocks/blocks/celery_tasks.py b/blocks/blocks/celery_tasks.py
index 65c5aac2..f859ef86 100644
--- a/blocks/blocks/celery_tasks.py
+++ b/blocks/blocks/celery_tasks.py
@@ -1,62 +1,39 @@
-from simulationAPI.helpers.scilab_manager import (
-    prestart_scilab_instances,
-    reap_scilab_instances,
-    clean_sessions_thread,
-    stop_scilab_instances,
-    clean_sessions
-)
-import os
-import redis
-import django
+import sys
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'blocks.settings')
-django.setup()
-import multiprocessing
+# Apply monkey patching only if running Celery
+if "celery" in sys.argv[0]:
+    from gevent.monkey import patch_all
 
-from celery import Celery
-from celery.signals import worker_shutdown
-import gevent
+    patch_all(aggresive=False)
 
+    from simulationAPI.helpers.scilab_manager import \
+        start_threads, stop_threads
+
+import os
+from celery import Celery
+from celery.signals import worker_ready, worker_shutdown
 from django.conf import settings
 
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'blocks.settings')
 
 app = Celery('blocks')
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 app.conf.broker_connection_retry_on_startup = True
 
-SCILAB_DIR = os.path.abspath(settings.SCILAB_DIR)
-SCILAB = os.path.join(SCILAB_DIR, 'bin', 'scilab-adv-cli')
-
 
 @app.task(bind=True)
 def debug_task(self):
     print('Request: {0!r}'.format(self.request))
 
 
-# Initialize Redis
-redis_client = redis.Redis(host="localhost", port=6379, db=0)
-
-# Global keys for startup and shutdown
-STARTUP_KEY = "celery_startup_done"
-SHUTDOWN_KEY = "celery_shutdown_done"
-
-
-# ✅ Ensure startup runs **only once** using Redis lock
-if multiprocessing.current_process().name == "MainProcess":
-    if not redis_client.get(STARTUP_KEY):  # Check if startup was already executed
-        print("🚀 Running global startup code only once!")
-        redis_client.set(STARTUP_KEY, "1", ex=60)  # Optional: expire after 60s
-    else:
-        print("🚀 Global startup already executed, skipping.")
+@worker_ready.connect
+def startup_code(**kwargs):
+    print("Running global startup code")
+    start_threads()
 
 
 @worker_shutdown.connect
-def global_shutdown_code(**kwargs):
-    """Ensures shutdown logic runs only once globally using Redis."""
-    if multiprocessing.current_process().name == "MainProcess":
-        if not redis_client.get(SHUTDOWN_KEY):  # Check if shutdown already executed
-            print("🔴 Running global shutdown code only once!")
-            redis_client.set(SHUTDOWN_KEY, "1", ex=60)  # Optional: expire after 60s
-        else:
-            print("🔴 Global shutdown already executed, skipping.")
+def shutdown_code(**kwargs):
+    print("Running global shutdown code")
+    stop_threads()
diff --git a/blocks/eda-frontend/src/static/images/.gitignore b/blocks/eda-frontend/src/static/images/.gitignore
new file mode 100644
index 00000000..88cc1070
--- /dev/null
+++ b/blocks/eda-frontend/src/static/images/.gitignore
@@ -0,0 +1,2 @@
+*.jpg
+*.png
diff --git a/blocks/run.sh b/blocks/run.sh
index ddb984eb..f04437a2 100755
--- a/blocks/run.sh
+++ b/blocks/run.sh
@@ -6,7 +6,7 @@ service nginx start
 service redis-server start
 
 . env/bin/activate
-celery -A blocks.celery_tasks worker --loglevel INFO --concurrency 1 &
+celery -A blocks.celery_tasks worker -c 10 -l INFO -P gevent &
 
 cd eda-frontend
 if test "$1" = 'prod'; then
diff --git a/blocks/simulationAPI/helpers/config.py b/blocks/simulationAPI/helpers/config.py
new file mode 100644
index 00000000..8824c884
--- /dev/null
+++ b/blocks/simulationAPI/helpers/config.py
@@ -0,0 +1,13 @@
+import os
+
+# The location to keep the session data on server.
+SESSIONDIR = '/tmp/sessiondir'
+IMAGEDIR = 'images'
+SESSIONTIMEOUT = 21600
+
+# the instances
+
+SCILAB_MIN_INSTANCES = int(os.environ.get('SCILAB_MIN_INSTANCES', '1'))
+SCILAB_START_INSTANCES = int(os.environ.get('SCILAB_START_INSTANCES', '2'))
+SCILAB_MAX_INSTANCES = int(os.environ.get('SCILAB_MAX_INSTANCES', '3'))
+SCILAB_INSTANCE_RETRY_INTERVAL = int(os.environ.get('SCILAB_INSTANCE_RETRY_INTERVAL', '5'))
diff --git a/blocks/simulationAPI/helpers/ngspice_helper.py b/blocks/simulationAPI/helpers/ngspice_helper.py
index 370ba248..33203ee9 100644
--- a/blocks/simulationAPI/helpers/ngspice_helper.py
+++ b/blocks/simulationAPI/helpers/ngspice_helper.py
@@ -1,5 +1,6 @@
 import json
 import os
+from os.path import abspath, join, splitext
 import re
 import subprocess
 from celery import current_task
@@ -10,9 +11,9 @@ from tempfile import mkstemp
 from django.conf import settings
 
 logger = get_task_logger(__name__)
-XmlToXcos = os.path.join(settings.BASE_DIR, 'Xcos/XmlToXcos.sh')
-SCILAB_DIR = os.path.abspath(settings.SCILAB_DIR)
-SCILAB = os.path.join(SCILAB_DIR, 'bin', 'scilab-adv-cli')
+XmlToXcos = join(settings.BASE_DIR, 'Xcos/XmlToXcos.sh')
+SCILAB_DIR = abspath(settings.SCILAB_DIR)
+SCILAB = join(SCILAB_DIR, 'bin', 'scilab-adv-cli')
 # handle scilab startup
 SCILAB_START = (
     "try;funcprot(0);lines(0,120);"
@@ -47,7 +48,7 @@ def CreateXml(file_path, parameters, file_id):
     # Make Unique Directory for simulation to run
     Path(current_dir).mkdir(parents=True, exist_ok=True)
     try:
-        (xcosfilebase, __) = os.path.splitext(file_path)
+        (xcosfilebase, __) = splitext(file_path)
         xcosfile = xcosfilebase + '.xcos'
         logger.info('will run %s %s', 'XmlToXcos', file_path)
         proc = subprocess.Popen([XmlToXcos, file_path],
@@ -72,7 +73,7 @@ def CreateXml(file_path, parameters, file_id):
         target = os.listdir(current_dir)
         for item in target:
             logger.info('removing %s', item)
-            os.remove(os.path.join(current_dir, item))
+            os.remove(join(current_dir, item))
         logger.info('removing %s', current_dir)
         os.rmdir(current_dir)
         logger.info('Deleted Files')
@@ -135,6 +136,7 @@ def ExecXml(file_obj):
                 out = '\n'.join(re.split(r'\n+', out, maxlines + 1)[:maxlines])
                 logger.info('out=%s', out)
         if err:
+            err = re.sub(r'Undefined variable: helpbrowser_update', '', err)
             err = err.rstrip()
             if err:
                 err = '\n'.join(re.split(r'\n+', err, maxlines + 1)[:maxlines])
@@ -151,7 +153,7 @@ def ExecXml(file_obj):
         target = os.listdir(current_dir)
         for item in target:
             logger.info('removing %s', item)
-            os.remove(os.path.join(current_dir, item))
+            os.remove(join(current_dir, item))
         logger.info('removing %s', current_dir)
         os.rmdir(current_dir)
         logger.info('Deleted Files')
diff --git a/blocks/simulationAPI/helpers/scilab_manager.py b/blocks/simulationAPI/helpers/scilab_manager.py
index 535b57d4..06fc0c4a 100644
--- a/blocks/simulationAPI/helpers/scilab_manager.py
+++ b/blocks/simulationAPI/helpers/scilab_manager.py
@@ -1,23 +1,77 @@
-from gevent.monkey import patch_all
-
-patch_all(aggressive=False, subprocess=True)
-
+from datetime import datetime
+from django.conf import settings
+import gevent
+from gevent.event import Event
+from gevent.lock import RLock
+import glob
 import os
+from os.path import abspath, exists, join
 import re
 import time
 import signal
 import logging
 import subprocess
-import gevent
-from gevent.event import Event
-from gevent.lock import RLock
+from tempfile import mkstemp
 from threading import current_thread
 
+from simulationAPI.helpers import config
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'blocks.settings')
+
+SCILAB_DIR = abspath(settings.SCILAB_DIR)
+SCILAB = join(SCILAB_DIR, 'bin', 'scilab-adv-cli')
+BASEDIR = abspath('src/static')
+IMAGEDIR = join(BASEDIR, config.IMAGEDIR)
 
-SCILAB_MIN_INSTANCES = int(os.environ.get('SCILAB_MIN_INSTANCES', '1'))
-SCILAB_MAX_INSTANCES = int(os.environ.get('SCILAB_MAX_INSTANCES', '3'))
-SCILAB_START_INSTANCES = int(os.environ.get('SCILAB_START_INSTANCES', '2'))
-SCILAB_INSTANCE_RETRY_INTERVAL = int(os.environ.get('SCILAB_INSTANCE_RETRY_INTERVAL', '15'))
+
+SESSIONDIR = abspath(config.SESSIONDIR)
+
+VALUES_FOLDER = 'values'  # to store files related to tkscale block
+
+# Delay time to look for new line (in s)
+LOOK_DELAY = 0.1
+
+SCILAB_START = (
+    "try;funcprot(0);lines(0,120);"
+    "clearfun('messagebox');"
+    "function messagebox(msg,title,icon,buttons,modal),disp(msg),endfunction;"
+    "funcprot(1);"
+    "catch;[error_message,error_number,error_line,error_func]=lasterror();"
+    "disp(error_message,error_number,error_line,error_func);exit(3);end;"
+)
+SCILAB_END = (
+    "catch;[error_message,error_number,error_line,error_func]=lasterror();"
+    "disp(error_message,error_number,error_line,error_func);exit(2);end;exit;"
+)
+
+SCILAB_CMD = [SCILAB,
+              "-noatomsautoload",
+              "-nogui",
+              "-nouserstartup",
+              "-nb",
+              "-nw",
+              "-e", SCILAB_START]
+
+USER_DATA = {}
+
+
+def makedirs(dirname, dirtype):
+    if not exists(dirname):
+        os.makedirs(dirname)
+
+
+def remove(filename):
+    if filename is None:
+        return False
+    if not config.REMOVEFILE:
+        logger.debug('not removing %s', filename)
+        return True
+    try:
+        os.remove(filename)
+        return True
+    except BaseException:
+        logger.error('could not remove %s', filename)
+        return False
 
 
 # Configure logger
@@ -35,6 +89,8 @@ console_handler.setFormatter(formatter)
 # Add the handler to the logger
 logger.addHandler(console_handler)
 
+makedirs(SESSIONDIR, 'top session')
+
 
 class ScilabInstance:
     proc = None
@@ -50,22 +106,59 @@ class ScilabInstance:
         return "{pid: %s, log_name: %s}" % (self.proc.pid, self.log_name)
 
 
+class Diagram:
+    diagram_id = None
+    # session dir
+    sessiondir = None
+    # store uploaded filename
+    xcos_file_name = None
+    # type of uploaded file
+    workspace_counter = 0
+    save_variables = set()
+    # workspace from script
+    workspace_filename = None
+    # tk count
+    tk_count = 0
+    # store log name
+    instance = None
+    # is thread running?
+    tkbool = False
+    tk_starttime = None
+    # in memory values
+    tk_deltatimes = None
+    tk_values = None
+    tk_times = None
+    # List to store figure IDs from log_name
+    figure_list = None
+    file_image = ''
+
+    def __init__(self):
+        self.figure_list = []
+
+    def __str__(self):
+        return "{instance: %s, tkbool: %s, figure_list: %s}" % (
+            self.instance, self.tkbool, self.figure_list)
+
+    def clean(self):
+        if self.instance is not None:
+            kill_scilab(self)
+            self.instance = None
+        if self.xcos_file_name is not None:
+            remove(self.xcos_file_name)
+            self.xcos_file_name = None
+        if self.workspace_filename is not None:
+            remove(self.workspace_filename)
+            self.workspace_filename = None
+        if self.file_image != '':
+            remove(join(IMAGEDIR, self.file_image))
+            self.file_image = ''
+
+
 INSTANCES_1 = []
 INSTANCES_2 = []
 evt = Event()
 
 
-def prestart_scilab():
-    try:
-        proc = subprocess.Popen(["scilab-adv-cli", "-noatomsautoload", "-nb"],
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        log_name = "scilab_log.txt"
-        return proc, log_name
-    except Exception as e:
-        print("Error starting Scilab:", e)
-        return None, None
-
-
 def no_free_scilab_instance():
     l1 = len(INSTANCES_1)
     return l1 == 0
@@ -74,15 +167,15 @@ def no_free_scilab_instance():
 def too_many_scilab_instances():
     l1 = len(INSTANCES_1)
     l2 = len(INSTANCES_2)
-    return l1 >= SCILAB_MIN_INSTANCES or \
-        l1 + l2 >= SCILAB_MAX_INSTANCES
+    return l1 >= config.SCILAB_MIN_INSTANCES or \
+        l1 + l2 >= config.SCILAB_MAX_INSTANCES
 
 
 def start_scilab_instances():
     l1 = len(INSTANCES_1)
     l2 = len(INSTANCES_2)
-    lssi = min(SCILAB_START_INSTANCES,
-               SCILAB_MAX_INSTANCES - l2) - l1
+    lssi = min(config.SCILAB_START_INSTANCES,
+               config.SCILAB_MAX_INSTANCES - l2) - l1
     if lssi > 0:
         logger.info('can start %s instances', lssi)
     return lssi
@@ -102,15 +195,6 @@ def print_scilab_instances():
 FIRST_INSTANCE = True
 
 
-def clean_sessions_thread():
-    # Ensure this function exists
-    print("Cleaning sessions...")
-
-
-def clean_sessions(force=False):
-    print("Cleaning sessions...")
-
-
 def prestart_scilab_instances():
     global FIRST_INSTANCE
 
@@ -164,7 +248,7 @@ def prestart_scilab_instances():
 
                 logger.error('retrying after %s %s: rc = %s',
                              attempt, msg, returncode)
-                gevent.sleep(SCILAB_INSTANCE_RETRY_INTERVAL * attempt)
+                gevent.sleep(config.SCILAB_INSTANCE_RETRY_INTERVAL * attempt)
                 attempt += 1
                 FIRST_INSTANCE = True
                 continue
@@ -292,6 +376,181 @@ def reap_scilab_instances():
             if base is None:
                 logger.warning('cannot stop instance %s', instance)
                 stop_instance(instance)
+            elif isinstance(base, Diagram):
+                kill_scilab(base)
             else:
                 logger.warning('cannot stop instance %s', instance)
                 stop_instance(instance)
+
+
+def clean_sessions(final=False):
+    current_thread().name = 'Clean'
+    totalcount = 0
+    cleanuids = []
+    for uid, ud in USER_DATA.items():
+        totalcount += 1
+        if final or time() - ud.timestamp > config.SESSIONTIMEOUT:
+            cleanuids.append(uid)
+
+    logger.info('cleaning %s/%s sessions', len(cleanuids), totalcount)
+    for uid in cleanuids:
+        current_thread().name = 'Clean-%s' % uid[:6]
+        try:
+            logger.info('cleaning')
+            ud = USER_DATA.pop(uid)
+            ud.clean()
+        except Exception as e:
+            logger.warning('could not clean: %s', str(e))
+
+
+def clean_sessions_thread():
+    current_thread().name = 'Clean'
+    while True:
+        gevent.sleep(config.SESSIONTIMEOUT / 2)
+        try:
+            clean_sessions()
+        except Exception as e:
+            logger.warning('Exception in clean_sessions: %s', str(e))
+
+
+logfilefdrlock = RLock()
+LOGFILEFD = 123
+
+
+def prestart_scilab():
+    cmd = SCILAB_START
+    cmdarray = [SCILAB,
+                "-nogui",
+                "-noatomsautoload",
+                "-nouserstartup",
+                "-nb",
+                "-nw",
+                "-e", cmd]
+
+    logfilefd, log_name = mkstemp(prefix=datetime.now().strftime(
+        'scilab-log-%Y%m%d-'), suffix='.txt', dir=SESSIONDIR)
+
+    with logfilefdrlock:
+        if logfilefd != LOGFILEFD:
+            os.dup2(logfilefd, LOGFILEFD)
+            os.close(logfilefd)
+
+        try:
+            proc = subprocess.Popen(
+                cmdarray,
+                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, start_new_session=True,
+                universal_newlines=True, pass_fds=(LOGFILEFD, ))
+        except FileNotFoundError:
+            logger.critical('scilab has not been built. '
+                            'Follow the installation instructions')
+            proc = None
+            remove(log_name)
+            log_name = None
+
+        os.close(LOGFILEFD)
+
+    return (proc, log_name)
+
+
+def run_scilab(command, base, createlogfile=False, timeout=70):
+    instance = get_scilab_instance()
+    if instance is None:
+        logger.error('cannot run command %s', command)
+        return None
+
+    cmd = command + SCILAB_END
+    logger.info('running command %s', cmd)
+    instance.proc.stdin.write(cmd)
+
+    if not createlogfile:
+        remove(instance.log_name)
+        instance.log_name = None
+
+    instance.base = base
+    instance.starttime = time()
+    instance.endtime = time() + timeout
+    return instance
+
+
+def stopDetailsThread(diagram):
+    diagram.tkbool = False  # stops the thread
+    gevent.sleep(LOOK_DELAY)
+    fname = join(diagram.sessiondir, VALUES_FOLDER,
+                 diagram.diagram_id + "_*")
+    for fn in glob.glob(fname):
+        # deletes all files created under the 'diagram_id' name
+        remove(fn)
+
+
+def get_diagram(xcos_file_id, remove=False):
+    if not xcos_file_id:
+        logger.warning('no id')
+        return None
+    xcos_file_id = int(xcos_file_id)
+
+    (diagrams, __, __, __, __, __, __) = init_session()
+
+    if xcos_file_id < 0 or xcos_file_id >= len(diagrams):
+        logger.warning('id %s not in diagrams', xcos_file_id)
+        return None
+
+    diagram = diagrams[xcos_file_id]
+
+    if remove:
+        diagrams[xcos_file_id] = Diagram()
+
+    return diagram
+
+
+def kill_scilab(task_id, diagram=None):
+    '''Define function to kill scilab(if still running) and remove files'''
+    if diagram is None:
+        diagram = get_diagram(task_id, True)
+
+    if diagram is None:
+        logger.warning('no diagram')
+        return
+    logger.info('kill_scilab: diagram=%s', diagram)
+
+    stop_scilab_instance(diagram, True)
+
+    if diagram.xcos_file_name is None:
+        logger.warning('empty diagram')
+    else:
+        # Remove xcos file
+        remove(diagram.xcos_file_name)
+        diagram.xcos_file_name = None
+
+    if diagram.file_image != '':
+        logger.warning('not removing %s', diagram.file_image)
+
+    stopDetailsThread(diagram)
+
+
+worker = None
+reaper = None
+cleaner = None
+
+
+def start_threads():
+    global worker, reaper, cleaner
+    worker = gevent.spawn(prestart_scilab_instances)
+    worker.name = 'PreStart'
+    reaper = gevent.spawn(reap_scilab_instances)
+    reaper.name = 'Reaper'
+    cleaner = gevent.spawn(clean_sessions_thread)
+    cleaner.name = 'Clean'
+
+
+def stop_threads():
+    global worker, reaper, cleaner
+    gevent.kill(worker)
+    worker = None
+    gevent.kill(reaper)
+    reaper = None
+    gevent.kill(cleaner)
+    cleaner = None
+    clean_sessions(True)
+    stop_scilab_instances()
+    logger.info('exiting')