added watchdog

Author: rok

Dockerfile

@@ -20,6 +20,7 @@ WORKDIR /etc/supervisor/conf.d
 COPY laravel-worker.conf.tpl /etc/supervisor/conf.d/laravel-worker.conf.tpl
 COPY laravel-horizon.conf.tpl /etc/supervisor/conf.d/laravel-horizon.conf.tpl
 COPY custom-php.ini.tpl /opt/etc/custom-php.ini.tpl
+COPY supervisor-watchdog.py /opt/supervisor-watchdog.py
 
 # Copy scripts
 COPY init.sh /usr/local/bin/init.sh

init.sh

@@ -27,4 +27,4 @@ if [ "$LARAVEL_HORIZON" = true ]; then
 	cp /etc/supervisor/conf.d/laravel-horizon.conf.tpl /etc/supervisor/supervisord.conf
 fi
 
-supervisord --nodaemon --configuration /etc/supervisor/supervisord.conf
+exec supervisord --nodaemon --configuration /etc/supervisor/supervisord.conf

laravel-horizon.conf.tpl

@@ -10,3 +10,7 @@ autostart=true
 autorestart=true
 stdout_events_enabled=1
 redirect_stderr=true
+
+[eventlistener:supervisord-watchdog]
+command=/usr/bin/python /opt/supervisord-watchdog.py
+events=PROCESS_STATE_FATAL

laravel-worker.conf.tpl

@@ -12,3 +12,7 @@ numprocs=1
 startretries=10
 stdout_events_enabled=1
 redirect_stderr=true
+
+[eventlistener:supervisord-watchdog]
+command=/usr/bin/python /opt/supervisord-watchdog.py
+events=PROCESS_STATE_FATAL

supervisor-watchdog.py

@@ -0,0 +1,43 @@
+import sys
+import os
+import logging
+import subprocess
+import time
+
+from supervisor.childutils import listener
+
+def main(args):
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(filename)s: %(message)s')
+    logger = logging.getLogger("supervisord-watchdog")
+    debug_mode = True if 'DEBUG' in os.environ else False
+
+    while True:
+        logger.info("Listening for events...")
+        headers, body = listener.wait(sys.stdin, sys.stdout)
+        body = dict([pair.split(":") for pair in body.split(" ")])
+
+        logger.debug("Headers: %r", repr(headers))
+        logger.debug("Body: %r", repr(body))
+        logger.debug("Args: %r", repr(args))
+
+        if debug_mode: continue
+
+        try:
+            if headers["eventname"] == "PROCESS_STATE_FATAL":
+                logger.info("Process entered FATAL state...")
+                if not args or body["processname"] in args:
+                    logger.error("Killing off supervisord instance ...")
+                    res = subprocess.call(["/usr/bin/pkill", "-15", "supervisord"], stdout=sys.stderr)
+                    logger.info("Sent TERM signal to init process")
+                    time.sleep( 5 )
+                    logger.critical("Why am I still alive? Send KILL to all processes...")
+                    res = subprocess.call(["/usr/bin/pkill", "-9", "supervisord"], stdout=sys.stderr)
+        except Exception as e:
+            logger.critical("Unexpected Exception: %s", str(e))
+            listener.fail(sys.stdout)
+            exit(1)
+        else:
+            listener.ok(sys.stdout)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])