ENT-14434: Made reactor-plugin into its own forked process

cf-reactor/cf-reactor.c

@@ -33,13 +33,23 @@
 #include <man.h>
 #include <cleanup.h>
 #include <prototypes3.h>
+#include <signals.h>            /* HandleSignalsForDaemon, IsPendingTermination */
+#include <unistd.h>             /* sleep */
+#include <signal.h>             /* signal, kill */
+#include <errno.h>              /* errno, EINTR */
+#include <sys/wait.h>           /* waitpid */
+#include <exec_tools.h>
 
 /*****************************************************************************/
 /* Globals                                                                   */
 /*****************************************************************************/
 
 int NO_FORK = false;
 
+
+#define REACTOR_RESTART_LIMIT   10
+#define REACTOR_MIN_UPTIME_SECS 60
+
 /*******************************************************************/
 /* Command line options                                            */
 /*******************************************************************/
@@ -177,6 +187,74 @@ static GenericAgentConfig *CheckOpts(int argc, char **argv)
     return config;
 }
 
+#ifndef __MINGW32__
+
+static bool ReactorEnterpriseHasExited(pid_t pid)
+{
+    int status;
+    pid_t ret = waitpid(pid, &status, WNOHANG);
+    if (ret == 0)
+    {
+        return false;
+    }
+
+    if (ret == -1)
+    {
+        Log(LOG_LEVEL_ERR,
+            "Failed to check status of cf-reactor enterprise process %jd (waitpid: %s)",
+            (intmax_t) pid, GetErrorStr());
+        return true;
+    }
+
+    if (WIFEXITED(status))
+    {
+        Log(LOG_LEVEL_ERR, "cf-reactor enterprise process %jd exited unexpectedly with code %d",
+            (intmax_t) pid, WEXITSTATUS(status));
+    }
+    else if (WIFSIGNALED(status))
+    {
+        Log(LOG_LEVEL_ERR, "cf-reactor enterprise process %jd was killed by signal %d",
+            (intmax_t) pid, WTERMSIG(status));
+    }
+    else
+    {
+        Log(LOG_LEVEL_ERR, "cf-reactor enterprise process %jd terminated abnormally",
+            (intmax_t) pid);
+    }
+
+    return true;
+}
+
+#endif /* !__MINGW32__ */
+
+static void TerminateReactorEnterprise(int pid)
+{
+#ifndef __MINGW32__
+    if (kill((pid_t) pid, SIGINT) == -1)
+    {
+        Log(LOG_LEVEL_ERR, "Failed to signal cf-reactor enterprise process %jd (kill: %s)",
+            (intmax_t) pid, GetErrorStr());
+        return;
+    }
+
+    int status;
+    pid_t ret;
+    while (((ret = waitpid(pid, &status, 0)) == -1) && (errno == EINTR))
+    {
+        /* Interrupted by a signal, try again. */
+    }
+
+    if (ret == -1)
+    {
+        Log(LOG_LEVEL_ERR,
+            "Failed to wait for cf-reactor enterprise process %jd to terminate (waitpid: %s)",
+            (intmax_t) pid, GetErrorStr());
+    }
+#else
+    (void) pid;
+#endif /* !__MINGW32__ */
+}
+
 /*****************************************************************************/
 
 int main(int argc, char *argv[])
@@ -185,10 +263,118 @@ int main(int argc, char *argv[])
     EvalContext *ctx = EvalContextNew();
     GenericAgentConfigApply(ctx, config);
 
-    int ret = ReactorEnterpriseMain(NO_FORK);
+#ifdef __MINGW32__
+
+    if (!NO_FORK)
+    {
+        Log(LOG_LEVEL_VERBOSE, "Windows does not support starting processes in the background - starting in foreground");
+    }
+
+#else /* !__MINGW32__ */
+    pid_t existing_pid = ReadPID("cf-reactor.pid");
+    if ((existing_pid != -1) && (kill(existing_pid, 0) == 0))
+    {
+        Log(LOG_LEVEL_ERR, "Another instance of cf-reactor is already running, terminating");
+        return 1;
+    }
+
+    if ((!NO_FORK) && (fork() != 0))
+    {
+        Log(LOG_LEVEL_INFO, "cf-reactor: starting");
+        _exit(EXIT_SUCCESS);
+    }
+
+    if (!NO_FORK)
+    {
+        ActAsDaemon();
+    }
+
+#endif /* !__MINGW32__ */
+
+    umask(077);
+    WritePID("cf-reactor.pid");
+
+    signal(SIGINT, HandleSignalsForDaemon);
+    signal(SIGTERM, HandleSignalsForDaemon);
+    signal(SIGBUS, HandleSignalsForDaemon);
+    signal(SIGHUP, HandleSignalsForDaemon);
+    signal(SIGUSR1, HandleSignalsForDaemon);
+    signal(SIGUSR2, HandleSignalsForDaemon);
+    signal(SIGPIPE, SIG_IGN);
+
+    int child = ReactorEnterpriseMain();
+    if (child == -1)
+    {
+        return 1;
+    }
+
+#ifndef __MINGW32__
+    time_t child_started_at = time(NULL);
+    int n_restarts = 0;
+#endif /* !__MINGW32__ */
+
+    while (!IsPendingTermination())
+    {
+        /* Do something */
+        sleep(1);
+
+#ifndef __MINGW32__
+        /* Here cf-reactor tries to restart the reactor plugin if it exited. It retries
+         * 10 times before giving up. The counter is set back to 0 after some time without failure */
+        if ((child > 0) && ReactorEnterpriseHasExited((pid_t) child))
+        {
+            if (IsPendingTermination())
+            {
+                /* Already shutting down, no point in restarting it. */
+                child = 0;
+                break;
+            }
+
+            if (time(NULL) - child_started_at > REACTOR_MIN_UPTIME_SECS)
+            {
+                /* It ran for a while before dying, don't hold that against it. */
+                n_restarts = 0;
+            }
+
+            if (n_restarts >= REACTOR_RESTART_LIMIT)
+            {
+                Log(LOG_LEVEL_CRIT,
+                    "cf-reactor enterprise process has died %d times in a row, "
+                    "giving up on restarting it. cf-reactor will keep running "
+                    "without the enterprise reactor extension until restarted",
+                    n_restarts);
+                child = 0;
+                continue;
+            }
+
+            n_restarts++;
+            Log(LOG_LEVEL_ERR,
+                "Restarting cf-reactor enterprise process (attempt %d/%d)",
+                n_restarts, REACTOR_RESTART_LIMIT);
+
+            child = ReactorEnterpriseMain();
+            child_started_at = time(NULL);
+            if (child == -1)
+            {
+                Log(LOG_LEVEL_ERR, "Failed to restart cf-reactor enterprise process, "
+                    "will keep running without the enterprise reactor extension");
+                child = 0;
+                continue;
+            }
+        }
+#endif /* !__MINGW32__ */
+    }
+
+    /* child == 0 means either that no child process was created (the
+       default value returned by enterprise stubs), or that we already gave
+       up on / reaped it above. */
+    if (child > 0)
+    {
+        TerminateReactorEnterprise(child);
+    }
 
     GenericAgentFinalize(ctx, config);
     CallCleanupFunctions();
 
-    return ret;
+    return 0;
 }

libpromises/enterprise_stubs.c

@@ -232,7 +232,7 @@ ENTERPRISE_VOID_FUNC_2ARG_DEFINE_STUB(void, Nova_ClassHistoryEnable,
 {
 }
 
-ENTERPRISE_FUNC_1ARG_DEFINE_STUB(int, ReactorEnterpriseMain, ARG_UNUSED bool, no_fork)
+ENTERPRISE_FUNC_0ARG_DEFINE_STUB(int, ReactorEnterpriseMain)
 {
     Log(LOG_LEVEL_VERBOSE, "Nova extension library is not available.");
     Log(LOG_LEVEL_VERBOSE, "Running cf-reactor community edition.");

libpromises/prototypes3.h

@@ -85,7 +85,7 @@ ENTERPRISE_VOID_FUNC_0ARG_DECLARE(void, ReloadHAConfig);
 ENTERPRISE_VOID_FUNC_2ARG_DECLARE(void, Nova_ClassHistoryAddContextName, const StringSet *, list, const char *, context_name);
 ENTERPRISE_VOID_FUNC_2ARG_DECLARE(void, Nova_ClassHistoryEnable, StringSet **, list, bool, enable);
 
-ENTERPRISE_FUNC_1ARG_DECLARE(int, ReactorEnterpriseMain, bool, no_fork);
+ENTERPRISE_FUNC_0ARG_DECLARE(int, ReactorEnterpriseMain);
 
 /* manual.c */