RTOP-302: Stop the PLC when a task is stuck in its scan

core/src/CMakeLists.txt

@@ -43,13 +43,13 @@ add_executable(plc_main
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/located_globals.c
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/journal_buffer.c
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/debug_write_journal.cpp
-    ${CMAKE_SOURCE_DIR}/core/src/plc_app/plc_io_cycle.cpp
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/plc_retain.cpp
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/plc_retain_file_store.cpp
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/plc_state_manager.cpp
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/plc_switch.c
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/plcapp_manager.c
     ${CMAKE_SOURCE_DIR}/core/src/plc_app/scan_cycle_manager.c
+    ${CMAKE_SOURCE_DIR}/core/src/plc_app/task_policy.c
     ${CMAKE_SOURCE_DIR}/core/src/drivers/plugin_driver.c
     ${CMAKE_SOURCE_DIR}/core/src/drivers/plugin_config.c
     ${CMAKE_SOURCE_DIR}/core/src/drivers/vpp_plugin_seal.c

core/src/plc_app/debug_write_journal.cpp

@@ -31,6 +31,7 @@
 
 #include "image_tables.h"  /* ext_strucpp_debug_set / _write / _locate */
 #include "journal_buffer.h" /* journal_write_* / journal_force_set/clear  */
+#include "utils/rt_mutex.h"
 
 extern "C" {
 #include "utils/log.h"
@@ -61,6 +62,11 @@ std::atomic<size_t> g_dbgw_count{0};
 pthread_mutex_t     g_dbgw_lock = PTHREAD_MUTEX_INITIALIZER;
 bool                g_overflow_logged = false;
 
+__attribute__((constructor)) void dbgw_lock_init_pi(void)
+{
+    rt_mutex_upgrade_static(&g_dbgw_lock, "g_dbgw_lock");
+}
+
 /* LocatedArea (strucpp_abi.hpp): Input=0, Output=1, Memory=2.
  * LocatedSize: Bit=0, Byte=1, Word=2, DWord=3, LWord=4. */
 enum { AREA_INPUT = 0, AREA_OUTPUT = 1, AREA_MEMORY = 2 };

core/src/plc_app/image_tables.cpp

@@ -694,6 +694,21 @@ void image_tables_fill_null_pointers(void)
     log_info("[image_tables] filled %d NULL slots with backing buffers", filled);
 }
 
+void image_tables_zero_outputs(void)
+{
+    for (int i = 0; i < BUFFER_SIZE; ++i)
+    {
+        for (int b = 0; b < 8; ++b)
+        {
+            if (bool_output[i][b]) *bool_output[i][b] = 0;
+        }
+        if (byte_output[i]) *byte_output[i] = 0;
+        if (int_output[i])  *int_output[i]  = 0;
+        if (dint_output[i]) *dint_output[i] = 0;
+        if (lint_output[i]) *lint_output[i] = 0;
+    }
+}
+
 void image_tables_clear_null_pointers(void)
 {
     // Threaded process-image state: free the dirty-diff snapshot. (The mutexes

core/src/plc_app/image_tables.h

@@ -153,6 +153,15 @@ extern "C"
      * --------------------------------------------------------------------- */
     void image_tables_clear_null_pointers(void);
 
+    /**
+     * @brief Write 0 to every output image slot (%QX, %QB, %QW, %QD, %QL).
+     *
+     * Used on every stop so plugins push de-energised outputs to the hardware
+     * before they are stopped. Program storage is not touched. Caller must hold
+     * the image-tables mutex.
+     */
+    void image_tables_zero_outputs(void);
+
     /* -------------------------------------------------------------------------
      * Image-tables mutex accessor. Returns a pointer to the runtime-owned
      * recursive PI mutex that protects the image tables. The runtime locks

core/src/plc_app/plc_main.c

@@ -5,6 +5,7 @@
 #include <Python.h>
 
 #include <dlfcn.h>
+#include <errno.h>
 #include <pthread.h>
 #include <signal.h>
 #include <stdatomic.h>
@@ -20,6 +21,7 @@
 #include "plc_state_manager.h"
 #include "plc_switch.h"
 #include "plcapp_manager.h"
+#include "task_policy.h"
 #include "unix_socket.h"
 #include "utils/log.h"
 #include "utils/utils.h"
@@ -53,6 +55,7 @@ int main(int argc, char *argv[])
 {
     bool print_debug = false;
     bool safe_mode   = false;
+    bool after_fault = false;
 
     // Check for command line arguments
     for (int i = 1; i < argc; i++)
@@ -69,6 +72,10 @@ int main(int argc, char *argv[])
         {
             safe_mode = true;
         }
+        else if (strcmp(argv[i], "--fault") == 0)
+        {
+            after_fault = true;
+        }
     }
 
     // Initialize logging system
@@ -124,6 +131,29 @@ int main(int argc, char *argv[])
     // and plc_set_state() is now the body of a claimed transition rather than a
     // setter -- calling it with nothing loaded would just log a failed unload.
 
+    bool skip_outputs_off = false;
+    if (access(PLC_WATCHDOG_FAULT_MARKER, F_OK) == 0)
+    {
+        char reason[512] = {0};
+        FILE *marker     = fopen(PLC_WATCHDOG_FAULT_MARKER, "r");
+        if (marker)
+        {
+            size_t n  = fread(reason, 1, sizeof(reason) - 1, marker);
+            reason[n] = '\0';
+            fclose(marker);
+        }
+        skip_outputs_off = strstr(reason, PLC_FAULT_CONTEXT_BOOT_OUTPUTS_OFF) != NULL;
+        if (unlink(PLC_WATCHDOG_FAULT_MARKER) != 0)
+            log_warn("Could not remove %s: %s", PLC_WATCHDOG_FAULT_MARKER, strerror(errno));
+        safe_mode   = true;
+        after_fault = true;
+    }
+    if (after_fault && !safe_mode)
+    {
+        log_warn("--fault is only honoured together with --safe-mode; ignoring it");
+        after_fault = false;
+    }
+
     // Initialize watchdog
     if (watchdog_init() != 0)
     {
@@ -176,6 +206,31 @@ int main(int argc, char *argv[])
         }
     }
 
+    // Before the socket exists, so no command can claim a transition underneath.
+    if (safe_mode)
+    {
+        log_info("Runtime started in SAFE MODE - PLC program will not be loaded");
+        log_info("Upload a corrected program to recover");
+        if (after_fault)
+        {
+            log_error("Previous run ended in an unrecoverable watchdog fault");
+            plc_force_error_state();
+            if (skip_outputs_off)
+            {
+                log_error("Outputs not driven off: the previous attempt did not complete");
+            }
+            else if (plc_claim_transition(PLC_STATE_STOPPED))
+            {
+                // Bounded by the watchdog's stop budget; the context breaks a restart loop.
+                watchdog_set_fault_context(PLC_FAULT_CONTEXT_BOOT_OUTPUTS_OFF);
+                if (!plc_outputs_off_without_program())
+                    log_error("Outputs could not be driven off after the watchdog fault");
+                watchdog_set_fault_context(NULL);
+                plc_publish_final_state(PLC_STATE_ERROR);
+            }
+        }
+    }
+
     // Start the command socket only now that the plugin driver is fully built.
     // Everything the socket can ask for -- START, STOP, PLUGIN_CMD, STATS --
     // reaches into the driver, so serving commands before this point was serving
@@ -197,11 +252,6 @@ int main(int argc, char *argv[])
     // finishes, causing two concurrent load_plc_program() calls — and two
     // dispatcher threads. plc_begin_transition() also makes the start
     // asynchronous, which is fine: the main thread just sleeps below.
-    if (safe_mode)
-    {
-        log_info("Runtime started in SAFE MODE - PLC program will not be loaded");
-        log_info("Upload a corrected program to recover");
-    }
     // Same gate as any other start, but note what it can and cannot see. A VPP
     // plugin that owns a physical mode switch is initialised as part of loading
     // the program — inside the start transition below — so at this point the
@@ -212,12 +262,12 @@ int main(int argc, char *argv[])
     // reconciliation stops the PLC as soon as the start lands. Safe, but the gate
     // only bites here for a switch position already known at this point (e.g. one
     // reported by a plugin the runtime loaded independently of the program).
-    else if (!plc_switch_allows_run())
+    if (!safe_mode && !plc_switch_allows_run())
     {
         log_info("Hardware mode switch is in STOP - PLC left stopped");
         log_info("Move the switch to RUN to start the PLC");
     }
-    else if (!plc_begin_transition(PLC_STATE_RUNNING))
+    else if (!safe_mode && !plc_begin_transition(PLC_STATE_RUNNING))
     {
         log_error("Failed to initiate PLC start");
     }

core/src/plc_app/plc_retain_file_store.cpp

@@ -9,6 +9,7 @@
 #include "plc_retain_file_store.h"
 
 #include "plc_retain.h"  // PLC_RETAIN_PROGRAM_ID_LEN — one definition for both sides
+#include "utils/rt_mutex.h"
 
 #include <atomic>
 #include <mutex>
@@ -43,7 +44,7 @@ static_assert(PROGRAM_ID_LEN == PLC_RETAIN_PROGRAM_ID_LEN,
               "identity the runtime hands to read() — a shorter or longer header would be "
               "indistinguishable from a torn write and every load would discard good values.");
 
-std::mutex           g_lock;
+RtMutex              g_lock;
 std::vector<uint8_t> g_pending;
 bool                 g_dirty = false;
 
@@ -194,7 +195,7 @@ void commit(const uint8_t *buf, uint16_t len, const std::string &program_id)
 void discard_stored()
 {
     {
-        std::lock_guard<std::mutex> guard(g_lock);
+        std::lock_guard<RtMutex> guard(g_lock);
         g_pending.clear();
         g_dirty = false;
     }
@@ -222,7 +223,7 @@ void flush_loop()
              * save() every cycle and must never wait on a disk write. The
              * identity is snapshotted with the bytes so the pair committed
              * below is the pair that was current at this instant. */
-            std::lock_guard<std::mutex> guard(g_lock);
+            std::lock_guard<RtMutex> guard(g_lock);
             if (!g_dirty) continue;
             snapshot    = g_pending;
             snapshot_id = g_program_md5;
@@ -254,7 +255,7 @@ void plc_retain_file_store_stop(void)
         if (g_flusher.joinable()) g_flusher.join();
 
         /* Final flush: a clean stop should not discard the last interval. */
-        std::lock_guard<std::mutex> guard(g_lock);
+        std::lock_guard<RtMutex> guard(g_lock);
         if (g_dirty && !g_pending.empty())
         {
             commit(g_pending.data(), (uint16_t)g_pending.size(), g_program_md5);
@@ -278,7 +279,7 @@ int plc_retain_file_store_save(const uint8_t *blob, uint16_t len)
 {
     if (!g_enabled.load() || !blob || len == 0 || len > RETAIN_MAX) return -1;
 
-    std::lock_guard<std::mutex> guard(g_lock);
+    std::lock_guard<RtMutex> guard(g_lock);
     /* Only mark dirty on an actual change. The runtime deliberately does not
      * diff — it cannot know what a write costs here — so doing it at this layer
      * is how a slow medium avoids rewriting an unchanged blob every interval. */
@@ -301,7 +302,7 @@ int plc_retain_file_store_load(const char *program_md5, uint16_t md5_len, uint8_
      * a store that just discarded a previous program's values still has to
      * label the new program's first commit. */
     {
-        std::lock_guard<std::mutex> guard(g_lock);
+        std::lock_guard<RtMutex> guard(g_lock);
         g_program_md5.assign(program_md5, md5_len);
     }
 
@@ -337,7 +338,7 @@ int plc_retain_file_store_load(const char *program_md5, uint16_t md5_len, uint8_
 
     /* Prime the in-memory copy so the first flush after start does not rewrite
      * a byte-identical file. */
-    std::lock_guard<std::mutex> guard(g_lock);
+    std::lock_guard<RtMutex> guard(g_lock);
     g_pending.assign(out, out + n);
     g_dirty = false;
     return 0;
@@ -349,7 +350,7 @@ int plc_retain_file_store_flush(void)
      * plc_retain_file_store_stop(): the PLC can be started again without the
      * daemon restarting, and joining the thread here would leave the next run
      * with nothing committing on a timer. */
-    std::lock_guard<std::mutex> guard(g_lock);
+    std::lock_guard<RtMutex> guard(g_lock);
     if (!g_dirty || g_pending.empty()) return 0;
     commit(g_pending.data(), (uint16_t)g_pending.size(), g_program_md5);
     g_dirty = false;