Validation audit fixes, each model's build released after its phases (0.3.1)

Project.toml

@@ -1,6 +1,6 @@
 name = "OMLibraryTesting"
 uuid = "4866894a-ca78-4bef-9a5c-0d35f2f81ce1"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["John Tinnerholm <john.tinnerholm@liu.se>"]
 
 [deps]
@@ -47,7 +47,7 @@ ImmutableList = "0.4.2"
 ListUtil = "0.4.1"
 MetaModelica = "0.4.2"
 OM = "0.5.0"
-OMBackend = "0.5.1"
+OMBackend = "0.5.5"
 OMFrontend = "1.3.1"
 OMParser = "0.2.1"
 SCode = "1.2.0"

models/models.toml

@@ -2279,8 +2279,8 @@ issue = "Apr25 full MSL run: backend passed; downstream phase failed (simulate=f
 name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.BevelGear1D"
 domain = "Mechanics.MultiBody"
 expected = "simulate"
-skipPhases = ["validate"]
-issue = "Apr25 full MSL run: backend passed; downstream phase failed (simulate=ok, validate=fail). Skipping simulate and validate to preserve frontend/backend coverage. 2026-09-27: reaches simulate without the registry's skips (coverage run on the 1.13 stack)."
+stopTime = 0.4
+issue = "2026-10-14: the motion diverges after 0.4 s (a sensitive system: revolute1.w at 0.55 s -3353 vs 150 rad/s), and MSL 4.1.0's reference of the model ends at 0.4 s (models_4.1.0.toml): compared over the same 0.4 s. There revolute1-3.w still fail the pointwise rule at single points (worst 3.3e-3 relative at 0.4 s, and near zero crossings), as in 4.1.0."
 
 [models.Apr25_Mechanics_MultiBody_Examples_Rotational3DEffects_GearConstraint]
 name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.GearConstraint"

models/models_4.1.0.toml

@@ -772,3 +772,9 @@ reltol = 0.01
 domain = "Utilities"
 name = "Modelica.Utilities.Examples.readRealParameterModel"
 stopTime = 0.0
+
+[models.Mechanics_MultiBody_Examples_Rotational3DEffects_BevelGear1D]
+name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.BevelGear1D"
+domain = "Mechanics.MultiBody"
+stopTime = 0.4
+issue = "2026-10-13: the MSL 4.1.0 reference ends at 0.4 s (experiment StopTime 1): compared over the reference's span."

src/comparison.jl

@@ -65,8 +65,9 @@ headers on line 1 and numeric data rows. The first column is "time".
 function load_reference_csv(path::String)::ReferenceData
     lines = readlines(path)
     isempty(lines) && error("Empty reference file: $path")
-    header = split(lines[1], ',')
-    header = [strip(h, ['"', ' ']) for h in header]
+    #= Quoted names may hold commas (MSL 4.1.0's `stack.cell[1, 1].cell.limIntegrator.y`):
+       split outside quotes, or every later column was read under the wrong name. =#
+    header = [strip(h, ['"', ' ']) for h in split_csv_header(lines[1])]
     ncols = length(header)
     nrows = length(lines) - 1
     data = zeros(Float64, nrows, ncols)
@@ -84,10 +85,40 @@ function load_reference_csv(path::String)::ReferenceData
     return ReferenceData(time_vec, signals)
 end
 
+"""
+    split_csv_header(line) -> Vector{String}
+
+The fields of a CSV header line, split at the commas outside double quotes.
+"""
+function split_csv_header(line::AbstractString)::Vector{String}
+    fields = String[]
+    buf = IOBuffer()
+    inquote = false
+    for c in line
+        if c == '"'
+            inquote = !inquote
+            print(buf, c)
+        elseif c == ',' && !inquote
+            push!(fields, String(take!(buf)))
+        else
+            print(buf, c)
+        end
+    end
+    push!(fields, String(take!(buf)))
+    return fields
+end
+
+#= The instants a signal is compared at, evenly spaced over [0, stopTime]: 101, so a
+   periodic signal is not sampled at its zero crossings only (21 points put every sample of
+   Blocks.Modulation's sine.y on one). =#
+const COMPARISON_POINTS = 101
+
 #= Whether a stopTime lies within a reference that ends at `ref_end`, up to
-   rounding (relative: some references span 1e-11 s). =#
+   rounding (relative: some references span 1e-11 s). The MSL 4.1.0 references
+   print 8 significant digits: 1e-8 ends at 9.9999999e-9 (DifferenceAmplifier),
+   0.32 at 0.31999999 (SMPM_NoLoad). =#
 stoptime_within_reference(stopTime::Real, ref_end::Real)::Bool =
-    stopTime <= ref_end + 1e-9 * abs(ref_end)
+    stopTime <= ref_end + 1e-7 * abs(ref_end)
 
 """
     load_comparison_signals(path) -> Vector{String}
@@ -97,7 +128,8 @@ Skips the "time" entry.
 """
 function load_comparison_signals(path::String)::Vector{String}
     lines = strip.(readlines(path))
-    filter(l -> !isempty(l) && l != "time", lines)
+    #= `//` lines are comments (the MAP-LIB comparisonSignals of 16 MSL 4.1.0 models). =#
+    filter(l -> !isempty(l) && l != "time" && !startswith(l, "//"), lines)
 end
 
 """
@@ -145,7 +177,9 @@ Convert Modelica dot-notation signal name to OM.jl underscore convention.
 E.g., "revolute1.phi" -> "revolute1_phi", "C1.v" -> "C1_v"
 """
 function modelica_to_omjl_name(name::String)::String
-    return replace(name, "." => "_")
+    #= OM.jl names a matrix element x[1][2] (scalarized): `x[1, 2]` from the reference. =#
+    local n = replace(name, r"\[([^\]]*)\]" => m -> "[" * join(strip.(split(m[2:end-1], ',')), "][") * "]")
+    return replace(n, "." => "_")
 end
 
 """
@@ -223,10 +257,16 @@ function compare_signal(sol, ref::ReferenceData, signal_name::String,
                 interpolate_reference(sol_times, vals, t)
             catch e
                 @warn "Signal $omjl_name not found in solution (eliminated by alias/simplification?)"
-                # Signal not available: return as skipped (passed=true, NaN errors)
-                return SignalComparison(signal_name, true, NaN, NaN, t, NaN, expected)
+                #= Not in the solution: not compared (passed = false, NaN errors), which fails the
+                   validation (validate_against_reference). It passed, so a model validated on a
+                   subset of its signals; and a sample after a failed one discarded that failure. =#
+                return SignalComparison(signal_name, false, NaN, NaN, t, NaN, expected)
             end
         end
+        #= A non-finite value is a failure: NaN compared as never above the threshold. =#
+        if !(actual isa Real) || !isfinite(actual)
+            return SignalComparison(signal_name, false, Inf, Inf, t, actual isa Real ? Float64(actual) : NaN, expected)
+        end
         abs_err = abs(actual - expected)
         rel_err = abs(expected) > 1e-15 ? abs_err / abs(expected) : abs_err
         threshold = atol + reltol * abs(expected)
@@ -268,7 +308,8 @@ Validate an OM.jl solution against the MAP-LIB reference CSV for a model.
 Returns (overall_passed, vector_of_SignalComparison).
 """
 function validate_against_reference(sol, spec,
-                                     ref_dir::String)::Tuple{Bool, Vector{SignalComparison}}
+                                     ref_dir::String;
+                                     terminatedAt::Union{Nothing, Float64} = nothing)::Tuple{Bool, Vector{SignalComparison}}
     ref_name = spec.referenceFile
     csv_path = joinpath(ref_dir, "csv", ref_name * ".csv")
     signals_path = joinpath(ref_dir, "signals", ref_name * ".txt")
@@ -279,37 +320,45 @@ function validate_against_reference(sol, spec,
     #= The reference covers the experiment, [0, StopTime]: a sample past its
        end would be compared with its last value, frozen (interpolate_reference
        clamps). A registry stopTime past it is an error in the registry. =#
-    stoptime_within_reference(spec.stopTime, ref.time[end]) ||
-        error("stopTime $(spec.stopTime) is past the end of reference $ref_name ($(ref.time[end]))")
+    #= A run that terminate()d before stopTime (MSL 4 RobotR3: the path planning ends it at
+       1.498 s) is compared up to where its reference ends, which must be where it ended. =#
+    local compareStop = spec.stopTime
+    if terminatedAt !== nothing && terminatedAt < spec.stopTime
+        abs(terminatedAt - ref.time[end]) <= 1e-6 * max(1.0, abs(ref.time[end])) ||
+            error("terminated at $terminatedAt, the reference $ref_name ends at $(ref.time[end])")
+        compareStop = min(terminatedAt, ref.time[end])
+    end
+    stoptime_within_reference(compareStop, ref.time[end]) ||
+        error("stopTime $(compareStop) is past the end of reference $ref_name ($(ref.time[end]))")
     signal_names = if isfile(signals_path)
         load_comparison_signals(signals_path)
     else
         collect(keys(ref.signals))
     end
     reltol = spec.reltol
     atol = spec.atol
+    #= A listed signal the reference does not have is an error of the reference data or of
+       its reading (a header split inside a quoted name), not a pass. =#
+    missing_ref = filter(n -> !haskey(ref.signals, n), signal_names)
+    isempty(missing_ref) ||
+        error("Reference $ref_name has no column for the listed signals: $(join(missing_ref, ", "))")
     comparisons = SignalComparison[]
     all_passed = true
     n_skipped = 0
     for name in signal_names
-        cmp = compare_signal(sol, ref, name, spec.stopTime;
-                              reltol = reltol, atol = atol,
+        cmp = compare_signal(sol, ref, name, compareStop;
+                              reltol = reltol, atol = atol, npoints = COMPARISON_POINTS,
                               signalMapping = spec.signalMapping)
         push!(comparisons, cmp)
-        if isnan(cmp.max_abs_err)
-            n_skipped += 1
-        elseif !cmp.passed
-            all_passed = false
-        end
+        isnan(cmp.max_abs_err) && (n_skipped += 1)
+        cmp.passed || (all_passed = false)
     end
+    #= Every listed signal is compared: one the solution lacks fails the validation (it was
+       skipped, and a model validated on a subset: IdealGasH2O on 1 of 3 signals, MSL 4
+       CCCV_Stack on 1 of 13). The harness keeps the reference signals observable
+       (_reference_signal_filter). =#
     if n_skipped > 0
-        @info "Validation: $n_skipped/$(length(signal_names)) signals not found in solution (skipped)"
-    end
-    #= A validation that compared no signal is not one (it passed until 2026-09-30:
-       GenerateRandomNumbers, whose outputs are not in the solution). =#
-    if !isempty(signal_names) && n_skipped == length(signal_names)
-        @warn "Validation: none of the $(length(signal_names)) reference signals is in the solution"
-        all_passed = false
+        @warn "Validation: $n_skipped/$(length(signal_names)) reference signals are not in the solution"
     end
     return (all_passed, comparisons)
 end

src/runner.jl

@@ -307,11 +307,10 @@ function _reference_signal_filter(ref_dir::String,
     end
     local patterns = String[]
     local seen = Set{String}()
-    for col in split(header_line, ',')
+    for col in split_csv_header(header_line)
         local raw = strip(col, ['"', ' ', '\t', '\r', '\n'])
         (isempty(raw) || raw == "time") && continue
-        local mapped = get(signal_mapping, String(raw), String(raw))
-        local under = replace(mapped, "." => "_")
+        local under = haskey(signal_mapping, String(raw)) ? signal_mapping[String(raw)] : modelica_to_omjl_name(String(raw))
         if !(under in seen)
             push!(seen, under)
             push!(patterns, string("^", _escape_regex(under), "\$"))
@@ -421,7 +420,11 @@ function _run_model_phases(model_name::String,
                 local _sa = isempty(solver_name) ? NamedTuple() : _resolve_solver(solver_name)
                 local _extra = dtmax > 0.0 ? (; dtmax = dtmax) : NamedTuple()
                 local _ia = isempty(init_alg) ? NamedTuple() : _resolve_init_alg(init_alg)
-                local _of = isempty(observed_filter) ? NamedTuple() : (; observedFilter = observed_filter)
+                #= The reference's signals stay observable (the backend drops eliminated array
+                   elements' aliases by default): a registry filter, else the reference header. =#
+                local _filter = isempty(observed_filter) ?
+                    _reference_signal_filter(ref_dir, reference_file, signal_mapping) : observed_filter
+                local _of = isempty(_filter) ? NamedTuple() : (; observedFilter = _filter)
                 local _mi = (; maxiters = maxiters > 0.0 ? round(Int, maxiters) : DEFAULT_MAXITERS)
                 local _tol = NamedTuple()
                 if solver_reltol > 0.0
@@ -443,12 +446,13 @@ function _run_model_phases(model_name::String,
                         model_name, "", "", stop_time, UNKNOWN,
                         reference, atol, reltol, reference_file, signal_mapping, "",
                         Set{Phase}())
-                    (passed, comparisons) = validate_against_reference(sol, temp_spec, ref_dir)
+                    (passed, comparisons) = validate_against_reference(sol, temp_spec, ref_dir;
+                        terminatedAt = sol.retcode == ReturnCode.Terminated ? Float64(sol.t[end]) : nothing)
                     if !passed
                         failed = filter(c -> !c.passed, comparisons)
                         isempty(failed) &&
                             error("Validation compared no signal: none of the $(length(comparisons)) reference signals is in the solution")
-                        names = join([c.name for c in failed], ", ")
+                        names = join([c.name * (isnan(c.max_abs_err) ? " (not in the solution)" : "") for c in failed], ", ")
                         error("Validation failed for signals: $names")
                     end
                 else
@@ -484,6 +488,15 @@ function _run_model_phases(model_name::String,
         end
     end
 
+    #= The model's build and its tables are not needed after its phases: a worker runs hundreds
+       of models, and every model's build and tables stayed for the whole run. =#
+    sol = nothing
+    try
+        OMBackend.forgetModel(model_name)
+    catch e
+        @warn "    $(model_name): forgetModel failed" exception = e
+    end
+
     return (results, highest)
 end
 
@@ -552,7 +565,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
         e isa InterruptException && rethrow()
         _remove_started_marker(marker_path)
         msg = "Worker spawn failed: $(_bounded_showerror(e, 500))"
-        @warn "    CRASH: $msg"
+        @warn "    $(spec.name): CRASH: $msg"
         kill_worker!(mgr)     # a worker that started but did not load the packages
         phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1]
         return ModelResult(spec, PhaseResult[PhaseResult(phase, false, 0.0, msg)], BROKEN,
@@ -588,15 +601,15 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
     timer_interrupt = Timer(timeout) do _t
         if _future_pending()
             interrupted[] = true
-            @warn "  TIMEOUT after $(round(timeout, digits=0))s, sending interrupt to worker..."
+            @warn "  $(spec.name): TIMEOUT after $(round(timeout, digits=0))s, sending interrupt to worker..."
             try; interrupt(pid); catch; end
         end
     end
 
     timer_kill = Timer(timeout + grace_period) do _t
         if _future_pending() && interrupted[]
             killed[] = true
-            @warn "  Worker did not respond to interrupt, killing..."
+            @warn "  $(spec.name): worker did not respond to interrupt, killing..."
             try; kill_worker!(mgr); catch; end
         end
     end
@@ -621,9 +634,9 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
         # Log phase results
         for pr in phase_results
             if pr.success
-                @info "    $(PHASE_NAMES[pr.phase]): PASS ($(round(pr.time_s, digits=1))s)"
+                @info "    $(spec.name): $(PHASE_NAMES[pr.phase]): PASS ($(round(pr.time_s, digits=1))s)"
             else
-                @info "    $(PHASE_NAMES[pr.phase]): FAIL: $(pr.error)"
+                @info "    $(spec.name): $(PHASE_NAMES[pr.phase]): FAIL: $(pr.error)"
             end
         end
 
@@ -634,7 +647,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
 
         # A Ctrl-C of the run itself; the timeout's interrupt of the worker arrives as a RemoteException.
         if e isa InterruptException
-            @warn "    INTERRUPTED by user"
+            @warn "    $(spec.name): INTERRUPTED by user"
             kill_worker!(mgr)
             rethrow()
         elseif interrupted[] || killed[]
@@ -644,15 +657,15 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
                 mgr.pid = nothing
                 mgr.ready = false
             end
-            @warn "    TIMEOUT: $msg"
+            @warn "    $(spec.name): TIMEOUT: $msg"
             phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1]
             phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)]
             return ModelResult(spec, phases, BROKEN,
                               Dates.format(Dates.now(), "yyyy-mm-dd HH:MM"))
         elseif e isa ProcessExitedException ||
                (e isa RemoteException && e.captured.ex isa ProcessExitedException)
             msg = "Worker process crashed: $(_bounded_showerror(e, 500))"
-            @warn "    CRASH: $msg"
+            @warn "    $(spec.name): CRASH: $msg"
             mgr.pid = nothing
             mgr.ready = false
             phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1]
@@ -662,7 +675,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
         elseif e isa RemoteException
             # Worker threw a normal exception that escaped _run_model_phases
             msg = _bounded_showerror(e, 500)
-            @info "    FAIL: $msg"
+            @info "    $(spec.name): FAIL: $msg"
             phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1]
             phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)]
             return ModelResult(spec, phases, BROKEN,