diff --git a/Project.toml b/Project.toml index 2fdca24..125f380 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "OMLibraryTesting" uuid = "4866894a-ca78-4bef-9a5c-0d35f2f81ce1" -version = "0.3.0" +version = "0.3.1" authors = ["John Tinnerholm "] [deps] @@ -47,7 +47,7 @@ ImmutableList = "0.4.2" ListUtil = "0.4.1" MetaModelica = "0.4.2" OM = "0.5.0" -OMBackend = "0.5.1" +OMBackend = "0.5.5" OMFrontend = "1.3.1" OMParser = "0.2.1" SCode = "1.2.0" diff --git a/models/models.toml b/models/models.toml index 58a7fd4..0b1ad4e 100644 --- a/models/models.toml +++ b/models/models.toml @@ -2279,8 +2279,8 @@ issue = "Apr25 full MSL run: backend passed; downstream phase failed (simulate=f name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.BevelGear1D" domain = "Mechanics.MultiBody" expected = "simulate" -skipPhases = ["validate"] -issue = "Apr25 full MSL run: backend passed; downstream phase failed (simulate=ok, validate=fail). Skipping simulate and validate to preserve frontend/backend coverage. 2026-09-27: reaches simulate without the registry's skips (coverage run on the 1.13 stack)." +stopTime = 0.4 +issue = "2026-10-14: the motion diverges after 0.4 s (a sensitive system: revolute1.w at 0.55 s -3353 vs 150 rad/s), and MSL 4.1.0's reference of the model ends at 0.4 s (models_4.1.0.toml): compared over the same 0.4 s. There revolute1-3.w still fail the pointwise rule at single points (worst 3.3e-3 relative at 0.4 s, and near zero crossings), as in 4.1.0." [models.Apr25_Mechanics_MultiBody_Examples_Rotational3DEffects_GearConstraint] name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.GearConstraint" diff --git a/models/models_4.1.0.toml b/models/models_4.1.0.toml index 1e88c46..b16e835 100644 --- a/models/models_4.1.0.toml +++ b/models/models_4.1.0.toml @@ -772,3 +772,9 @@ reltol = 0.01 domain = "Utilities" name = "Modelica.Utilities.Examples.readRealParameterModel" stopTime = 0.0 + +[models.Mechanics_MultiBody_Examples_Rotational3DEffects_BevelGear1D] +name = "Modelica.Mechanics.MultiBody.Examples.Rotational3DEffects.BevelGear1D" +domain = "Mechanics.MultiBody" +stopTime = 0.4 +issue = "2026-10-13: the MSL 4.1.0 reference ends at 0.4 s (experiment StopTime 1): compared over the reference's span." diff --git a/src/comparison.jl b/src/comparison.jl index 08f9ac7..41a94d8 100644 --- a/src/comparison.jl +++ b/src/comparison.jl @@ -65,8 +65,9 @@ headers on line 1 and numeric data rows. The first column is "time". function load_reference_csv(path::String)::ReferenceData lines = readlines(path) isempty(lines) && error("Empty reference file: $path") - header = split(lines[1], ',') - header = [strip(h, ['"', ' ']) for h in header] + #= Quoted names may hold commas (MSL 4.1.0's `stack.cell[1, 1].cell.limIntegrator.y`): + split outside quotes, or every later column was read under the wrong name. =# + header = [strip(h, ['"', ' ']) for h in split_csv_header(lines[1])] ncols = length(header) nrows = length(lines) - 1 data = zeros(Float64, nrows, ncols) @@ -84,10 +85,40 @@ function load_reference_csv(path::String)::ReferenceData return ReferenceData(time_vec, signals) end +""" + split_csv_header(line) -> Vector{String} + +The fields of a CSV header line, split at the commas outside double quotes. +""" +function split_csv_header(line::AbstractString)::Vector{String} + fields = String[] + buf = IOBuffer() + inquote = false + for c in line + if c == '"' + inquote = !inquote + print(buf, c) + elseif c == ',' && !inquote + push!(fields, String(take!(buf))) + else + print(buf, c) + end + end + push!(fields, String(take!(buf))) + return fields +end + +#= The instants a signal is compared at, evenly spaced over [0, stopTime]: 101, so a + periodic signal is not sampled at its zero crossings only (21 points put every sample of + Blocks.Modulation's sine.y on one). =# +const COMPARISON_POINTS = 101 + #= Whether a stopTime lies within a reference that ends at `ref_end`, up to - rounding (relative: some references span 1e-11 s). =# + rounding (relative: some references span 1e-11 s). The MSL 4.1.0 references + print 8 significant digits: 1e-8 ends at 9.9999999e-9 (DifferenceAmplifier), + 0.32 at 0.31999999 (SMPM_NoLoad). =# stoptime_within_reference(stopTime::Real, ref_end::Real)::Bool = - stopTime <= ref_end + 1e-9 * abs(ref_end) + stopTime <= ref_end + 1e-7 * abs(ref_end) """ load_comparison_signals(path) -> Vector{String} @@ -97,7 +128,8 @@ Skips the "time" entry. """ function load_comparison_signals(path::String)::Vector{String} lines = strip.(readlines(path)) - filter(l -> !isempty(l) && l != "time", lines) + #= `//` lines are comments (the MAP-LIB comparisonSignals of 16 MSL 4.1.0 models). =# + filter(l -> !isempty(l) && l != "time" && !startswith(l, "//"), lines) end """ @@ -145,7 +177,9 @@ Convert Modelica dot-notation signal name to OM.jl underscore convention. E.g., "revolute1.phi" -> "revolute1_phi", "C1.v" -> "C1_v" """ function modelica_to_omjl_name(name::String)::String - return replace(name, "." => "_") + #= OM.jl names a matrix element x[1][2] (scalarized): `x[1, 2]` from the reference. =# + local n = replace(name, r"\[([^\]]*)\]" => m -> "[" * join(strip.(split(m[2:end-1], ',')), "][") * "]") + return replace(n, "." => "_") end """ @@ -223,10 +257,16 @@ function compare_signal(sol, ref::ReferenceData, signal_name::String, interpolate_reference(sol_times, vals, t) catch e @warn "Signal $omjl_name not found in solution (eliminated by alias/simplification?)" - # Signal not available: return as skipped (passed=true, NaN errors) - return SignalComparison(signal_name, true, NaN, NaN, t, NaN, expected) + #= Not in the solution: not compared (passed = false, NaN errors), which fails the + validation (validate_against_reference). It passed, so a model validated on a + subset of its signals; and a sample after a failed one discarded that failure. =# + return SignalComparison(signal_name, false, NaN, NaN, t, NaN, expected) end end + #= A non-finite value is a failure: NaN compared as never above the threshold. =# + if !(actual isa Real) || !isfinite(actual) + return SignalComparison(signal_name, false, Inf, Inf, t, actual isa Real ? Float64(actual) : NaN, expected) + end abs_err = abs(actual - expected) rel_err = abs(expected) > 1e-15 ? abs_err / abs(expected) : abs_err threshold = atol + reltol * abs(expected) @@ -268,7 +308,8 @@ Validate an OM.jl solution against the MAP-LIB reference CSV for a model. Returns (overall_passed, vector_of_SignalComparison). """ function validate_against_reference(sol, spec, - ref_dir::String)::Tuple{Bool, Vector{SignalComparison}} + ref_dir::String; + terminatedAt::Union{Nothing, Float64} = nothing)::Tuple{Bool, Vector{SignalComparison}} ref_name = spec.referenceFile csv_path = joinpath(ref_dir, "csv", ref_name * ".csv") signals_path = joinpath(ref_dir, "signals", ref_name * ".txt") @@ -279,8 +320,16 @@ function validate_against_reference(sol, spec, #= The reference covers the experiment, [0, StopTime]: a sample past its end would be compared with its last value, frozen (interpolate_reference clamps). A registry stopTime past it is an error in the registry. =# - stoptime_within_reference(spec.stopTime, ref.time[end]) || - error("stopTime $(spec.stopTime) is past the end of reference $ref_name ($(ref.time[end]))") + #= A run that terminate()d before stopTime (MSL 4 RobotR3: the path planning ends it at + 1.498 s) is compared up to where its reference ends, which must be where it ended. =# + local compareStop = spec.stopTime + if terminatedAt !== nothing && terminatedAt < spec.stopTime + abs(terminatedAt - ref.time[end]) <= 1e-6 * max(1.0, abs(ref.time[end])) || + error("terminated at $terminatedAt, the reference $ref_name ends at $(ref.time[end])") + compareStop = min(terminatedAt, ref.time[end]) + end + stoptime_within_reference(compareStop, ref.time[end]) || + error("stopTime $(compareStop) is past the end of reference $ref_name ($(ref.time[end]))") signal_names = if isfile(signals_path) load_comparison_signals(signals_path) else @@ -288,28 +337,28 @@ function validate_against_reference(sol, spec, end reltol = spec.reltol atol = spec.atol + #= A listed signal the reference does not have is an error of the reference data or of + its reading (a header split inside a quoted name), not a pass. =# + missing_ref = filter(n -> !haskey(ref.signals, n), signal_names) + isempty(missing_ref) || + error("Reference $ref_name has no column for the listed signals: $(join(missing_ref, ", "))") comparisons = SignalComparison[] all_passed = true n_skipped = 0 for name in signal_names - cmp = compare_signal(sol, ref, name, spec.stopTime; - reltol = reltol, atol = atol, + cmp = compare_signal(sol, ref, name, compareStop; + reltol = reltol, atol = atol, npoints = COMPARISON_POINTS, signalMapping = spec.signalMapping) push!(comparisons, cmp) - if isnan(cmp.max_abs_err) - n_skipped += 1 - elseif !cmp.passed - all_passed = false - end + isnan(cmp.max_abs_err) && (n_skipped += 1) + cmp.passed || (all_passed = false) end + #= Every listed signal is compared: one the solution lacks fails the validation (it was + skipped, and a model validated on a subset: IdealGasH2O on 1 of 3 signals, MSL 4 + CCCV_Stack on 1 of 13). The harness keeps the reference signals observable + (_reference_signal_filter). =# if n_skipped > 0 - @info "Validation: $n_skipped/$(length(signal_names)) signals not found in solution (skipped)" - end - #= A validation that compared no signal is not one (it passed until 2026-09-30: - GenerateRandomNumbers, whose outputs are not in the solution). =# - if !isempty(signal_names) && n_skipped == length(signal_names) - @warn "Validation: none of the $(length(signal_names)) reference signals is in the solution" - all_passed = false + @warn "Validation: $n_skipped/$(length(signal_names)) reference signals are not in the solution" end return (all_passed, comparisons) end diff --git a/src/runner.jl b/src/runner.jl index 112fa15..ef06c35 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -307,11 +307,10 @@ function _reference_signal_filter(ref_dir::String, end local patterns = String[] local seen = Set{String}() - for col in split(header_line, ',') + for col in split_csv_header(header_line) local raw = strip(col, ['"', ' ', '\t', '\r', '\n']) (isempty(raw) || raw == "time") && continue - local mapped = get(signal_mapping, String(raw), String(raw)) - local under = replace(mapped, "." => "_") + local under = haskey(signal_mapping, String(raw)) ? signal_mapping[String(raw)] : modelica_to_omjl_name(String(raw)) if !(under in seen) push!(seen, under) push!(patterns, string("^", _escape_regex(under), "\$")) @@ -421,7 +420,11 @@ function _run_model_phases(model_name::String, local _sa = isempty(solver_name) ? NamedTuple() : _resolve_solver(solver_name) local _extra = dtmax > 0.0 ? (; dtmax = dtmax) : NamedTuple() local _ia = isempty(init_alg) ? NamedTuple() : _resolve_init_alg(init_alg) - local _of = isempty(observed_filter) ? NamedTuple() : (; observedFilter = observed_filter) + #= The reference's signals stay observable (the backend drops eliminated array + elements' aliases by default): a registry filter, else the reference header. =# + local _filter = isempty(observed_filter) ? + _reference_signal_filter(ref_dir, reference_file, signal_mapping) : observed_filter + local _of = isempty(_filter) ? NamedTuple() : (; observedFilter = _filter) local _mi = (; maxiters = maxiters > 0.0 ? round(Int, maxiters) : DEFAULT_MAXITERS) local _tol = NamedTuple() if solver_reltol > 0.0 @@ -443,12 +446,13 @@ function _run_model_phases(model_name::String, model_name, "", "", stop_time, UNKNOWN, reference, atol, reltol, reference_file, signal_mapping, "", Set{Phase}()) - (passed, comparisons) = validate_against_reference(sol, temp_spec, ref_dir) + (passed, comparisons) = validate_against_reference(sol, temp_spec, ref_dir; + terminatedAt = sol.retcode == ReturnCode.Terminated ? Float64(sol.t[end]) : nothing) if !passed failed = filter(c -> !c.passed, comparisons) isempty(failed) && error("Validation compared no signal: none of the $(length(comparisons)) reference signals is in the solution") - names = join([c.name for c in failed], ", ") + names = join([c.name * (isnan(c.max_abs_err) ? " (not in the solution)" : "") for c in failed], ", ") error("Validation failed for signals: $names") end else @@ -484,6 +488,15 @@ function _run_model_phases(model_name::String, end end + #= The model's build and its tables are not needed after its phases: a worker runs hundreds + of models, and every model's build and tables stayed for the whole run. =# + sol = nothing + try + OMBackend.forgetModel(model_name) + catch e + @warn " $(model_name): forgetModel failed" exception = e + end + return (results, highest) end @@ -552,7 +565,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, e isa InterruptException && rethrow() _remove_started_marker(marker_path) msg = "Worker spawn failed: $(_bounded_showerror(e, 500))" - @warn " CRASH: $msg" + @warn " $(spec.name): CRASH: $msg" kill_worker!(mgr) # a worker that started but did not load the packages phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1] return ModelResult(spec, PhaseResult[PhaseResult(phase, false, 0.0, msg)], BROKEN, @@ -588,7 +601,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, timer_interrupt = Timer(timeout) do _t if _future_pending() interrupted[] = true - @warn " TIMEOUT after $(round(timeout, digits=0))s, sending interrupt to worker..." + @warn " $(spec.name): TIMEOUT after $(round(timeout, digits=0))s, sending interrupt to worker..." try; interrupt(pid); catch; end end end @@ -596,7 +609,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, timer_kill = Timer(timeout + grace_period) do _t if _future_pending() && interrupted[] killed[] = true - @warn " Worker did not respond to interrupt, killing..." + @warn " $(spec.name): worker did not respond to interrupt, killing..." try; kill_worker!(mgr); catch; end end end @@ -621,9 +634,9 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, # Log phase results for pr in phase_results if pr.success - @info " $(PHASE_NAMES[pr.phase]): PASS ($(round(pr.time_s, digits=1))s)" + @info " $(spec.name): $(PHASE_NAMES[pr.phase]): PASS ($(round(pr.time_s, digits=1))s)" else - @info " $(PHASE_NAMES[pr.phase]): FAIL: $(pr.error)" + @info " $(spec.name): $(PHASE_NAMES[pr.phase]): FAIL: $(pr.error)" end end @@ -634,7 +647,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, # A Ctrl-C of the run itself; the timeout's interrupt of the worker arrives as a RemoteException. if e isa InterruptException - @warn " INTERRUPTED by user" + @warn " $(spec.name): INTERRUPTED by user" kill_worker!(mgr) rethrow() elseif interrupted[] || killed[] @@ -644,7 +657,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, mgr.pid = nothing mgr.ready = false end - @warn " TIMEOUT: $msg" + @warn " $(spec.name): TIMEOUT: $msg" phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1] phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)] return ModelResult(spec, phases, BROKEN, @@ -652,7 +665,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, elseif e isa ProcessExitedException || (e isa RemoteException && e.captured.ex isa ProcessExitedException) msg = "Worker process crashed: $(_bounded_showerror(e, 500))" - @warn " CRASH: $msg" + @warn " $(spec.name): CRASH: $msg" mgr.pid = nothing mgr.ready = false phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1] @@ -662,7 +675,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, elseif e isa RemoteException # Worker threw a normal exception that escaped _run_model_phases msg = _bounded_showerror(e, 500) - @info " FAIL: $msg" + @info " $(spec.name): FAIL: $msg" phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1] phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)] return ModelResult(spec, phases, BROKEN, diff --git a/test/comparisonTests.jl b/test/comparisonTests.jl index d896189..3cd3de8 100644 --- a/test/comparisonTests.jl +++ b/test/comparisonTests.jl @@ -75,6 +75,9 @@ end @test OLT.stoptime_within_reference(1e-7, 1e-7) @test !OLT.stoptime_within_reference(1.01e-7, 1e-7) @test OLT.stoptime_within_reference(0.0, 0.0) + #= 8 significant digits in the MSL 4.1.0 references. =# + @test OLT.stoptime_within_reference(1e-8, 9.9999999e-9) + @test OLT.stoptime_within_reference(0.32, 0.31999999) end @testset "comparison: no reference signal in the solution" begin @@ -89,8 +92,53 @@ end 1e-4, 3e-3, "M", Dict{String, String}(), "", Set{OLT.Phase}()) local absent = SignalOf(t -> error("not in the solution")) @test !first(OLT.validate_against_reference(absent, spec, dir)) - #= x found and right, y missing: validated on x. =# + #= x found and right, y missing: every listed signal is compared, so not validated + (it validated on x alone until 2026-10-13). =# local onlyX = (t; idxs = nothing) -> idxs === :x ? 2t : error("not in the solution") - @test first(OLT.validate_against_reference(onlyX, spec, dir)) + @test !first(OLT.validate_against_reference(onlyX, spec, dir)) + local both = (t; idxs = nothing) -> 2t + @test first(OLT.validate_against_reference(both, spec, dir)) + end +end + +@testset "comparison: what a validation must not let pass (audit 2026-10-13)" begin + local ref = OLT.ReferenceData([0.0, 0.5, 1.0], Dict("x" => [0.0, 1.0, 2.0])) + #= NaN was never above the threshold. =# + @test !OLT.compare_signal(SignalOf(t -> NaN), ref, "x", 1.0; npoints = 5).passed + @test !OLT.compare_signal(SignalOf(t -> t < 0.5 ? 2t : Inf), ref, "x", 1.0; npoints = 5).passed + @test OLT.compare_signal(SignalOf(t -> 2t), ref, "x", 1.0; npoints = 5).passed + #= A quoted name with a comma is one column (MSL 4.1.0's stack.cell[1, 1]...). =# + @test OLT.split_csv_header("\"time\",\"a.b\",\"c[1, 2].d\",\"e\"") == + ["\"time\"", "\"a.b\"", "\"c[1, 2].d\"", "\"e\""] + @test OLT.modelica_to_omjl_name("stack.cell[1, 2].y") == "stack_cell[1][2]_y" + @test OLT.modelica_to_omjl_name("r.v[3]") == "r_v[3]" + mktempdir() do dir + mkpath(joinpath(dir, "csv")) + mkpath(joinpath(dir, "signals")) + write(joinpath(dir, "csv", "M.csv"), "\"time\",\"c[1, 2].d\",\"e\"\n0.0,0.0,5.0\n0.5,1.0,5.0\n") + local r = OLT.load_reference_csv(joinpath(dir, "csv", "M.csv")) + @test r.signals["c[1, 2].d"] == [0.0, 1.0] && r.signals["e"] == [5.0, 5.0] + #= `//` lines of a comparisonSignals file are comments. =# + write(joinpath(dir, "signals", "M.txt"), "time\n// a comment\nc[1, 2].d\ne\n") + @test OLT.load_comparison_signals(joinpath(dir, "signals", "M.txt")) == ["c[1, 2].d", "e"] + #= A listed signal without a column is an error of the reference, not a pass. =# + write(joinpath(dir, "signals", "M.txt"), "c[1, 2].d\nf\n") + local spec = OLT.ModelSpec("M", "M", "Test", 0.5, OLT.VALIDATE, Dict{String, Float64}(), + 1e-4, 3e-3, "M", Dict{String, String}(), "", Set{OLT.Phase}()) + @test_throws ErrorException OLT.validate_against_reference(SignalOf(t -> 2t), spec, dir) + end +end + +@testset "comparison: a run that terminate()d" begin + #= Compared up to where it ended, which must be where its reference ends (MSL 4 RobotR3). =# + mktempdir() do dir + mkpath(joinpath(dir, "csv")) + write(joinpath(dir, "csv", "M.csv"), "time,x\n0.0,0.0\n0.25,0.5\n0.5,1.0\n") + local spec = OLT.ModelSpec("M", "M", "Test", 1.0, OLT.VALIDATE, Dict{String, Float64}(), + 1e-4, 3e-3, "M", Dict{String, String}(), "", Set{OLT.Phase}()) + local s = SignalOf(t -> 2t) + @test_throws ErrorException OLT.validate_against_reference(s, spec, dir) + @test first(OLT.validate_against_reference(s, spec, dir; terminatedAt = 0.5 + 1e-8)) + @test_throws ErrorException OLT.validate_against_reference(s, spec, dir; terminatedAt = 0.3) end end