From 898283575bb53dfa900c806b80d1296d0e96a93b Mon Sep 17 00:00:00 2001 From: JKRT Date: Tue, 13 Oct 2026 11:58:07 +0200 Subject: [PATCH 1/2] Coverage in two shard jobs, merged and published by a third (0.3.0) The single coverage job used 324 of the 360 job minutes for MSL 4.1.0. - run_coverage(shard = (k, n)), OMJL_COVERAGE_SHARD=k/n: every n-th model (shard_specs); merge_shards and scripts/merge_coverage.jl join the results (every model of the version once) and write the reports. - coverage.yml: shards 1/2 and 2/2 in parallel; the publish job checks they measured the same commits, merges and publishes as before. The setup is a local composite action (.github/actions/coverage-setup) with one Julia depot cache for all jobs. Standard runners only. Assisted-by: Claude Opus 5.5 --- .github/actions/coverage-setup/action.yml | 55 ++++++++++ .github/workflows/coverage.yml | 127 +++++++++++++++------- Project.toml | 2 +- README.md | 6 +- scripts/merge_coverage.jl | 51 +++++++++ scripts/run_full_coverage.jl | 12 +- src/OMLibraryTesting.jl | 1 + src/runner.jl | 39 ++++++- test/runnerTests.jl | 14 +++ 9 files changed, 261 insertions(+), 46 deletions(-) create mode 100644 .github/actions/coverage-setup/action.yml create mode 100644 scripts/merge_coverage.jl diff --git a/.github/actions/coverage-setup/action.yml b/.github/actions/coverage-setup/action.yml new file mode 100644 index 0000000..3aa046a --- /dev/null +++ b/.github/actions/coverage-setup/action.yml @@ -0,0 +1,55 @@ +name: Coverage setup +description: >- + The rest of the coverage environment, after OM.jl (at the workspace root) and + OMLibraryTesting (in OM.jl/OMLibraryTesting.jl) are checked out: the other + packages, Julia, the parser library and the precompiled environments, shared by + the shard jobs and the publish job of coverage.yml. + +inputs: + line: + description: OM.jl line to check out (empty for the branch the workflow runs on) + required: false + default: "" + julia: + description: Julia version + required: false + default: "1.13" + +runs: + using: composite + steps: + - name: Check out the other packages + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + path: OM.jl + exclude: OMLibraryTesting + line: ${{ inputs.line }} + + - uses: actions/setup-java@v4 # ANTLR, if the parser is built from source + with: + distribution: zulu + java-version: "21" + + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ inputs.julia }} + + # One depot cache for all jobs of the workflow (the default name has the job in it). + - uses: julia-actions/cache@v2 + with: + cache-name: julia-cache;workflow=${{ github.workflow }} + include-matrix: false + + - name: OMParser library + shell: bash + env: + PACKAGE: OMLibraryTesting + run: OM.jl/ci/build-parser.sh + + - name: Set up OM.jl + shell: bash + run: julia --color=yes --project=OM.jl OM.jl/ci/setup.jl + + - name: Set up OMLibraryTesting + shell: bash + run: julia --color=yes --project=OM.jl/OMLibraryTesting.jl -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 9ec77cc..f181207 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,6 +11,11 @@ name: MSL coverage # skipPhases and broken marks are removed, so every model goes through every # phase and the documentation shows what really stops each one. # +# Two shard jobs measure every second model each (OMJL_COVERAGE_SHARD), each +# within the 6-hour job limit; the publish job merges their results +# (scripts/merge_coverage.jl, every model exactly once, the same commits) and +# publishes them. Standard runners only. +# # Two MSL versions: 3.2.3 weekly (docs/COVERAGE.md, the README's coverage # section, the site's root) and 4.1.0 every third day (docs/COVERAGE_4.1.0.md, # its own README section, the site's msl-4.1.0/). OMFrontend bundles 3.2.3; @@ -70,9 +75,13 @@ concurrency: jobs: coverage: - name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, Julia ${{ inputs.julia || '1.13' }} + name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, shard ${{ matrix.shard }}/2, Julia ${{ inputs.julia || '1.13' }} runs-on: ubuntu-latest timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + shard: [1, 2] env: JULIA_PKG_PRECOMPILE_AUTO: "0" LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu @@ -92,40 +101,11 @@ jobs: with: path: OM.jl/OMLibraryTesting.jl - - name: Check out the other packages - uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup with: - path: OM.jl - exclude: OMLibraryTesting line: ${{ inputs.line }} - - - uses: actions/setup-java@v4 # ANTLR, if the parser is built from source - with: - distribution: zulu - java-version: "21" - - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ inputs.julia || '1.13' }} - - - uses: julia-actions/cache@v2 - - - name: OMParser library - env: - PACKAGE: OMLibraryTesting - run: OM.jl/ci/build-parser.sh - - - name: Set up OM.jl - run: julia --color=yes --project=OM.jl OM.jl/ci/setup.jl - - - name: Set up OMLibraryTesting - run: julia --color=yes --project=OM.jl/OMLibraryTesting.jl -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' - - - name: The version's documentation - run: | - doc=docs/COVERAGE.md - [ "$MSL" = 3.2.3 ] || doc="docs/COVERAGE_$MSL.md" - echo "COVERAGE_MD=$doc" >> "$GITHUB_ENV" + julia: ${{ inputs.julia || '1.13' }} # OMFrontend bundles MSL 3.2.3 and 4.0.0; another version is the installed # library: Modelica, ModelicaServices and Complex (a single file in the @@ -173,19 +153,88 @@ jobs: env: OMJL_COVERAGE_TAG: ci OMJL_COVERAGE_NOSKIP: "1" + OMJL_COVERAGE_SHARD: ${{ matrix.shard }}/2 OMJL_COVERAGE_WORKERS: ${{ inputs.workers || '1' }} OMJL_COVERAGE_TIMEOUT: ${{ inputs.timeout || '600' }} # One thread per worker: OMLibraryTesting runs no parallel workloads. JULIA_NUM_THREADS: "1" run: julia --color=yes --project=. scripts/run_full_coverage.jl + - name: Upload the shard's results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: msl-coverage-${{ env.MSL }}-shard-${{ matrix.shard }} + path: | + commits.txt + OM.jl/OMLibraryTesting.jl/reports/ + OM.jl/OMLibraryTesting.jl/logs/partial_ci_*.jls + if-no-files-found: error + + # Both shards' results as one run: the reports, the documentation and the site. + publish: + name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, publish + needs: coverage + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" + LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu + MSL: ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} + OMJL_COVERAGE_MSL: MSL:${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} + OMJL_COVERAGE_TAG: ci + NOTE: >- + No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), + 2 shards of ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model + steps: + - name: Check out OM.jl + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + packages: OM + path: . + line: ${{ inputs.line }} + + - name: Check out OMLibraryTesting + uses: actions/checkout@v4 + with: + path: OM.jl/OMLibraryTesting.jl + + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup + with: + line: ${{ inputs.line }} + julia: ${{ inputs.julia || '1.13' }} + + - name: The version's documentation + run: | + doc=docs/COVERAGE.md + [ "$MSL" = 3.2.3 ] || doc="docs/COVERAGE_$MSL.md" + echo "COVERAGE_MD=$doc" >> "$GITHUB_ENV" + + - name: Download the shards' results + uses: actions/download-artifact@v4 + with: + pattern: msl-coverage-${{ env.MSL }}-shard-* + path: ${{ runner.temp }}/shards + + # Both shards measured the same commits (checked out at their start). + - name: Commits measured + run: | + shards="$RUNNER_TEMP/shards" + for f in "$shards"/*/commits.txt; do + cmp -s "$f" "$shards/msl-coverage-$MSL-shard-1/commits.txt" || + { echo "::error::The shards measured different commits"; diff "$shards"/*/commits.txt; exit 1; } + done + cp "$shards/msl-coverage-$MSL-shard-1/commits.txt" commits.txt + cat commits.txt + + - name: Merge the shards + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/merge_coverage.jl "$RUNNER_TEMP"/shards/*/OM.jl/OMLibraryTesting.jl/logs/partial_ci_*.jls + - name: Publish id: publish working-directory: OM.jl/OMLibraryTesting.jl - env: - NOTE: >- - No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), - ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model run: | results=$(ls -t logs/partial_ci_*.jls | head -1) julia --color=yes --project=. scripts/publish_coverage.jl "$results" "$GITHUB_WORKSPACE/commits.txt" "$NOTE" @@ -220,10 +269,6 @@ jobs: - name: Publish the coverage site if: steps.publish.outcome == 'success' && (github.event_name == 'schedule' || inputs.commit) working-directory: OM.jl/OMLibraryTesting.jl - env: - NOTE: >- - No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), - ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model run: | export OMJL_SITE_URL="https://${GITHUB_REPOSITORY_OWNER,,}.github.io/${GITHUB_REPOSITORY#*/}/" site="$RUNNER_TEMP/site" diff --git a/Project.toml b/Project.toml index 795073c..2fdca24 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "OMLibraryTesting" uuid = "4866894a-ca78-4bef-9a5c-0d35f2f81ce1" -version = "0.2.0" +version = "0.3.0" authors = ["John Tinnerholm "] [deps] diff --git a/README.md b/README.md index dffd0d1..dbc0fc2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,9 @@ OMLibraryTesting.print_error_analysis(results) The "Latest coverage" section above and docs/COVERAGE.md are published by the `MSL coverage` workflow (.github/workflows/coverage.yml: MSL 3.2.3 weekly, MSL 4.1.0 every third day, or by hand from the Actions tab with `msl` and `commit` set). It runs `scripts/run_full_coverage.jl` with `OMJL_COVERAGE_NOSKIP=1` (every -model through every phase) and passes the serialized results to the publisher. It also adds the run to the +model through every phase) in two shard jobs (`OMJL_COVERAGE_SHARD=1/2` and `2/2`: every second model each, +within the 6-hour job limit); the publish job merges their results (`scripts/merge_coverage.jl`, which checks +that every model is there once) and passes them to the publisher. It also adds the run to the coverage site, served by GitHub Pages from the `gh-pages` branch (`scripts/publish_site.jl`): `runs//` with the analysis and the per-model report, an index over all runs, and the badges (`badge.json`, models that validate; `badges/.json`); MSL 4.1.0 has the same in `msl-4.1.0/`, docs/COVERAGE_4.1.0.md and its own @@ -105,6 +107,8 @@ section above. The same by hand (`OMJL_COVERAGE_MSL`, default `MSL:3.2.3`, selec ```bash OMJL_COVERAGE_NOSKIP=1 julia --project=. scripts/run_full_coverage.jl +# or in shards, merged: OMJL_COVERAGE_SHARD=1/2 ... and 2/2 ..., then +# julia --project=. scripts/merge_coverage.jl logs/partial__.jls logs/partial__.jls julia --project=. scripts/publish_coverage.jl logs/partial_full_msl_.jls [commits.txt] ["how the run was made"] julia --project=. scripts/publish_site.jl logs/partial_full_msl_.jls reports/.html docs/COVERAGE.md [commits.txt] ["how the run was made"] ``` diff --git a/scripts/merge_coverage.jl b/scripts/merge_coverage.jl new file mode 100644 index 0000000..5d4521c --- /dev/null +++ b/scripts/merge_coverage.jl @@ -0,0 +1,51 @@ +#= +Merge the results of a coverage run's shards (scripts/run_full_coverage.jl with OMJL_COVERAGE_SHARD +"k/n") into one run, written as an unsharded run writes it: the serialized results and the HTML and +Markdown reports. + + julia --project=. scripts/merge_coverage.jl ... + +Writes logs/partial__.jls, with the earliest shard's timestamp so the run keeps its +start date, and reports/coverage__start__end_.{html,md}. Every experiment model +of the version must be in exactly one shard, unless OMJL_COVERAGE_DOMAIN or OMJL_COVERAGE_MODEL +restricted the run. Environment as scripts/run_full_coverage.jl: OMJL_COVERAGE_TAG (default +"full_msl"), OMJL_COVERAGE_MSL (default "MSL:3.2.3"). +=# + +using Dates +using Serialization + +import OMLibraryTesting +const OLT = OMLibraryTesting + +isempty(ARGS) && error("usage: merge_coverage.jl ...") +const TAG = get(ENV, "OMJL_COVERAGE_TAG", "full_msl") +const MSL = get(ENV, "OMJL_COVERAGE_MSL", "MSL:3.2.3") +const LIBRARY_VERSION = OLT.msl_library_version(MSL) +const RESTRICTED = !isempty(get(ENV, "OMJL_COVERAGE_DOMAIN", "")) || !isempty(get(ENV, "OMJL_COVERAGE_MODEL", "")) +const ROOT = joinpath(@__DIR__, "..") |> abspath + +const RESULTS = OLT.merge_shards([deserialize(p)::Vector{OLT.ModelResult} for p in ARGS]) +RESTRICTED || let listed = Set(String(first(split(l, '|'))) + for l in eachline(OLT.recorded_experiments_path("Modelica", LIBRARY_VERSION)) if !isempty(l)) + absent = sort!(collect(setdiff(listed, Set(r.spec.name for r in RESULTS)))) + isempty(absent) || + error("$(length(absent)) of the $(length(listed)) models are in no shard: $(join(first(absent, 10), ", "))") +end + +const STAMPS = [m[1] for m in (match(r"_(\d{4}-\d{2}-\d{2}_\d{4})\.jls$", p) for p in ARGS) if m !== nothing] +const START = isempty(STAMPS) ? Dates.format(Dates.now(), "yyyy-mm-dd_HHMM") : minimum(STAMPS) +const OUT = joinpath(ROOT, "logs", "partial_$(TAG)_$(START).jls") +mkpath(dirname(OUT)) +abspath(OUT) in abspath.(ARGS) && error("$OUT is one of the shards' results") +open(io -> serialize(io, RESULTS), OUT, "w") +@info "Merged" shards = length(ARGS) models = length(RESULTS) path = OUT + +const NAME = "coverage_$(TAG)_start_$(START)_end_$(Dates.format(Dates.now(), "yyyy-mm-dd_HHMM"))" +const CHANGELOG = "Run tag: $TAG. MSL: $MSL. Merged from $(length(ARGS)) shards." +for format in (:html, :markdown) + path = OLT.generate_report(RESULTS; format, filename = NAME * (format === :html ? ".html" : ".md"), + dir = joinpath(ROOT, "reports"), msl_version = LIBRARY_VERSION, + changelog = CHANGELOG) + @info "Report generated" path +end diff --git a/scripts/run_full_coverage.jl b/scripts/run_full_coverage.jl index 45042dc..72d39b8 100644 --- a/scripts/run_full_coverage.jl +++ b/scripts/run_full_coverage.jl @@ -18,6 +18,8 @@ Environment variables: OMJL_COVERAGE_WORKERS Worker processes, models run in parallel (default: nprocs() - 1, at least 1). OMJL_COVERAGE_NOSKIP "1": the registry's skipPhases and expected = "broken" marks removed, so every model goes through every phase (what really stops each one; as docs/COVERAGE.md). + OMJL_COVERAGE_SHARD "k/n": only the k-th of every n models (run_coverage's `shard`), for a run split + over n jobs; scripts/merge_coverage.jl joins their results (default "1/1"). =# using Dates @@ -39,6 +41,10 @@ const TOPHASE = phase_from_string(get(ENV, "OMJL_COVERAGE_TOPHASE", "validat const WORKERS = parse(Int, get(ENV, "OMJL_COVERAGE_WORKERS", string(max(1, nprocs() - 1)))) WORKERS >= 1 || error("OMJL_COVERAGE_WORKERS must be >= 1, got $WORKERS") const NOSKIP = lowercase(get(ENV, "OMJL_COVERAGE_NOSKIP", "")) in ("1", "true", "yes") +const SHARD = let m = match(r"^(\d+)/(\d+)$", get(ENV, "OMJL_COVERAGE_SHARD", "1/1")) + m === nothing && error("OMJL_COVERAGE_SHARD must be k/n, got $(ENV["OMJL_COVERAGE_SHARD"])") + (parse(Int, m[1]), parse(Int, m[2])) +end const RUN_START = Dates.now() const RUN_TS = Dates.format(RUN_START, "yyyy-mm-dd_HHMM") @@ -67,7 +73,7 @@ function noskip_registry()::String end const OVERRIDES = NOSKIP ? noskip_registry() : OMLibraryTesting.default_models_path(LIBRARY_VERSION) -@info "Coverage run starting" tag=TAG msl=MSL timeout=TIMEOUT ts=RUN_TS from=FROMPHASE to=TOPHASE workers=WORKERS noskip=NOSKIP +@info "Coverage run starting" tag=TAG msl=MSL timeout=TIMEOUT ts=RUN_TS from=FROMPHASE to=TOPHASE workers=WORKERS noskip=NOSKIP shard=SHARD @info "Report directory: $REPORT_DIR" @info "Partial-results snapshot: $PARTIAL_PATH" @@ -81,7 +87,8 @@ results = try from_phase = FROMPHASE, to_phase = TOPHASE, overrides = OVERRIDES, - n_workers = WORKERS) + n_workers = WORKERS, + shard = SHARD) catch e @error "run_coverage threw" exception=(e, catch_backtrace()) OMLibraryTesting.ModelResult[] @@ -109,6 +116,7 @@ else "End: ", Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"), ". ", "Timeout per model: ", Int(round(TIMEOUT)), "s. ", NOSKIP ? "Registry skips and broken marks removed. " : "", + SHARD == (1, 1) ? "" : "Shard $(SHARD[1]) of $(SHARD[2]). ", "Total wall-clock: ", string(floor(Int, total_time รท 60)), "m ", string(round(Int, total_time % 60)), "s.") @info "Generating HTML report" filename=fname diff --git a/src/OMLibraryTesting.jl b/src/OMLibraryTesting.jl index 0a0f5d5..8a3b9c1 100644 --- a/src/OMLibraryTesting.jl +++ b/src/OMLibraryTesting.jl @@ -57,6 +57,7 @@ export load_models, load_models_by_domain, list_domains, load_registry_meta export load_reference_csv, load_comparison_signals, validate_against_reference export discover_experiments, record_experiments, merge_overrides!, clear_discovery_cache!, auto_detect_references! export run_coverage, run_coverage_and_report, run_model, WorkerManager, WorkerPool, ensure_pool! +export shard_specs, merge_shards export generate_report, print_summary export coverage_funnel, publish_site, site_history export categorize_error, categorize_results, print_error_analysis diff --git a/src/runner.jl b/src/runner.jl index 8e857ea..112fa15 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -723,6 +723,8 @@ experiment list, `overrides` the registry and, beside it, the reference data (`default_models_path(version)`, `reference_dir`). Optionally filter by `domain` regex, single `model` name, or general `filter` regex. +`shard = (k, n)` runs the `k`-th of every `n` of those models (`shard_specs`), for a run split +over `n` processes or CI jobs; `merge_shards` joins their results. Use `from_phase` and `to_phase` to restrict which phases are run. For example, `from_phase=BACKEND` skips the frontend phase (useful when frontend is 100% pass). @@ -750,7 +752,8 @@ function run_coverage(; library::String = "Modelica", from_phase::Phase = FRONTEND, to_phase::Phase = VALIDATE, check_sim_code::Bool = false, - n_workers::Int = max(1, nprocs() - 1))::Vector{ModelResult} + n_workers::Int = max(1, nprocs() - 1), + shard::Tuple{Int, Int} = (1, 1))::Vector{ModelResult} t_start = time() n_workers >= 1 || throw(ArgumentError("n_workers must be >= 1, got $n_workers")) phases = Base.filter(p -> Int(from_phase) <= Int(p) <= Int(to_phase), phases) @@ -773,6 +776,11 @@ function run_coverage(; library::String = "Modelica", error("No models found matching domain: $domain") end end + if shard != (1, 1) + local n_all = length(specs) + specs = shard_specs(specs, shard) + @info "Shard $(shard[1]) of $(shard[2]): $(length(specs)) of $n_all models" + end n_broken = count(s -> s.expected == BROKEN, specs) n_active = length(specs) - n_broken total_models = length(specs) @@ -827,6 +835,35 @@ function run_coverage(; library::String = "Modelica", return results end +""" + shard_specs(specs, shard) -> Vector{ModelSpec} + +The `k`-th of every `n` models for `shard = (k, n)`, in the order given (run_coverage's: by +domain, then name), so each domain is spread over the shards; `(1, 1)` is all of them. +""" +function shard_specs(specs::Vector{ModelSpec}, shard::Tuple{Int, Int})::Vector{ModelSpec} + local (k, n) = shard + 1 <= k <= n || throw(ArgumentError("shard must be (k, n) with 1 <= k <= n, got $shard")) + return specs[k:n:end] +end + +""" + merge_shards(parts) -> Vector{ModelResult} + +The results of a run's shards as one run, in run_coverage's order (domain, then name). A +model in more than one shard is an error. +""" +function merge_shards(parts::Vector{Vector{ModelResult}})::Vector{ModelResult} + local merged = reduce(vcat, parts; init = ModelResult[]) + local seen = Set{String}() + local dups = String[] + for r in merged + r.spec.name in seen ? push!(dups, r.spec.name) : push!(seen, r.spec.name) + end + isempty(dups) || error("Models in more than one shard: $(join(sort!(unique!(dups)), ", "))") + return sort!(merged; by = r -> (r.spec.domain, r.spec.name)) +end + function print_summary(results::Vector{ModelResult}; total_time::Float64 = 0.0) total = length(results) broken = count(r -> r.spec.expected == BROKEN, results) diff --git a/test/runnerTests.jl b/test/runnerTests.jl index e2977c8..3c58cec 100644 --- a/test/runnerTests.jl +++ b/test/runnerTests.jl @@ -77,3 +77,17 @@ end OLT.cleanup!(mgr) end end + +@testset "shards: every n-th model, merged back without loss or duplicates" begin + local spec(name, domain) = OLT.ModelSpec(name, name, domain, 1.0, OLT.VALIDATE, Dict{String, Float64}(), + 0.0, 0.0, "", Dict{String, String}(), "", Set{OLT.Phase}()) + local specs = [spec("M$i", i <= 3 ? "A" : "B") for i in 1:7] + @test OLT.shard_specs(specs, (1, 1)) == specs + local s1, s2 = OLT.shard_specs(specs, (1, 2)), OLT.shard_specs(specs, (2, 2)) + @test [s.name for s in s1] == ["M1", "M3", "M5", "M7"] && [s.name for s in s2] == ["M2", "M4", "M6"] + @test_throws ArgumentError OLT.shard_specs(specs, (3, 2)) + local result(s) = OLT.ModelResult(s, OLT.PhaseResult[], OLT.VALIDATE, "") + local merged = OLT.merge_shards([result.(s2), result.(s1)]) + @test [r.spec.name for r in merged] == [s.name for s in specs] # run_coverage's order (domain, name) + @test_throws ErrorException OLT.merge_shards([result.(s1), result.(s1[1:1])]) +end From 101cc0466da97efff24ce6978e06e07095a76d21 Mon Sep 17 00:00:00 2001 From: JKRT Date: Tue, 13 Oct 2026 12:15:52 +0200 Subject: [PATCH 2/2] CI: a quick smoke on pull requests instead of OM.jl's integration suite OM.jl does not use this package, so its test suite (4 jobs) checked nothing here. smoke.yml (pull requests, master): the package's tests and scripts/smoke.jl, three MSL 3.2.3 models (one per domain) and one MSL 4.1.0 model through every phase, each of which must validate. The MSL install moved into the shared setup action (input msl). Assisted-by: Claude Opus 5.5 --- .github/actions/coverage-setup/action.yml | 27 ++++++++- .github/workflows/coverage.yml | 18 +----- .github/workflows/omjl.yml | 27 --------- .github/workflows/smoke.yml | 74 +++++++++++++++++++++++ scripts/smoke.jl | 26 ++++++++ 5 files changed, 127 insertions(+), 45 deletions(-) delete mode 100644 .github/workflows/omjl.yml create mode 100644 .github/workflows/smoke.yml create mode 100644 scripts/smoke.jl diff --git a/.github/actions/coverage-setup/action.yml b/.github/actions/coverage-setup/action.yml index 3aa046a..28c097a 100644 --- a/.github/actions/coverage-setup/action.yml +++ b/.github/actions/coverage-setup/action.yml @@ -3,7 +3,8 @@ description: >- The rest of the coverage environment, after OM.jl (at the workspace root) and OMLibraryTesting (in OM.jl/OMLibraryTesting.jl) are checked out: the other packages, Julia, the parser library and the precompiled environments, shared by - the shard jobs and the publish job of coverage.yml. + the shard jobs and the publish job of coverage.yml and the smoke job of + smoke.yml; with `msl`, the MSL version installed if OMFrontend does not bundle it. inputs: line: @@ -14,6 +15,10 @@ inputs: description: Julia version required: false default: "1.13" + msl: + description: MSL version the job runs models of (installed when OMFrontend does not bundle it) + required: false + default: "3.2.3" runs: using: composite @@ -53,3 +58,23 @@ runs: - name: Set up OMLibraryTesting shell: bash run: julia --color=yes --project=OM.jl/OMLibraryTesting.jl -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' + + # OMFrontend bundles MSL 3.2.3 and 4.0.0; another version is the installed + # library: Modelica, ModelicaServices and Complex (a single file in the + # repository, a folder with its package.mo when installed). + - name: Install MSL ${{ inputs.msl }} + if: inputs.msl != '3.2.3' && inputs.msl != '4.0.0' + shell: bash + env: + MSL_VERSION: ${{ inputs.msl }} + run: | + src="$RUNNER_TEMP/msl" + git clone -q --depth 1 --branch "v$MSL_VERSION" --filter=blob:none --sparse \ + https://github.com/modelica/ModelicaStandardLibrary.git "$src" + git -C "$src" sparse-checkout set Modelica ModelicaServices + libs="$HOME/.openmodelica/libraries" + mkdir -p "$libs/Complex $MSL_VERSION" + cp -r "$src/Modelica" "$libs/Modelica $MSL_VERSION" + cp -r "$src/ModelicaServices" "$libs/ModelicaServices $MSL_VERSION" + cp "$src/Complex.mo" "$libs/Complex $MSL_VERSION/package.mo" + ls "$libs" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f181207..919e9d7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -106,23 +106,7 @@ jobs: with: line: ${{ inputs.line }} julia: ${{ inputs.julia || '1.13' }} - - # OMFrontend bundles MSL 3.2.3 and 4.0.0; another version is the installed - # library: Modelica, ModelicaServices and Complex (a single file in the - # repository, a folder with its package.mo when installed). - - name: Install MSL ${{ env.MSL }} - if: env.MSL != '3.2.3' && env.MSL != '4.0.0' - run: | - src="$RUNNER_TEMP/msl" - git clone -q --depth 1 --branch "v$MSL" --filter=blob:none --sparse \ - https://github.com/modelica/ModelicaStandardLibrary.git "$src" - git -C "$src" sparse-checkout set Modelica ModelicaServices - libs="$HOME/.openmodelica/libraries" - mkdir -p "$libs/Complex $MSL" - cp -r "$src/Modelica" "$libs/Modelica $MSL" - cp -r "$src/ModelicaServices" "$libs/ModelicaServices $MSL" - cp "$src/Complex.mo" "$libs/Complex $MSL/package.mo" - ls "$libs" + msl: ${{ env.MSL }} # The MAP-LIB reference results (1.3 GB for 3.2.3, 2 GB for 4.1.0) change with # download_refs.sh and the experiment lists only. diff --git a/.github/workflows/omjl.yml b/.github/workflows/omjl.yml deleted file mode 100644 index c141d8d..0000000 --- a/.github/workflows/omjl.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: OM.jl integration - -# Tests this package against the other OM.jl packages at the matching branch -# and runs OM.jl's test suite with it, on Julia 1.13, so that a change -# here cannot break OM.jl unnoticed. The workflow lives in JKRT/OM.jl: -# .github/workflows/omjl-ci.yml. -on: - pull_request: - push: - branches: [master, "1.13"] - workflow_dispatch: - -permissions: - contents: read - actions: write # julia-actions/cache deletes the caches it replaces - -concurrency: - group: omjl-${{ github.ref }} - # Superseded PR runs only; every push to master/1.13 is tested. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - omjl: - uses: JKRT/OM.jl/.github/workflows/omjl-ci.yml@master - with: - package: OMLibraryTesting - julia-versions: '["1.13"]' # OM.jl's packages require Julia 1.13 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..39747d4 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,74 @@ +name: Smoke + +# A quick check of a change: the package's tests and a few MSL models through +# every phase, validation included (scripts/smoke.jl), with the other OM.jl +# packages at the matching branch, on Julia 1.13. OM.jl does not use this +# package, so OM.jl's own test suite is not run here; the coverage runs +# (coverage.yml) measure every model. + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + actions: write # julia-actions/cache deletes the caches it replaces + +concurrency: + group: smoke-${{ github.ref }} + # Superseded PR runs only; every push to master is tested. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + smoke: + name: Package tests and smoke models, Julia 1.13 + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" + LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu + # One thread: OMLibraryTesting runs no parallel workloads. + JULIA_NUM_THREADS: "1" + # One model per domain for the bundled MSL; one for an installed one. + MODELS_323: >- + Modelica.Blocks.Examples.PID_Controller + Modelica.Mechanics.Rotational.Examples.First + Modelica.Electrical.Analog.Examples.CauerLowPassAnalog + MODELS_410: Modelica.Blocks.Examples.PID_Controller + steps: + - name: Check out OM.jl + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + packages: OM + path: . + + - name: Check out OMLibraryTesting + uses: actions/checkout@v4 + with: + path: OM.jl/OMLibraryTesting.jl + + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup + with: + msl: "4.1.0" + + - name: Package tests + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. -e 'using Pkg; Pkg.test()' + + - name: Reference data of the smoke models + working-directory: OM.jl/OMLibraryTesting.jl/reference + run: | + short() { for m in "$@"; do s="${m#Modelica.}"; echo "${s//./_}"; done; } + bash download_refs.sh $(short $MODELS_323) + bash download_refs.sh --msl 4.1.0 $(short $MODELS_410) + + - name: Smoke, MSL 3.2.3 + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/smoke.jl MSL:3.2.3 $MODELS_323 + + - name: Smoke, MSL 4.1.0 + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/smoke.jl MSL:4.1.0 $MODELS_410 diff --git a/scripts/smoke.jl b/scripts/smoke.jl new file mode 100644 index 0000000..9c3c45c --- /dev/null +++ b/scripts/smoke.jl @@ -0,0 +1,26 @@ +#= +Quick smoke test of the harness with OM.jl: the given MSL models through every phase, each of +which must validate. Pull requests run it (.github/workflows/smoke.yml); the coverage runs +measure everything. + + julia --project=. scripts/smoke.jl MSL: ... + +The models' reference data must be there first: reference/download_refs.sh [--msl ] +with the models' short names (Modelica.Blocks.Examples.PID_Controller -> Blocks_Examples_PID_Controller). +=# + +import OMLibraryTesting as OLT + +length(ARGS) >= 2 || error("usage: smoke.jl MSL: ...") +const MSL = ARGS[1] +const MODELS = ARGS[2:end] + +const RESULTS = OLT.run_coverage(; msl_version = MSL, n_workers = 1, timeout = 900.0, + filter = Regex("^(" * join(replace.(MODELS, "." => "\\."), "|") * ")\$")) +const REACHED = Dict(r.spec.name => r.highest for r in RESULTS) +const FAILED = [m for m in MODELS if get(REACHED, m, OLT.UNKNOWN) != OLT.VALIDATE] +for m in MODELS + println(rpad(m, 70), get(REACHED, m, "not run")) +end +isempty(FAILED) || error("$(length(FAILED)) of $(length(MODELS)) $MSL models do not validate: $(join(FAILED, ", "))") +println("Smoke ($MSL): every model validates ($(length(MODELS)))")