diff --git a/.github/actions/coverage-setup/action.yml b/.github/actions/coverage-setup/action.yml new file mode 100644 index 0000000..28c097a --- /dev/null +++ b/.github/actions/coverage-setup/action.yml @@ -0,0 +1,80 @@ +name: Coverage setup +description: >- + The rest of the coverage environment, after OM.jl (at the workspace root) and + OMLibraryTesting (in OM.jl/OMLibraryTesting.jl) are checked out: the other + packages, Julia, the parser library and the precompiled environments, shared by + the shard jobs and the publish job of coverage.yml and the smoke job of + smoke.yml; with `msl`, the MSL version installed if OMFrontend does not bundle it. + +inputs: + line: + description: OM.jl line to check out (empty for the branch the workflow runs on) + required: false + default: "" + julia: + description: Julia version + required: false + default: "1.13" + msl: + description: MSL version the job runs models of (installed when OMFrontend does not bundle it) + required: false + default: "3.2.3" + +runs: + using: composite + steps: + - name: Check out the other packages + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + path: OM.jl + exclude: OMLibraryTesting + line: ${{ inputs.line }} + + - uses: actions/setup-java@v4 # ANTLR, if the parser is built from source + with: + distribution: zulu + java-version: "21" + + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ inputs.julia }} + + # One depot cache for all jobs of the workflow (the default name has the job in it). + - uses: julia-actions/cache@v2 + with: + cache-name: julia-cache;workflow=${{ github.workflow }} + include-matrix: false + + - name: OMParser library + shell: bash + env: + PACKAGE: OMLibraryTesting + run: OM.jl/ci/build-parser.sh + + - name: Set up OM.jl + shell: bash + run: julia --color=yes --project=OM.jl OM.jl/ci/setup.jl + + - name: Set up OMLibraryTesting + shell: bash + run: julia --color=yes --project=OM.jl/OMLibraryTesting.jl -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' + + # OMFrontend bundles MSL 3.2.3 and 4.0.0; another version is the installed + # library: Modelica, ModelicaServices and Complex (a single file in the + # repository, a folder with its package.mo when installed). + - name: Install MSL ${{ inputs.msl }} + if: inputs.msl != '3.2.3' && inputs.msl != '4.0.0' + shell: bash + env: + MSL_VERSION: ${{ inputs.msl }} + run: | + src="$RUNNER_TEMP/msl" + git clone -q --depth 1 --branch "v$MSL_VERSION" --filter=blob:none --sparse \ + https://github.com/modelica/ModelicaStandardLibrary.git "$src" + git -C "$src" sparse-checkout set Modelica ModelicaServices + libs="$HOME/.openmodelica/libraries" + mkdir -p "$libs/Complex $MSL_VERSION" + cp -r "$src/Modelica" "$libs/Modelica $MSL_VERSION" + cp -r "$src/ModelicaServices" "$libs/ModelicaServices $MSL_VERSION" + cp "$src/Complex.mo" "$libs/Complex $MSL_VERSION/package.mo" + ls "$libs" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 9ec77cc..919e9d7 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,6 +11,11 @@ name: MSL coverage # skipPhases and broken marks are removed, so every model goes through every # phase and the documentation shows what really stops each one. # +# Two shard jobs measure every second model each (OMJL_COVERAGE_SHARD), each +# within the 6-hour job limit; the publish job merges their results +# (scripts/merge_coverage.jl, every model exactly once, the same commits) and +# publishes them. Standard runners only. +# # Two MSL versions: 3.2.3 weekly (docs/COVERAGE.md, the README's coverage # section, the site's root) and 4.1.0 every third day (docs/COVERAGE_4.1.0.md, # its own README section, the site's msl-4.1.0/). OMFrontend bundles 3.2.3; @@ -70,9 +75,13 @@ concurrency: jobs: coverage: - name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, Julia ${{ inputs.julia || '1.13' }} + name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, shard ${{ matrix.shard }}/2, Julia ${{ inputs.julia || '1.13' }} runs-on: ubuntu-latest timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + shard: [1, 2] env: JULIA_PKG_PRECOMPILE_AUTO: "0" LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu @@ -92,57 +101,12 @@ jobs: with: path: OM.jl/OMLibraryTesting.jl - - name: Check out the other packages - uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup with: - path: OM.jl - exclude: OMLibraryTesting line: ${{ inputs.line }} - - - uses: actions/setup-java@v4 # ANTLR, if the parser is built from source - with: - distribution: zulu - java-version: "21" - - - uses: julia-actions/setup-julia@v2 - with: - version: ${{ inputs.julia || '1.13' }} - - - uses: julia-actions/cache@v2 - - - name: OMParser library - env: - PACKAGE: OMLibraryTesting - run: OM.jl/ci/build-parser.sh - - - name: Set up OM.jl - run: julia --color=yes --project=OM.jl OM.jl/ci/setup.jl - - - name: Set up OMLibraryTesting - run: julia --color=yes --project=OM.jl/OMLibraryTesting.jl -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' - - - name: The version's documentation - run: | - doc=docs/COVERAGE.md - [ "$MSL" = 3.2.3 ] || doc="docs/COVERAGE_$MSL.md" - echo "COVERAGE_MD=$doc" >> "$GITHUB_ENV" - - # OMFrontend bundles MSL 3.2.3 and 4.0.0; another version is the installed - # library: Modelica, ModelicaServices and Complex (a single file in the - # repository, a folder with its package.mo when installed). - - name: Install MSL ${{ env.MSL }} - if: env.MSL != '3.2.3' && env.MSL != '4.0.0' - run: | - src="$RUNNER_TEMP/msl" - git clone -q --depth 1 --branch "v$MSL" --filter=blob:none --sparse \ - https://github.com/modelica/ModelicaStandardLibrary.git "$src" - git -C "$src" sparse-checkout set Modelica ModelicaServices - libs="$HOME/.openmodelica/libraries" - mkdir -p "$libs/Complex $MSL" - cp -r "$src/Modelica" "$libs/Modelica $MSL" - cp -r "$src/ModelicaServices" "$libs/ModelicaServices $MSL" - cp "$src/Complex.mo" "$libs/Complex $MSL/package.mo" - ls "$libs" + julia: ${{ inputs.julia || '1.13' }} + msl: ${{ env.MSL }} # The MAP-LIB reference results (1.3 GB for 3.2.3, 2 GB for 4.1.0) change with # download_refs.sh and the experiment lists only. @@ -173,19 +137,88 @@ jobs: env: OMJL_COVERAGE_TAG: ci OMJL_COVERAGE_NOSKIP: "1" + OMJL_COVERAGE_SHARD: ${{ matrix.shard }}/2 OMJL_COVERAGE_WORKERS: ${{ inputs.workers || '1' }} OMJL_COVERAGE_TIMEOUT: ${{ inputs.timeout || '600' }} # One thread per worker: OMLibraryTesting runs no parallel workloads. JULIA_NUM_THREADS: "1" run: julia --color=yes --project=. scripts/run_full_coverage.jl + - name: Upload the shard's results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: msl-coverage-${{ env.MSL }}-shard-${{ matrix.shard }} + path: | + commits.txt + OM.jl/OMLibraryTesting.jl/reports/ + OM.jl/OMLibraryTesting.jl/logs/partial_ci_*.jls + if-no-files-found: error + + # Both shards' results as one run: the reports, the documentation and the site. + publish: + name: MSL ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} coverage, publish + needs: coverage + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" + LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu + MSL: ${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} + OMJL_COVERAGE_MSL: MSL:${{ inputs.msl || (github.event.schedule == '0 14 */3 * *' && '4.1.0') || '3.2.3' }} + OMJL_COVERAGE_TAG: ci + NOTE: >- + No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), + 2 shards of ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model + steps: + - name: Check out OM.jl + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + packages: OM + path: . + line: ${{ inputs.line }} + + - name: Check out OMLibraryTesting + uses: actions/checkout@v4 + with: + path: OM.jl/OMLibraryTesting.jl + + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup + with: + line: ${{ inputs.line }} + julia: ${{ inputs.julia || '1.13' }} + + - name: The version's documentation + run: | + doc=docs/COVERAGE.md + [ "$MSL" = 3.2.3 ] || doc="docs/COVERAGE_$MSL.md" + echo "COVERAGE_MD=$doc" >> "$GITHUB_ENV" + + - name: Download the shards' results + uses: actions/download-artifact@v4 + with: + pattern: msl-coverage-${{ env.MSL }}-shard-* + path: ${{ runner.temp }}/shards + + # Both shards measured the same commits (checked out at their start). + - name: Commits measured + run: | + shards="$RUNNER_TEMP/shards" + for f in "$shards"/*/commits.txt; do + cmp -s "$f" "$shards/msl-coverage-$MSL-shard-1/commits.txt" || + { echo "::error::The shards measured different commits"; diff "$shards"/*/commits.txt; exit 1; } + done + cp "$shards/msl-coverage-$MSL-shard-1/commits.txt" commits.txt + cat commits.txt + + - name: Merge the shards + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/merge_coverage.jl "$RUNNER_TEMP"/shards/*/OM.jl/OMLibraryTesting.jl/logs/partial_ci_*.jls + - name: Publish id: publish working-directory: OM.jl/OMLibraryTesting.jl - env: - NOTE: >- - No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), - ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model run: | results=$(ls -t logs/partial_ci_*.jls | head -1) julia --color=yes --project=. scripts/publish_coverage.jl "$results" "$GITHUB_WORKSPACE/commits.txt" "$NOTE" @@ -220,10 +253,6 @@ jobs: - name: Publish the coverage site if: steps.publish.outcome == 'success' && (github.event_name == 'schedule' || inputs.commit) working-directory: OM.jl/OMLibraryTesting.jl - env: - NOTE: >- - No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), - ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model run: | export OMJL_SITE_URL="https://${GITHUB_REPOSITORY_OWNER,,}.github.io/${GITHUB_REPOSITORY#*/}/" site="$RUNNER_TEMP/site" diff --git a/.github/workflows/omjl.yml b/.github/workflows/omjl.yml deleted file mode 100644 index c141d8d..0000000 --- a/.github/workflows/omjl.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: OM.jl integration - -# Tests this package against the other OM.jl packages at the matching branch -# and runs OM.jl's test suite with it, on Julia 1.13, so that a change -# here cannot break OM.jl unnoticed. The workflow lives in JKRT/OM.jl: -# .github/workflows/omjl-ci.yml. -on: - pull_request: - push: - branches: [master, "1.13"] - workflow_dispatch: - -permissions: - contents: read - actions: write # julia-actions/cache deletes the caches it replaces - -concurrency: - group: omjl-${{ github.ref }} - # Superseded PR runs only; every push to master/1.13 is tested. - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - omjl: - uses: JKRT/OM.jl/.github/workflows/omjl-ci.yml@master - with: - package: OMLibraryTesting - julia-versions: '["1.13"]' # OM.jl's packages require Julia 1.13 diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..39747d4 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,74 @@ +name: Smoke + +# A quick check of a change: the package's tests and a few MSL models through +# every phase, validation included (scripts/smoke.jl), with the other OM.jl +# packages at the matching branch, on Julia 1.13. OM.jl does not use this +# package, so OM.jl's own test suite is not run here; the coverage runs +# (coverage.yml) measure every model. + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + actions: write # julia-actions/cache deletes the caches it replaces + +concurrency: + group: smoke-${{ github.ref }} + # Superseded PR runs only; every push to master is tested. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + smoke: + name: Package tests and smoke models, Julia 1.13 + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + JULIA_PKG_PRECOMPILE_AUTO: "0" + LD_LIBRARY_PATH: ${{ github.workspace }}/OM.jl/OMRuntimeExternalC.jl/lib/ext/shared/x86_64-linux-gnu + # One thread: OMLibraryTesting runs no parallel workloads. + JULIA_NUM_THREADS: "1" + # One model per domain for the bundled MSL; one for an installed one. + MODELS_323: >- + Modelica.Blocks.Examples.PID_Controller + Modelica.Mechanics.Rotational.Examples.First + Modelica.Electrical.Analog.Examples.CauerLowPassAnalog + MODELS_410: Modelica.Blocks.Examples.PID_Controller + steps: + - name: Check out OM.jl + uses: JKRT/OM.jl/.github/actions/checkout-siblings@master + with: + packages: OM + path: . + + - name: Check out OMLibraryTesting + uses: actions/checkout@v4 + with: + path: OM.jl/OMLibraryTesting.jl + + - name: Set up + uses: ./OM.jl/OMLibraryTesting.jl/.github/actions/coverage-setup + with: + msl: "4.1.0" + + - name: Package tests + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. -e 'using Pkg; Pkg.test()' + + - name: Reference data of the smoke models + working-directory: OM.jl/OMLibraryTesting.jl/reference + run: | + short() { for m in "$@"; do s="${m#Modelica.}"; echo "${s//./_}"; done; } + bash download_refs.sh $(short $MODELS_323) + bash download_refs.sh --msl 4.1.0 $(short $MODELS_410) + + - name: Smoke, MSL 3.2.3 + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/smoke.jl MSL:3.2.3 $MODELS_323 + + - name: Smoke, MSL 4.1.0 + working-directory: OM.jl/OMLibraryTesting.jl + run: julia --color=yes --project=. scripts/smoke.jl MSL:4.1.0 $MODELS_410 diff --git a/Project.toml b/Project.toml index 795073c..2fdca24 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "OMLibraryTesting" uuid = "4866894a-ca78-4bef-9a5c-0d35f2f81ce1" -version = "0.2.0" +version = "0.3.0" authors = ["John Tinnerholm "] [deps] diff --git a/README.md b/README.md index dffd0d1..dbc0fc2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,9 @@ OMLibraryTesting.print_error_analysis(results) The "Latest coverage" section above and docs/COVERAGE.md are published by the `MSL coverage` workflow (.github/workflows/coverage.yml: MSL 3.2.3 weekly, MSL 4.1.0 every third day, or by hand from the Actions tab with `msl` and `commit` set). It runs `scripts/run_full_coverage.jl` with `OMJL_COVERAGE_NOSKIP=1` (every -model through every phase) and passes the serialized results to the publisher. It also adds the run to the +model through every phase) in two shard jobs (`OMJL_COVERAGE_SHARD=1/2` and `2/2`: every second model each, +within the 6-hour job limit); the publish job merges their results (`scripts/merge_coverage.jl`, which checks +that every model is there once) and passes them to the publisher. It also adds the run to the coverage site, served by GitHub Pages from the `gh-pages` branch (`scripts/publish_site.jl`): `runs//` with the analysis and the per-model report, an index over all runs, and the badges (`badge.json`, models that validate; `badges/.json`); MSL 4.1.0 has the same in `msl-4.1.0/`, docs/COVERAGE_4.1.0.md and its own @@ -105,6 +107,8 @@ section above. The same by hand (`OMJL_COVERAGE_MSL`, default `MSL:3.2.3`, selec ```bash OMJL_COVERAGE_NOSKIP=1 julia --project=. scripts/run_full_coverage.jl +# or in shards, merged: OMJL_COVERAGE_SHARD=1/2 ... and 2/2 ..., then +# julia --project=. scripts/merge_coverage.jl logs/partial__.jls logs/partial__.jls julia --project=. scripts/publish_coverage.jl logs/partial_full_msl_.jls [commits.txt] ["how the run was made"] julia --project=. scripts/publish_site.jl logs/partial_full_msl_.jls reports/.html docs/COVERAGE.md [commits.txt] ["how the run was made"] ``` diff --git a/scripts/merge_coverage.jl b/scripts/merge_coverage.jl new file mode 100644 index 0000000..5d4521c --- /dev/null +++ b/scripts/merge_coverage.jl @@ -0,0 +1,51 @@ +#= +Merge the results of a coverage run's shards (scripts/run_full_coverage.jl with OMJL_COVERAGE_SHARD +"k/n") into one run, written as an unsharded run writes it: the serialized results and the HTML and +Markdown reports. + + julia --project=. scripts/merge_coverage.jl ... + +Writes logs/partial__.jls, with the earliest shard's timestamp so the run keeps its +start date, and reports/coverage__start__end_.{html,md}. Every experiment model +of the version must be in exactly one shard, unless OMJL_COVERAGE_DOMAIN or OMJL_COVERAGE_MODEL +restricted the run. Environment as scripts/run_full_coverage.jl: OMJL_COVERAGE_TAG (default +"full_msl"), OMJL_COVERAGE_MSL (default "MSL:3.2.3"). +=# + +using Dates +using Serialization + +import OMLibraryTesting +const OLT = OMLibraryTesting + +isempty(ARGS) && error("usage: merge_coverage.jl ...") +const TAG = get(ENV, "OMJL_COVERAGE_TAG", "full_msl") +const MSL = get(ENV, "OMJL_COVERAGE_MSL", "MSL:3.2.3") +const LIBRARY_VERSION = OLT.msl_library_version(MSL) +const RESTRICTED = !isempty(get(ENV, "OMJL_COVERAGE_DOMAIN", "")) || !isempty(get(ENV, "OMJL_COVERAGE_MODEL", "")) +const ROOT = joinpath(@__DIR__, "..") |> abspath + +const RESULTS = OLT.merge_shards([deserialize(p)::Vector{OLT.ModelResult} for p in ARGS]) +RESTRICTED || let listed = Set(String(first(split(l, '|'))) + for l in eachline(OLT.recorded_experiments_path("Modelica", LIBRARY_VERSION)) if !isempty(l)) + absent = sort!(collect(setdiff(listed, Set(r.spec.name for r in RESULTS)))) + isempty(absent) || + error("$(length(absent)) of the $(length(listed)) models are in no shard: $(join(first(absent, 10), ", "))") +end + +const STAMPS = [m[1] for m in (match(r"_(\d{4}-\d{2}-\d{2}_\d{4})\.jls$", p) for p in ARGS) if m !== nothing] +const START = isempty(STAMPS) ? Dates.format(Dates.now(), "yyyy-mm-dd_HHMM") : minimum(STAMPS) +const OUT = joinpath(ROOT, "logs", "partial_$(TAG)_$(START).jls") +mkpath(dirname(OUT)) +abspath(OUT) in abspath.(ARGS) && error("$OUT is one of the shards' results") +open(io -> serialize(io, RESULTS), OUT, "w") +@info "Merged" shards = length(ARGS) models = length(RESULTS) path = OUT + +const NAME = "coverage_$(TAG)_start_$(START)_end_$(Dates.format(Dates.now(), "yyyy-mm-dd_HHMM"))" +const CHANGELOG = "Run tag: $TAG. MSL: $MSL. Merged from $(length(ARGS)) shards." +for format in (:html, :markdown) + path = OLT.generate_report(RESULTS; format, filename = NAME * (format === :html ? ".html" : ".md"), + dir = joinpath(ROOT, "reports"), msl_version = LIBRARY_VERSION, + changelog = CHANGELOG) + @info "Report generated" path +end diff --git a/scripts/run_full_coverage.jl b/scripts/run_full_coverage.jl index 45042dc..72d39b8 100644 --- a/scripts/run_full_coverage.jl +++ b/scripts/run_full_coverage.jl @@ -18,6 +18,8 @@ Environment variables: OMJL_COVERAGE_WORKERS Worker processes, models run in parallel (default: nprocs() - 1, at least 1). OMJL_COVERAGE_NOSKIP "1": the registry's skipPhases and expected = "broken" marks removed, so every model goes through every phase (what really stops each one; as docs/COVERAGE.md). + OMJL_COVERAGE_SHARD "k/n": only the k-th of every n models (run_coverage's `shard`), for a run split + over n jobs; scripts/merge_coverage.jl joins their results (default "1/1"). =# using Dates @@ -39,6 +41,10 @@ const TOPHASE = phase_from_string(get(ENV, "OMJL_COVERAGE_TOPHASE", "validat const WORKERS = parse(Int, get(ENV, "OMJL_COVERAGE_WORKERS", string(max(1, nprocs() - 1)))) WORKERS >= 1 || error("OMJL_COVERAGE_WORKERS must be >= 1, got $WORKERS") const NOSKIP = lowercase(get(ENV, "OMJL_COVERAGE_NOSKIP", "")) in ("1", "true", "yes") +const SHARD = let m = match(r"^(\d+)/(\d+)$", get(ENV, "OMJL_COVERAGE_SHARD", "1/1")) + m === nothing && error("OMJL_COVERAGE_SHARD must be k/n, got $(ENV["OMJL_COVERAGE_SHARD"])") + (parse(Int, m[1]), parse(Int, m[2])) +end const RUN_START = Dates.now() const RUN_TS = Dates.format(RUN_START, "yyyy-mm-dd_HHMM") @@ -67,7 +73,7 @@ function noskip_registry()::String end const OVERRIDES = NOSKIP ? noskip_registry() : OMLibraryTesting.default_models_path(LIBRARY_VERSION) -@info "Coverage run starting" tag=TAG msl=MSL timeout=TIMEOUT ts=RUN_TS from=FROMPHASE to=TOPHASE workers=WORKERS noskip=NOSKIP +@info "Coverage run starting" tag=TAG msl=MSL timeout=TIMEOUT ts=RUN_TS from=FROMPHASE to=TOPHASE workers=WORKERS noskip=NOSKIP shard=SHARD @info "Report directory: $REPORT_DIR" @info "Partial-results snapshot: $PARTIAL_PATH" @@ -81,7 +87,8 @@ results = try from_phase = FROMPHASE, to_phase = TOPHASE, overrides = OVERRIDES, - n_workers = WORKERS) + n_workers = WORKERS, + shard = SHARD) catch e @error "run_coverage threw" exception=(e, catch_backtrace()) OMLibraryTesting.ModelResult[] @@ -109,6 +116,7 @@ else "End: ", Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"), ". ", "Timeout per model: ", Int(round(TIMEOUT)), "s. ", NOSKIP ? "Registry skips and broken marks removed. " : "", + SHARD == (1, 1) ? "" : "Shard $(SHARD[1]) of $(SHARD[2]). ", "Total wall-clock: ", string(floor(Int, total_time รท 60)), "m ", string(round(Int, total_time % 60)), "s.") @info "Generating HTML report" filename=fname diff --git a/scripts/smoke.jl b/scripts/smoke.jl new file mode 100644 index 0000000..9c3c45c --- /dev/null +++ b/scripts/smoke.jl @@ -0,0 +1,26 @@ +#= +Quick smoke test of the harness with OM.jl: the given MSL models through every phase, each of +which must validate. Pull requests run it (.github/workflows/smoke.yml); the coverage runs +measure everything. + + julia --project=. scripts/smoke.jl MSL: ... + +The models' reference data must be there first: reference/download_refs.sh [--msl ] +with the models' short names (Modelica.Blocks.Examples.PID_Controller -> Blocks_Examples_PID_Controller). +=# + +import OMLibraryTesting as OLT + +length(ARGS) >= 2 || error("usage: smoke.jl MSL: ...") +const MSL = ARGS[1] +const MODELS = ARGS[2:end] + +const RESULTS = OLT.run_coverage(; msl_version = MSL, n_workers = 1, timeout = 900.0, + filter = Regex("^(" * join(replace.(MODELS, "." => "\\."), "|") * ")\$")) +const REACHED = Dict(r.spec.name => r.highest for r in RESULTS) +const FAILED = [m for m in MODELS if get(REACHED, m, OLT.UNKNOWN) != OLT.VALIDATE] +for m in MODELS + println(rpad(m, 70), get(REACHED, m, "not run")) +end +isempty(FAILED) || error("$(length(FAILED)) of $(length(MODELS)) $MSL models do not validate: $(join(FAILED, ", "))") +println("Smoke ($MSL): every model validates ($(length(MODELS)))") diff --git a/src/OMLibraryTesting.jl b/src/OMLibraryTesting.jl index 0a0f5d5..8a3b9c1 100644 --- a/src/OMLibraryTesting.jl +++ b/src/OMLibraryTesting.jl @@ -57,6 +57,7 @@ export load_models, load_models_by_domain, list_domains, load_registry_meta export load_reference_csv, load_comparison_signals, validate_against_reference export discover_experiments, record_experiments, merge_overrides!, clear_discovery_cache!, auto_detect_references! export run_coverage, run_coverage_and_report, run_model, WorkerManager, WorkerPool, ensure_pool! +export shard_specs, merge_shards export generate_report, print_summary export coverage_funnel, publish_site, site_history export categorize_error, categorize_results, print_error_analysis diff --git a/src/runner.jl b/src/runner.jl index 8e857ea..112fa15 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -723,6 +723,8 @@ experiment list, `overrides` the registry and, beside it, the reference data (`default_models_path(version)`, `reference_dir`). Optionally filter by `domain` regex, single `model` name, or general `filter` regex. +`shard = (k, n)` runs the `k`-th of every `n` of those models (`shard_specs`), for a run split +over `n` processes or CI jobs; `merge_shards` joins their results. Use `from_phase` and `to_phase` to restrict which phases are run. For example, `from_phase=BACKEND` skips the frontend phase (useful when frontend is 100% pass). @@ -750,7 +752,8 @@ function run_coverage(; library::String = "Modelica", from_phase::Phase = FRONTEND, to_phase::Phase = VALIDATE, check_sim_code::Bool = false, - n_workers::Int = max(1, nprocs() - 1))::Vector{ModelResult} + n_workers::Int = max(1, nprocs() - 1), + shard::Tuple{Int, Int} = (1, 1))::Vector{ModelResult} t_start = time() n_workers >= 1 || throw(ArgumentError("n_workers must be >= 1, got $n_workers")) phases = Base.filter(p -> Int(from_phase) <= Int(p) <= Int(to_phase), phases) @@ -773,6 +776,11 @@ function run_coverage(; library::String = "Modelica", error("No models found matching domain: $domain") end end + if shard != (1, 1) + local n_all = length(specs) + specs = shard_specs(specs, shard) + @info "Shard $(shard[1]) of $(shard[2]): $(length(specs)) of $n_all models" + end n_broken = count(s -> s.expected == BROKEN, specs) n_active = length(specs) - n_broken total_models = length(specs) @@ -827,6 +835,35 @@ function run_coverage(; library::String = "Modelica", return results end +""" + shard_specs(specs, shard) -> Vector{ModelSpec} + +The `k`-th of every `n` models for `shard = (k, n)`, in the order given (run_coverage's: by +domain, then name), so each domain is spread over the shards; `(1, 1)` is all of them. +""" +function shard_specs(specs::Vector{ModelSpec}, shard::Tuple{Int, Int})::Vector{ModelSpec} + local (k, n) = shard + 1 <= k <= n || throw(ArgumentError("shard must be (k, n) with 1 <= k <= n, got $shard")) + return specs[k:n:end] +end + +""" + merge_shards(parts) -> Vector{ModelResult} + +The results of a run's shards as one run, in run_coverage's order (domain, then name). A +model in more than one shard is an error. +""" +function merge_shards(parts::Vector{Vector{ModelResult}})::Vector{ModelResult} + local merged = reduce(vcat, parts; init = ModelResult[]) + local seen = Set{String}() + local dups = String[] + for r in merged + r.spec.name in seen ? push!(dups, r.spec.name) : push!(seen, r.spec.name) + end + isempty(dups) || error("Models in more than one shard: $(join(sort!(unique!(dups)), ", "))") + return sort!(merged; by = r -> (r.spec.domain, r.spec.name)) +end + function print_summary(results::Vector{ModelResult}; total_time::Float64 = 0.0) total = length(results) broken = count(r -> r.spec.expected == BROKEN, results) diff --git a/test/runnerTests.jl b/test/runnerTests.jl index e2977c8..3c58cec 100644 --- a/test/runnerTests.jl +++ b/test/runnerTests.jl @@ -77,3 +77,17 @@ end OLT.cleanup!(mgr) end end + +@testset "shards: every n-th model, merged back without loss or duplicates" begin + local spec(name, domain) = OLT.ModelSpec(name, name, domain, 1.0, OLT.VALIDATE, Dict{String, Float64}(), + 0.0, 0.0, "", Dict{String, String}(), "", Set{OLT.Phase}()) + local specs = [spec("M$i", i <= 3 ? "A" : "B") for i in 1:7] + @test OLT.shard_specs(specs, (1, 1)) == specs + local s1, s2 = OLT.shard_specs(specs, (1, 2)), OLT.shard_specs(specs, (2, 2)) + @test [s.name for s in s1] == ["M1", "M3", "M5", "M7"] && [s.name for s in s2] == ["M2", "M4", "M6"] + @test_throws ArgumentError OLT.shard_specs(specs, (3, 2)) + local result(s) = OLT.ModelResult(s, OLT.PhaseResult[], OLT.VALIDATE, "") + local merged = OLT.merge_shards([result.(s2), result.(s1)]) + @test [r.spec.name for r in merged] == [s.name for s in specs] # run_coverage's order (domain, name) + @test_throws ErrorException OLT.merge_shards([result.(s1), result.(s1[1:1])]) +end