From 32d93b95ac48b9d0b3529594dbafe778d83bd43b Mon Sep 17 00:00:00 2001 From: JKRT Date: Tue, 6 Oct 2026 08:20:29 +0200 Subject: [PATCH] Coverage site: a page per run, an index and badges on GitHub Pages (0.1.2) publish_site (src/site.jl, scripts/publish_site.jl) adds a run to the gh-pages branch: runs//, history.toml, index.html with the trend, shields.io badges. The MSL coverage workflow publishes each run; README badge. Assisted-by: Claude Opus 5.5 --- .github/workflows/coverage.yml | 37 ++++- Project.toml | 3 +- README.md | 10 +- scripts/publish_coverage.jl | 4 +- scripts/publish_site.jl | 32 ++++ src/OMLibraryTesting.jl | 3 + src/site.jl | 264 +++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/siteTests.jl | 41 +++++ 9 files changed, 388 insertions(+), 7 deletions(-) create mode 100644 scripts/publish_site.jl create mode 100644 src/site.jl create mode 100644 test/siteTests.jl diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e4b90f0..f240343 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -3,7 +3,9 @@ name: MSL coverage # The coverage of the Modelica Standard Library examples through OM.jl # (flatten, translate, simulate, validate against the reference results), # published as documentation: docs/COVERAGE.md and the "Latest coverage" -# section of README.md (scripts/publish_coverage.jl). +# section of README.md (scripts/publish_coverage.jl), and on the coverage site +# (GitHub Pages from the gh-pages branch, scripts/publish_site.jl): one page +# per measured date, an index over all runs and the README badge. # # The run is a no-skip run (OMJL_COVERAGE_NOSKIP=1): the registry's # skipPhases and broken marks are removed, so every model goes through every @@ -15,7 +17,8 @@ name: MSL coverage # on the runner; record_experiments() refreshes the file). # # Manual runs (Actions > MSL coverage > Run workflow) commit the documentation -# back when `commit` is set; the weekly run always does. A schedule only fires +# back and publish the run on the site when `commit` is set; the weekly run +# always does. A schedule only fires # from the default branch, so it measures that branch's line. on: @@ -38,7 +41,7 @@ on: required: false default: "600" commit: - description: Commit docs/COVERAGE.md and README.md back to the branch + description: Commit docs/COVERAGE.md and README.md back to the branch and add the run to the coverage site type: boolean required: false default: false @@ -169,3 +172,31 @@ jobs: if git diff --cached --quiet; then echo "Coverage unchanged"; exit 0; fi git commit -m "Coverage of $(date -u +%Y-%m-%d) [skip ci]" git push origin "HEAD:${GITHUB_REF_NAME}" + + # The run on the coverage site: runs//, the index and the badges on + # the gh-pages branch (created on the first run), which Pages serves. + - name: Publish the coverage site + if: steps.publish.outcome == 'success' && (github.event_name == 'schedule' || inputs.commit) + working-directory: OM.jl/OMLibraryTesting.jl + env: + NOTE: >- + No-skip run on GitHub Actions (ubuntu-latest, Julia ${{ inputs.julia || '1.13' }}), + ${{ inputs.workers || '1' }} workers, ${{ inputs.timeout || '600' }} s per model + run: | + export OMJL_SITE_URL="https://${GITHUB_REPOSITORY_OWNER,,}.github.io/${GITHUB_REPOSITORY#*/}/" + site="$RUNNER_TEMP/site" + if git fetch --depth=1 origin gh-pages; then + git worktree add -B gh-pages "$site" FETCH_HEAD + else + git worktree add --orphan -b gh-pages "$site" + fi + results=$(ls -t logs/partial_ci_*.jls | head -1) + report=$(ls -t reports/coverage_ci_*.html | head -1) + julia --color=yes --project=. scripts/publish_site.jl "$site" "$results" "$report" docs/COVERAGE.md "$GITHUB_WORKSPACE/commits.txt" "$NOTE" + cd "$site" + git add -A + if git diff --cached --quiet; then echo "Site unchanged"; exit 0; fi + git -c user.name="github-actions[bot]" -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -q -m "Coverage of $(date -u +%Y-%m-%d)" + git push origin gh-pages + echo "Coverage site: $OMJL_SITE_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/Project.toml b/Project.toml index 74f5cfb..447363a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "OMLibraryTesting" uuid = "4866894a-ca78-4bef-9a5c-0d35f2f81ce1" -version = "0.1.1" +version = "0.1.2" authors = ["John Tinnerholm "] [deps] @@ -13,6 +13,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" DoubleEnded = "1bc4e7c2-c1e1-11e9-03fe-315bbef43fde" ImmutableList = "4a558cac-c1ed-11e9-20da-3584bcd8709a" ListUtil = "b2a72b7e-c1ef-11e9-3a87-6ddc5900c366" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MetaModelica = "9d7f2a79-07b5-5542-8b19-c0100dda6b06" OM = "2f925a0b-2436-437b-858e-49aee461894b" OMBackend = "9400bd5f-7561-42f6-8760-03756753da50" diff --git a/README.md b/README.md index 03f0e65..a165fd4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # OMLibraryTesting.jl +[![MSL coverage](https://img.shields.io/endpoint?url=https%3A%2F%2Fopenmodelica.github.io%2FOMLibraryTesting.jl%2Fbadge.json)](https://openmodelica.github.io/OMLibraryTesting.jl/) + A Julia package for running MSL (Modelica Standard Library) coverage tests against [OM.jl](https://github.com/OpenModelica/OM.jl). It auto-discovers all experiment models in the MSL using `omc`, runs each through the OM.jl pipeline (flatten -> translate -> simulate -> validate), and generates HTML coverage reports. ## Overview @@ -21,6 +23,8 @@ Results are compared against Dymola-generated reference trajectories from the MA Measured 2026-10-04 on 425 MSL example models: frontend 423 (99.5 %), backend 398 (93.6 %), simulate 324 (76.2 %), validate 299 (70.4 %). Per domain and what stops the rest: [docs/COVERAGE.md](docs/COVERAGE.md). +Every published run, by date, with the trend: the [coverage site](https://openmodelica.github.io/OMLibraryTesting.jl/). + ## Directory Structure ``` @@ -34,6 +38,7 @@ OMLibraryTesting.jl/ │ ├── comparison.jl # CSV trajectory comparison │ ├── analysis.jl # Error categorization and analysis utilities │ ├── report.jl # HTML report generation +│ ├── site.jl # Coverage site: one page per run, index, badges │ └── testsuite.jl # Fetch + run the upstream OpenModelica testsuite (rtest / OM.jl modes) ├── models/ │ └── models.toml # Per-model overrides (atol, reltol, expected phase, signal mapping) @@ -78,11 +83,14 @@ OMLibraryTesting.print_error_analysis(results) The "Latest coverage" section above and docs/COVERAGE.md are published by the `MSL coverage` workflow (.github/workflows/coverage.yml: weekly, or by hand from the Actions tab with `commit` set). It runs `scripts/run_full_coverage.jl` with `OMJL_COVERAGE_NOSKIP=1` (every model through every phase) and passes the -serialized results to the publisher. The same by hand: +serialized results to the publisher. It also adds the run to the coverage site, served by GitHub Pages from +the `gh-pages` branch (`scripts/publish_site.jl`): `runs//` with the analysis and the per-model report, an +index over all runs, and the badges (`badge.json`, models that validate; `badges/.json`). The same by hand: ```bash OMJL_COVERAGE_NOSKIP=1 julia --project=. scripts/run_full_coverage.jl julia --project=. scripts/publish_coverage.jl logs/partial_full_msl_.jls [commits.txt] ["how the run was made"] +julia --project=. scripts/publish_site.jl logs/partial_full_msl_.jls reports/.html docs/COVERAGE.md [commits.txt] ["how the run was made"] ``` ## Hot Validation Loop (`scripts/warm_validate.jl`) diff --git a/scripts/publish_coverage.jl b/scripts/publish_coverage.jl index abec790..101426a 100644 --- a/scripts/publish_coverage.jl +++ b/scripts/publish_coverage.jl @@ -26,8 +26,8 @@ const NOTE = length(ARGS) >= 3 ? ARGS[3] : "" const ROOT = joinpath(@__DIR__, "..") |> abspath const MEASURED = Dates.format(Dates.unix2datetime(mtime(ARGS[1])), "yyyy-mm-dd") -ok(r) = r.highest != OLT.BROKEN && r.highest != OLT.UNKNOWN -reached(r, p) = ok(r) && r.highest >= p +const ok = OLT.coverage_ok # the definitions the coverage site uses (src/site.jl) +const reached = OLT.coverage_reached domainOf(r) = join(split(r.spec.domain, '.')[1:min(2, end)], '.') short(name) = replace(name, "Modelica." => "") function firstError(r) diff --git a/scripts/publish_site.jl b/scripts/publish_site.jl new file mode 100644 index 0000000..982bdcf --- /dev/null +++ b/scripts/publish_site.jl @@ -0,0 +1,32 @@ +#= +Add a coverage run to the coverage site (GitHub Pages, gh-pages branch): runs// with the +analysis and the per-model report, the index over all runs and the badges (src/site.jl). + + julia --project=. scripts/publish_site.jl [report.html] [COVERAGE.md] [commits.txt] [note] + + is a serialized Vector{ModelResult} (the partial_*.jls of scripts/run_full_coverage.jl); +the date is the one in its name (partial___.jls), else its modification date. +[report.html] is the run's generate_report output, [COVERAGE.md] the analysis of +scripts/publish_coverage.jl, [commits.txt] " " lines, [note] one line on how +the run was made. An empty argument skips that part. OMJL_SITE_URL (optional) is the site's address, +shown in the badge snippet. +=# + +using Dates +using Serialization + +import OMLibraryTesting +const OLT = OMLibraryTesting + +length(ARGS) >= 2 || error("usage: publish_site.jl [report.html] [COVERAGE.md] [commits.txt] [note]") +arg(i) = length(ARGS) >= i ? ARGS[i] : "" +const RESULTS_PATH = ARGS[2] +const RESULTS = deserialize(RESULTS_PATH)::Vector{OLT.ModelResult} +const NAMED = match(r"_(\d{4}-\d{2}-\d{2})_\d{4}\.jls$", RESULTS_PATH) +const DATE = NAMED === nothing ? Date(Dates.unix2datetime(mtime(RESULTS_PATH))) : Date(NAMED[1]) +const COMMITS = isfile(arg(5)) ? + [String(strip(l)) for l in eachline(arg(5)) if occursin(r"^\S+\.jl \S+$", strip(l))] : String[] + +dir = OLT.publish_site(ARGS[1], RESULTS; date = DATE, report = arg(3), analysis = arg(4), + commits = COMMITS, note = arg(6), url = get(ENV, "OMJL_SITE_URL", "")) +@info "Coverage site updated" run = dir funnel = OLT.coverage_funnel(RESULTS) diff --git a/src/OMLibraryTesting.jl b/src/OMLibraryTesting.jl index 2f4aae4..0a0f5d5 100644 --- a/src/OMLibraryTesting.jl +++ b/src/OMLibraryTesting.jl @@ -34,6 +34,7 @@ module OMLibraryTesting using TOML using Dates using Distributed +using Markdown import OM import OMBackend import OMFrontend @@ -45,6 +46,7 @@ include("discovery.jl") include("runner.jl") include("analysis.jl") include("report.jl") +include("site.jl") include("rerun_failed.jl") include("testsuite.jl") @@ -56,6 +58,7 @@ export load_reference_csv, load_comparison_signals, validate_against_reference export discover_experiments, record_experiments, merge_overrides!, clear_discovery_cache!, auto_detect_references! export run_coverage, run_coverage_and_report, run_model, WorkerManager, WorkerPool, ensure_pool! export generate_report, print_summary +export coverage_funnel, publish_site, site_history export categorize_error, categorize_results, print_error_analysis export models_with_error, error_for_model, run_frontend_coverage export subcategorize_meta_errors diff --git a/src/site.jl b/src/site.jl new file mode 100644 index 0000000..754f35b --- /dev/null +++ b/src/site.jl @@ -0,0 +1,264 @@ +#= +* This file is part of OpenModelica. +* +* Copyright (c) 1998-CurrentYear, Open Source Modelica Consortium (OSMC), +* c/o Linköpings universitet, Department of Computer and Information Science, +* SE-58183 Linköping, Sweden. +* +* All rights reserved. +* +* THIS PROGRAM IS PROVIDED UNDER THE TERMS OF GPL VERSION 3 LICENSE OR +* THIS OSMC PUBLIC LICENSE (OSMC-PL) VERSION 1.2. +* ANY USE, REPRODUCTION OR DISTRIBUTION OF THIS PROGRAM CONSTITUTES +* RECIPIENT'S ACCEPTANCE OF THE OSMC PUBLIC LICENSE OR THE GPL VERSION 3, +* ACCORDING TO RECIPIENTS CHOICE. +* +* The OpenModelica software and the Open Source Modelica +* Consortium (OSMC) Public License (OSMC-PL) are obtained +* from OSMC, either from the above address, +* from the URLs: http:www.ida.liu.se/projects/OpenModelica or +* http:www.openmodelica.org, and in the OpenModelica distribution. +* GNU version 3 is obtained from: http:www.gnu.org/copyleft/gpl.html. +* +* This program is distributed WITHOUT ANY WARRANTY; without +* even the implied warranty of MERCHANTABILITY or FITNESS +* FOR A PARTICULAR PURPOSE, EXCEPT AS EXPRESSLY SET FORTH +* IN THE BY RECIPIENT SELECTED SUBSIDIARY LICENSE CONDITIONS OF OSMC-PL. +* +* See the full OSMC Public License conditions for more details. +* +*/ =# + +#= A static coverage site (served by GitHub Pages from the gh-pages branch): one directory per + measured date, an index over all runs with the trend, and shields.io endpoint badges. + scripts/publish_site.jl adds a run; the MSL coverage workflow calls it after each run. + + site/ + index.html all runs, the latest funnel, the trend + history.toml one [[runs]] entry per date (the index is rebuilt from it) + badge.json shields.io endpoint: models that validate + badges/.json the same per phase + latest/index.html redirect to the latest run + runs//index.html the run's analysis (docs/COVERAGE.md rendered) + runs//report.html the per-model report (generate_report) + runs//summary.toml =# + +const SITE_PHASES = (FRONTEND, BACKEND, SIMULATE, VALIDATE) + +coverage_ok(r::ModelResult) = r.highest != BROKEN && r.highest != UNKNOWN +coverage_reached(r::ModelResult, p::Phase) = coverage_ok(r) && r.highest >= p + +""" + coverage_funnel(results) -> Vector{Pair{String, Int}} + +The number of models that reached each phase, frontend through validate. A BROKEN or UNKNOWN +result reaches none. +""" +coverage_funnel(results::Vector{ModelResult}) = + [PHASE_NAMES[p] => count(r -> coverage_reached(r, p), results) for p in SITE_PHASES] + +""" + publish_site(site, results; date = today(), report = "", analysis = "", commits = String[], + note = "", library = "MSL 3.2.3", url = "") -> String + +Add a coverage run to the static site in the directory `site` (created if missing) and rebuild +the index, the badges and the `latest` redirect. Runs are keyed by `date`: a second run of the +same date replaces the first. `report` is the HTML report of `generate_report`, `analysis` the +Markdown of scripts/publish_coverage.jl (docs/COVERAGE.md), `commits` lines such as +"OMBackend.jl f661a4a", `url` the site's address (for the badge snippet). Returns the run's directory. +""" +function publish_site(site::AbstractString, results::Vector{ModelResult}; date::Date = today(), + report::AbstractString = "", analysis::AbstractString = "", + commits::Vector{String} = String[], note::AbstractString = "", + library::AbstractString = "MSL 3.2.3", url::AbstractString = "") + local run = Dict{String, Any}("date" => string(date), "models" => length(results), + "note" => String(note), "commits" => commits) + for (p, c) in coverage_funnel(results) + run[p] = c + end + local dir = joinpath(site, "runs", string(date)) + mkpath(dir) + isempty(report) || cp(report, joinpath(dir, "report.html"); force = true) + open(io -> TOML.print(io, run; sorted = true), joinpath(dir, "summary.toml"), "w") + write(joinpath(dir, "index.html"), _site_run_page(run, isempty(analysis) ? "" : read(analysis, String), + !isempty(report), library)) + local history = filter(r -> r["date"] != run["date"], site_history(site)) + push!(history, run) + sort!(history; by = r -> r["date"], rev = true) + open(io -> TOML.print(io, Dict("runs" => history); sorted = true), joinpath(site, "history.toml"), "w") + _write_site_index(site, history, library, url) + return dir +end + +""" + site_history(site) -> Vector{Dict{String, Any}} + +The runs recorded in `site/history.toml`, newest first (empty for a new site). +""" +function site_history(site::AbstractString) + local path = joinpath(site, "history.toml") + isfile(path) || return Dict{String, Any}[] + return sort!(Vector{Dict{String, Any}}(get(TOML.parsefile(path), "runs", Any[])); by = r -> r["date"], rev = true) +end + +_site_pct(n, total) = total == 0 ? 0.0 : 100n / total +_site_pctstr(n, total) = string(round(_site_pct(n, total); digits = 1), " %") + +function _badge_color(pct) + pct >= 90 && return "brightgreen" + pct >= 75 && return "green" + pct >= 60 && return "yellowgreen" + pct >= 40 && return "yellow" + pct >= 20 && return "orange" + return "red" +end + +_json_string(s) = string('"', replace(s, "\\" => "\\\\", "\"" => "\\\""), '"') + +function _write_badge(path, label, n, total) + mkpath(dirname(path)) + write(path, string("{\"schemaVersion\": 1, \"label\": ", _json_string(label), + ", \"message\": ", _json_string("$(_site_pctstr(n, total)) ($n/$total)"), + ", \"color\": ", _json_string(_badge_color(_site_pct(n, total))), "}\n")) +end + +function _write_site_index(site, history, library, url) + local latest = first(history) + _write_badge(joinpath(site, "badge.json"), "$library validate", latest["validate"], latest["models"]) + for p in SITE_PHASES + local name = PHASE_NAMES[p] + _write_badge(joinpath(site, "badges", "$name.json"), "$library $name", latest[name], latest["models"]) + end + mkpath(joinpath(site, "latest")) + write(joinpath(site, "latest", "index.html"), string(""" + + Latest coverageLatest coverage: $(latest["date"]) + """)) + touch(joinpath(site, ".nojekyll")) # plain files, no Jekyll build + write(joinpath(site, "index.html"), _site_index_page(history, library, url)) +end + +const _SITE_CSS = """ +:root { --bg: #ffffff; --fg: #111418; --muted: #5b6470; --line: #d9dee5; --accent: #1f5fa8; + --bar: #e8eef6; --frontend: #7aa7d9; --backend: #4f86c6; --simulate: #2c66ab; --validate: #0f3f7a; } +@media (prefers-color-scheme: dark) { + :root { --bg: #0f1216; --fg: #e8ebef; --muted: #9aa4b0; --line: #2a313a; --accent: #7fb2ec; + --bar: #1b222b; --frontend: #3d6a99; --backend: #4f86c6; --simulate: #7aa7d9; --validate: #b5d2f2; } +} +* { box-sizing: border-box; } +body { margin: 0; background: var(--bg); color: var(--fg); + font: 15px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; } +main { max-width: 960px; margin: 0 auto; padding: 24px 16px 48px; } +h1 { font-size: 1.6em; margin: 0 0 4px; } h2 { font-size: 1.2em; margin: 32px 0 8px; } +p.lead, .muted { color: var(--muted); } +a { color: var(--accent); } +nav { margin-bottom: 16px; } nav a { margin-right: 16px; } +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; width: 100%; font-variant-numeric: tabular-nums; } +th, td { text-align: right; padding: 6px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; } +th:first-child, td:first-child { text-align: left; } +th { font-weight: 600; color: var(--muted); } +.funnel { display: grid; grid-template-columns: 6em 1fr 8.5em; gap: 6px 12px; align-items: center; } +.funnel .track { background: var(--bar); height: 14px; border-radius: 3px; overflow: hidden; } +.funnel .fill { height: 100%; } +.funnel .num { text-align: right; font-variant-numeric: tabular-nums; } +svg text { fill: var(--muted); font-size: 11px; } +.legend span { display: inline-block; margin-right: 14px; } .legend i { display: inline-block; width: 10px; height: 10px; margin-right: 5px; border-radius: 2px; } +code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.9em; } +pre { background: var(--bar); padding: 10px 12px; border-radius: 4px; overflow-x: auto; } +.analysis td { white-space: normal; } +""" + +_site_page(title, body) = string(""" + +""", html_escape(String(title)), "\n
\n", body, + "\n
\n") + +function _site_funnel(run) + local io = IOBuffer() + println(io, "
") + for p in SITE_PHASES + local name = PHASE_NAMES[p] + local pct = _site_pct(run[name], run["models"]) + println(io, "
$name
", + "
$(run[name]) · $(_site_pctstr(run[name], run["models"]))
") + end + println(io, "
") + return String(take!(io)) +end + +#= The share of models reaching each phase over the runs, oldest left; one SVG polyline per phase. =# +function _site_trend(history) + local runs = reverse(history) + local w, h, l, r, t, b = 900, 220, 40, 12, 10, 26 + local n = length(runs) + local x(i) = n == 1 ? l + (w - l - r) / 2 : l + (i - 1) * (w - l - r) / (n - 1) + local y(pct) = t + (100 - pct) * (h - t - b) / 100 + local io = IOBuffer() + println(io, "") + for g in 0:25:100 + println(io, "", + "$g%") + end + local step = max(1, cld(n, 8)) + for (i, run) in enumerate(runs) + (i - 1) % step == 0 || i == n || continue + local anchor = n == 1 ? "middle" : i == 1 ? "start" : i == n ? "end" : "middle" + println(io, "$(run["date"])") + end + for p in SITE_PHASES + local name = PHASE_NAMES[p] + local pts = join(("$(round(x(i); digits = 1)),$(round(y(_site_pct(run[name], run["models"])); digits = 1))" + for (i, run) in enumerate(runs)), " ") + n > 1 && println(io, "") + for (i, run) in enumerate(runs) + println(io, "", + "$(run["date"]) $name $(run[name])/$(run["models"])") + end + end + println(io, "") + println(io, "
", join(("$(PHASE_NAMES[p])" + for p in SITE_PHASES)), "
") + return String(take!(io)) +end + +function _site_index_page(history, library, url) + local latest = first(history) + local io = IOBuffer() + println(io, "

OM.jl coverage of the $(html_escape(String(library))) examples

") + println(io, "

Each experiment model goes through OM.jl's pipeline: flatten (frontend), translate (backend), ", + "simulate, and validate against the reference trajectories. Measured by ", + "OMLibraryTesting.jl.

") + println(io, "

Latest: $(latest["date"]), $(latest["models"]) models

") + println(io, _site_funnel(latest)) + println(io, "

Per run

", _site_trend(history)) + println(io, "
", + join(("" for p in SITE_PHASES)), "") + for run in history + local cells = join(("" + for p in SITE_PHASES)) + println(io, "", cells, + "") + end + println(io, "
DateModels$(PHASE_NAMES[p])
$(run[PHASE_NAMES[p]]) ($(_site_pctstr(run[PHASE_NAMES[p]], run["models"])))
$(run["date"])$(run["models"])per model
") + println(io, "

Badge

The share of models that validate in the latest run ", + "(badge.json; badges/<phase>.json per phase), for a README:

") + local base = isempty(url) ? "<this site>/" : html_escape(String(rstrip(url, '/') * "/")) + println(io, "
[![Coverage](https://img.shields.io/endpoint?url=$(base)badge.json)]($base)
") + return _site_page("OM.jl coverage", String(take!(io))) +end + +function _site_run_page(run, analysis, hasReport, library) + local io = IOBuffer() + print(io, "") + if !isempty(analysis) # it names the note and the commits itself + println(io, "
", Markdown.html(Markdown.parse(analysis)), "
") + else + println(io, "

$(html_escape(String(library))) coverage, $(run["date"])

", _site_funnel(run)) + isempty(run["note"]) || println(io, "

", html_escape(run["note"]), "

") + isempty(run["commits"]) || println(io, "

Commits: ", html_escape(join(run["commits"], ", ")), "

") + end + return _site_page("OM.jl coverage $(run["date"])", String(take!(io))) +end diff --git a/test/runtests.jl b/test/runtests.jl index 6b9e6c0..c8f797d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,4 +14,5 @@ using Test include("comparisonTests.jl") include("registryTests.jl") include("runnerTests.jl") + include("siteTests.jl") end diff --git a/test/siteTests.jl b/test/siteTests.jl new file mode 100644 index 0000000..e99ccf8 --- /dev/null +++ b/test/siteTests.jl @@ -0,0 +1,41 @@ +#= +Tests for the coverage site (src/site.jl): the funnel, a run's files, the history across runs +(a second run of the same date replaces the first), the badges and the index. +=# + +using Dates +using TOML + +@testset "Coverage site" begin + local spec(name) = OLT.ModelSpec(name, name, "Test", 1.0, OLT.VALIDATE, Dict{String, Float64}(), + 0.0, 0.0, "", Dict{String, String}(), "", Set{OLT.Phase}()) + local result(name, highest) = OLT.ModelResult(spec(name), OLT.PhaseResult[], highest, "") + local results = [result("A", OLT.VALIDATE), result("B", OLT.SIMULATE), result("C", OLT.FRONTEND), + result("D", OLT.BROKEN), result("E", OLT.UNKNOWN)] + + @test OLT.coverage_funnel(results) == ["frontend" => 3, "backend" => 2, "simulate" => 2, "validate" => 1] + + mktempdir() do site + local report = joinpath(site, "in.html"); write(report, "per model") + local analysis = joinpath(site, "in.md"); write(analysis, "# Coverage\n\n| Phase | Models |\n|---|---|\n| frontend | 3 |\n") + local dir = OLT.publish_site(site, results; date = Date(2026, 10, 3), report, analysis, + commits = ["OM.jl abc1234"], note = "test run") + @test dir == joinpath(site, "runs", "2026-10-03") + @test read(joinpath(dir, "report.html"), String) == "per model" + @test occursin("", read(joinpath(dir, "index.html"), String)) + @test TOML.parsefile(joinpath(dir, "summary.toml"))["validate"] == 1 + @test isfile(joinpath(site, ".nojekyll")) + + OLT.publish_site(site, results[1:2]; date = Date(2026, 10, 4)) + OLT.publish_site(site, results[1:4]; date = Date(2026, 10, 3)) # replaces the first run + local history = OLT.site_history(site) + @test [r["date"] for r in history] == ["2026-10-04", "2026-10-03"] + @test history[2]["models"] == 4 + @test occursin("../runs/2026-10-04/", read(joinpath(site, "latest", "index.html"), String)) + local badge = read(joinpath(site, "badge.json"), String) + @test occursin("\"message\": \"50.0 % (1/2)\"", badge) && occursin("\"color\": \"yellow\"", badge) + @test isfile(joinpath(site, "badges", "frontend.json")) + local index = read(joinpath(site, "index.html"), String) + @test occursin("runs/2026-10-04/report.html", index) && occursin("