From ccd7d6cdcb9b172e11be068d32a63d55d42f940e Mon Sep 17 00:00:00 2001 From: Bill Hlavacek Date: Thu, 17 Sep 2026 15:58:50 -0600 Subject: [PATCH] fix(information-criteria): report how many simulations of the best fit were run beside how many produced a usable log-likelihood (#741) A simulation that fails, that scores nothing, or that scores a different number of points from the rest is left out of the mean behind AIC, BIC and AICc, and replicated_information_criteria set `replicates` to the surviving count. That number was honest. Nothing in the file said a run had been lost, and the header explained `replicates` as though none could be: "the best fit is run best_fit_replicates times and log_likelihood is the mean over those runs". So a log-likelihood averaged over 3 of 10 runs read exactly like one averaged over 3 of 3. The `%d of %d simulation(s)` warning went to the log alone. The console's only mention of the count rides on a clause that appears once two runs have produced a value, so the case where nine of ten were lost was the case it hid. This matters where the criteria are used. _emit_information_criteria's own docstring says they "rank this fit against competing models"; that comparison is made by a reader holding two of these files side by side. A model whose best fit lost seven of its ten runs is scored on a mean over the three that worked, optimistic in the way #720 described, and can win the AIC comparison on it. InformationCriteria now carries replicates_requested and replicated_information_criteria takes `requested`; _compute_information_criteria passes len(jobs). The file carries a replicates_requested line beside replicates and, when they differ, says above the numbers how many runs produced nothing and that the mean is optimistic by however much they would have pulled it down. The console says the same, on its own line rather than folded into the standard-error clause, so a single surviving run still reports what was lost. A criteria object built without a request count is written exactly as before, and a run that lost nothing says nothing. The averaging is unchanged and no minimum-success threshold was added, unlike #720's confirmation ranking. That bar exists because candidate means were ranked against each other inside one run, and a candidate measured over one run could beat one measured over ten. This file holds one parameter set the run has already settled on, so there is nothing for a survivor mean to beat unfairly, and refusing to emit criteria over a low count would remove information rather than add it. ADR-0131 is amended rather than rewritten: its file example and its sentence about dropped runs describe the file as it shipped, with a note pointing here. ADR-0146's Consequences entry, which recorded this as deliberately out of scope for #720, is corrected -- it judged the averaging and missed that the sibling file did not even say it had dropped anything, a gap #740 itself widened. Signed-off-by: Bill Hlavacek --- CHANGELOG.md | 27 ++++ ...-a-few-draws-is-the-luckiest-draw-again.md | 12 ++ ...e-lucky-draw-the-stage-exists-to-remove.md | 6 +- docs/algorithms.rst | 11 +- docs/config_keys.rst | 2 +- docs/troubleshooting.rst | 12 +- pybnf/algorithms/base.py | 56 ++++++-- pybnf/objective.py | 26 ++-- tests/test_information_criteria_replicates.py | 120 +++++++++++++++++- 9 files changed, 246 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9e7ce5..2b8dc746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -252,6 +252,33 @@ All notable changes to PyBNF are documented below. This project adheres to by default. Both surfaces are documented under gradient-based fitting. ### Fixed +- **`information_criteria.txt` reports how many simulations of the best fit were run beside + how many produced a usable log-likelihood, so an average over 3 of 10 runs no longer reads + as an average over 3 of 3 (#741, ADR-0131 amended).** A simulation that fails, scores + nothing, or scores a different number of points from the rest is left out of the mean, and + `replicated_information_criteria` set `replicates` to the surviving count. That number was + honest, but nothing in the file said a run had been lost, and the header explained + `replicates` as though none could be: "the best fit is run `best_fit_replicates` times and + log_likelihood is the mean over those runs". The `%d of %d simulation(s)` warning went to + the log alone, and the console's only mention of the count rides on a clause that appears + once two runs have produced a value — so the case where nine of ten were lost was the case + it hid. + This matters where the criteria are used. The writer's own docstring says they "rank this + fit against competing models"; that comparison is made by a reader holding two of these + files side by side. A model whose best fit lost seven of its ten runs is scored on a mean + over the three that worked, optimistic in the way #720 described, and could win the AIC + comparison on it. The file now carries `replicates_requested` beside `replicates`, says in + words how many runs produced nothing and that the mean is optimistic by however much they + would have pulled it down, and prints the same on the console — including when a single run + survived. `InformationCriteria` carries the count and + `replicated_information_criteria(..., requested=N)` sets it; a criteria object built without + one is written exactly as before. + The averaging is unchanged and no minimum-success threshold was added, unlike #720's + confirmation ranking. That bar exists because candidate means were ranked against each + other inside one run and a candidate measured over one run could beat one measured over + ten; this file holds one parameter set the run has already settled on, so there is nothing + for a survivor mean to beat unfairly, and refusing to emit criteria over a low count would + remove information rather than add it. - **The best-fit confirmation stage ranks a candidate on all of its replicate runs, so one that fails most of them can no longer be pinned as the run's answer (#720, ADR-0146).** The stage exists to stop a fit reporting whichever parameter set got a lucky simulation: it diff --git a/docs/adr/0131-the-information-criteria-of-a-stochastic-fit-are-the-mean-over-several-simulations-of-the-best-fit-with-the-spread-beside-them-because-one-simulation-is-a-draw-and-the-log-of-a-mean-likelihood-over-a-few-draws-is-the-luckiest-draw-again.md b/docs/adr/0131-the-information-criteria-of-a-stochastic-fit-are-the-mean-over-several-simulations-of-the-best-fit-with-the-spread-beside-them-because-one-simulation-is-a-draw-and-the-log-of-a-mean-likelihood-over-a-few-draws-is-the-luckiest-draw-again.md index 30c45485..b8622aa2 100644 --- a/docs/adr/0131-the-information-criteria-of-a-stochastic-fit-are-the-mean-over-several-simulations-of-the-best-fit-with-the-spread-beside-them-because-one-simulation-is-a-draw-and-the-log-of-a-mean-likelihood-over-a-few-draws-is-the-luckiest-draw-again.md +++ b/docs/adr/0131-the-information-criteria-of-a-stochastic-fit-are-the-mean-over-several-simulations-of-the-best-fit-with-the-spread-beside-them-because-one-simulation-is-a-draw-and-the-log-of-a-mean-likelihood-over-a-few-draws-is-the-luckiest-draw-again.md @@ -5,6 +5,11 @@ Accepted. Completes #659, the best-fit confirmation stage, for the one end-of-fit artifact that stage left on a single simulation. +Amended by #741 (2026-09-17): the file gained a `replicates_requested` line beside +`replicates`, and says in words how many runs produced nothing when the two differ. The +example and the sentence about dropped runs below describe the file as this ADR shipped it; +nothing about the decision changed, only what the file discloses about it. See ADR-0146. + ## The defect `Results/information_criteria.txt` reports AIC, BIC and AICc from the full normalized @@ -73,6 +78,13 @@ client they run one after another. A run that fails or scores nothing is left ou rest are left out too, since a sum over a different `n` is a different quantity. A profiled noise scale (ADR-0108) is averaged over the same runs. +That last rule is the one #741 came back to. Reporting only the number used left an average +over 3 of 10 runs reading exactly like an average over 3 of 3, in the file whose whole +purpose is a comparison between two such averages; the fix reports the number run beside it +and says what was lost. The averaging itself is unchanged, and no minimum-success rule was +added here -- #720's threshold exists because candidate means were ranked against each other, +and there is no competing candidate in this file. + ## Why the mean of the log-likelihoods, and not the log of the mean likelihood #676 says this has to be decided before any code, and that the two numbers answer different diff --git a/docs/adr/0146-a-best-fit-candidate-must-produce-a-usable-objective-value-in-more-than-half-of-its-replicate-runs-to-win-the-confirmation-ranking-because-an-average-over-only-the-runs-that-worked-is-the-lucky-draw-the-stage-exists-to-remove.md b/docs/adr/0146-a-best-fit-candidate-must-produce-a-usable-objective-value-in-more-than-half-of-its-replicate-runs-to-win-the-confirmation-ranking-because-an-average-over-only-the-runs-that-worked-is-the-lucky-draw-the-stage-exists-to-remove.md index 61a652d0..ab60940b 100644 --- a/docs/adr/0146-a-best-fit-candidate-must-produce-a-usable-objective-value-in-more-than-half-of-its-replicate-runs-to-win-the-confirmation-ranking-because-an-average-over-only-the-runs-that-worked-is-the-lucky-draw-the-stage-exists-to-remove.md +++ b/docs/adr/0146-a-best-fit-candidate-must-produce-a-usable-objective-value-in-more-than-half-of-its-replicate-runs-to-win-the-confirmation-ranking-because-an-average-over-only-the-runs-that-worked-is-the-lucky-draw-the-stage-exists-to-remove.md @@ -168,4 +168,8 @@ reported as the first. the one pinned parameter set. That is an estimate of a single parameter set rather than a ranking, so the selection argument above does not apply to it in the same way; with the winner now required to be reliable, the parameter set it describes is one whose runs mostly - work. Left as it is. + work. The averaging is left as it is. What this ADR missed is that the sibling file did not + even *say* it had dropped anything, so the two files stopped reporting the same kind of + measurement the same way the moment this change landed; #741 closed that gap by reporting + the count of runs made beside the count used. The threshold argument above still does not + transfer, and no minimum-success rule was added there. diff --git a/docs/algorithms.rst b/docs/algorithms.rst index f57ad6d1..ab98734d 100644 --- a/docs/algorithms.rst +++ b/docs/algorithms.rst @@ -1019,10 +1019,13 @@ answer, so :math:`\ln L` from a single simulation is a noisy number and an AIC b would change from one run to the next for the same parameter set. At the end of such a fit the best fit is therefore simulated ``best_fit_replicates`` times, the same number of runs the :ref:`best-fit confirmation stage ` uses, and :math:`\ln L` is -the mean over those runs. ``Results/information_criteria.txt`` says how many runs went into -it (``replicates``) and how far they spread (``log_likelihood_standard_error``); AIC, BIC -and AICc each carry twice that standard error, so two models whose AIC values differ by -less than it have not been told apart. The mean of the log-likelihoods is the same average +the mean over those runs. ``Results/information_criteria.txt`` says how many simulations +were run (``replicates_requested``), how many of them produced a usable log-likelihood and +so went into the mean (``replicates``), and how far those spread +(``log_likelihood_standard_error``); AIC, BIC and AICc each carry twice that standard error, +so two models whose AIC values differ by less than it have not been told apart. When the two +counts differ the file says how many runs produced nothing, since the mean is then over the +runs that worked and the comparison between two models is made on those means. The mean of the log-likelihoods is the same average the confirmation stage reports for the objective value, so the two files describe the same parameter set the same way. It is a lower bound on the log marginal likelihood, and the reported spread says how far below it could be. diff --git a/docs/config_keys.rst b/docs/config_keys.rst index ba8a12ca..e64db700 100644 --- a/docs/config_keys.rst +++ b/docs/config_keys.rst @@ -1706,7 +1706,7 @@ Algorithm Options A candidate has to produce a usable objective value in more than half of those runs to be eligible to win. A failed or non-finite run is a bad outcome rather than a missing measurement, so averaging only the runs that worked would hand the answer back to whichever parameter set got a lucky simulation. The table marks each candidate ``confirmed`` or not and shows how many runs each average came from. - The same number of runs goes into ``Results/information_criteria.txt``: the winning parameter set is simulated ``best_fit_replicates`` more times at fresh seeds, and the log-likelihood behind its AIC, BIC and AICc is the mean over those runs, with its standard error reported beside it. With this key at 0 or 1 the criteria come from a single simulation, as they always did. + The same number of runs goes into ``Results/information_criteria.txt``: the winning parameter set is simulated ``best_fit_replicates`` more times at fresh seeds, and the log-likelihood behind its AIC, BIC and AICc is the mean over those runs, with its standard error reported beside it. That file reports both how many simulations were run and how many produced a usable log-likelihood, and says so when the two differ, since the mean is then over the runs that worked and is optimistic. With this key at 0 or 1 the criteria come from a single simulation, as they always did. The cost is ``best_fit_candidates`` times ``best_fit_replicates`` simulations after the search has ended, all submitted at once, plus ``best_fit_replicates`` more for the information criteria when the objective is a likelihood, and times ``smoothing`` again when that is also set. Ignored when no model is stochastic. diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 951ef849..b6978f5d 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -276,9 +276,15 @@ to come from one more simulation of the best fit, which for a stochastic model i draw, so its ``log_likelihood`` and the AIC built on it moved from run to run and disagreed with the averaged objective value above. The best fit is now simulated ``best_fit_replicates`` times for it too, at fresh seeds, and ``log_likelihood`` is the mean -over those runs. The file carries a ``replicates`` line saying how many runs that was and a -``log_likelihood_standard_error`` line saying how far they spread; AIC, BIC and AICc each -carry twice that. Two models whose AIC values differ by less than it have not been told +over those runs. The file carries a ``replicates_requested`` line saying how many +simulations were run, a ``replicates`` line saying how many of them produced a usable +log-likelihood and so went into the mean, and a ``log_likelihood_standard_error`` line +saying how far those spread; AIC, BIC and AICc each carry twice that. The two counts differ +when a simulation fails, scores nothing, or scores a different number of points from the +rest, and the file then says how many runs produced nothing and that the mean is optimistic +by however much they would have pulled it down. That matters most where these criteria are +used: comparing two models means comparing two averages, and one taken over three runs of +ten is not the same measurement as one taken over three of three. Two models whose AIC values differ by less than it have not been told apart, so raise ``best_fit_replicates`` if you need to separate them. The periodic checkpoint ``information_criteria_backup.txt`` stays a single simulation, and says so in the same line. diff --git a/pybnf/algorithms/base.py b/pybnf/algorithms/base.py index 7d22e59b..d66cf0d7 100644 --- a/pybnf/algorithms/base.py +++ b/pybnf/algorithms/base.py @@ -2122,7 +2122,8 @@ def _compute_information_criteria(self, best_pset, replicates=1, client=None): for the same parameter set. The simulations are independent, so they go out through ``client`` together when one is given, and run one after another in-process otherwise. A replicate that fails or scores nothing is left out and - the count reported is the number that were used; a profiled noise scale + the count reported is the number that were used, beside the number that were run + so the report can say how many were lost (#741); a profiled noise scale (ADR-0108) is averaged over the same runs. Every failure is logged and swallowed (returns ``None``): the run has @@ -2220,7 +2221,12 @@ def _compute_information_criteria(self, best_pset, replicates=1, client=None): logger.warning('%d of %d simulation(s) of the best fit produced a usable ' 'log-likelihood for the information criteria' % (len(log_likelihoods), len(jobs))) - return replicated_information_criteria(log_likelihoods, k, n) + # The count of simulations run travels with the criteria so the report can say + # that the average is over fewer runs than the fit asked for, rather than + # leaving a reader to spot that `replicates` is below best_fit_replicates and + # guess why (#741). + return replicated_information_criteria(log_likelihoods, k, n, + requested=len(jobs)) except Exception: logger.exception('Failed to compute information criteria for the best fit') return None @@ -2471,6 +2477,10 @@ def _emit_information_criteria(self, ic, name='', preamble=()): aicc_str = ('%.10g' % ic.aicc) if ic.aicc is not None else 'n/a (n <= k+1)' se = ic.log_likelihood_standard_error se_str = ('%.10g' % se) if se is not None else 'n/a (one simulation)' + # `lost` is clamped, and a missing request count reads as "nothing was lost", so a + # partly-filled criteria object can never make the end of a finished run raise (#741). + requested = ic.replicates if ic.replicates_requested is None else ic.replicates_requested + lost = max(0, requested - ic.replicates) lines = list(preamble) + [ '# Information criteria for the best-fit parameter set (lower is better).', '# Valid for a likelihood objective only (normal / lognormal / lnnormal / laplace /', @@ -2480,15 +2490,38 @@ def _emit_information_criteria(self, ic, name='', preamble=()): '# AIC = 2k - 2*lnL', '# BIC = k*ln(n) - 2*lnL', '# AICc = AIC + 2k(k+1)/(n-k-1) (undefined when n <= k+1)', - '# replicates is how many simulations of the best fit log_likelihood is averaged', - '# over. One for a deterministic model, whose simulation is exact. For a', + '# replicates_requested is how many simulations of the best fit were run for', + '# this file. One for a deterministic model, whose simulation is exact. For a', '# stochastic model every simulation is a draw, so at the end of the fit the', - '# best fit is run best_fit_replicates times and log_likelihood is the mean over', - '# those runs; log_likelihood_standard_error is the uncertainty in that mean,', - '# and AIC, BIC and AICc each carry twice it. Two models whose AIC values', - '# differ by less than that have not been told apart by these runs.', + '# best fit is run best_fit_replicates times.', + '# replicates is how many of those produced a usable log-likelihood, and so how', + '# many log_likelihood is the mean over. A simulation that fails, that scores', + '# nothing, or that scores a different number of points from the rest is left', + '# out, so this can be lower than replicates_requested; when it is, a line', + '# below says so. Comparing two models means comparing two of these averages,', + '# and one taken over fewer runs than were asked for is worth knowing about', + '# before the comparison is made.', + '# log_likelihood_standard_error is the uncertainty in that mean, and AIC, BIC', + '# and AICc each carry twice it. Two models whose AIC values differ by less', + '# than that have not been told apart by these runs.', + ] + # Above the numbers rather than below them, so it is read before they are. + if lost: + lines += [ + '#', + '# %d of the %d simulations of the best fit produced no usable log-likelihood,' + % (lost, requested), + '# so log_likelihood below is the mean over the %d that did. A run that' + % ic.replicates, + '# produces nothing is a bad outcome rather than a missing measurement, so', + '# that mean is optimistic by however much the lost runs would have pulled', + '# it down. Weigh it against another model\'s accordingly, and look into why', + '# the simulations are failing. The log says what went wrong with each.', + ] + lines += [ 'k\t%d' % ic.k, 'n\t%d' % ic.n, + 'replicates_requested\t%d' % requested, 'replicates\t%d' % ic.replicates, 'log_likelihood\t%.10g' % ic.log_likelihood, 'log_likelihood_standard_error\t%s' % se_str, @@ -2513,6 +2546,13 @@ def _emit_information_criteria(self, ic, name='', preamble=()): print1('Information criteria (best fit): AIC=%.6g BIC=%.6g AICc=%s ' '(k=%d, n=%d, lnL=%.6g%s)' % (ic.aic, ic.bic, aicc_str, ic.k, ic.n, ic.log_likelihood, spread)) + if lost: + # Not folded into `spread`, which only appears once two runs have produced + # a value -- the case where the most was lost is the case it would hide. + print1(' %d of the %d simulations of the best fit produced no usable ' + 'log-likelihood, so that is the mean over the %d that did and is ' + 'optimistic by however much the rest would have pulled it down.' + % (lost, requested, ic.replicates)) return True def multistart_records(self): diff --git a/pybnf/objective.py b/pybnf/objective.py index 1fc31fba..124ed9cc 100644 --- a/pybnf/objective.py +++ b/pybnf/objective.py @@ -49,11 +49,15 @@ # measured (#676, ADR-0131): over how many simulations of the best fit it is averaged, and # the uncertainty in that average. One and ``None`` for a deterministic model, whose single # simulation is exact; :func:`replicated_information_criteria` fills them for a stochastic one. +# ``replicates_requested`` is how many simulations were run to get there, which is larger +# than ``replicates`` when some of them produced nothing usable (#741). It defaults to one +# beside ``replicates``, so the plain criteria of an exact simulation describe one run that +# was asked for and one that delivered. InformationCriteria = namedtuple( 'InformationCriteria', ['k', 'n', 'log_likelihood', 'aic', 'bic', 'aicc', - 'replicates', 'log_likelihood_standard_error'], - defaults=(1, None)) + 'replicates', 'log_likelihood_standard_error', 'replicates_requested'], + defaults=(1, None, 1)) def information_criteria(log_likelihood, k, n): @@ -81,7 +85,7 @@ def information_criteria(log_likelihood, k, n): aic=aic, bic=bic, aicc=aicc) -def replicated_information_criteria(log_likelihoods, k, n): +def replicated_information_criteria(log_likelihoods, k, n, requested=None): """AIC / BIC / AICc from several log-likelihoods of ONE parameter set, each from its own simulation, with the standard error of their mean (#676, ADR-0131). @@ -99,11 +103,16 @@ def replicated_information_criteria(log_likelihoods, k, n): :param log_likelihoods: one full normalized log-likelihood per simulation; at least one. :param k: free-parameter count, as for :func:`information_criteria`. :param n: scored point count, the same for every simulation. + :param requested: how many simulations were run to produce those values. Larger than + ``len(log_likelihoods)`` when some of them failed, scored nothing, or scored a + different number of points; the report says so rather than leaving the reader to + notice that the count is below what the fit asked for (#741). Defaults to the number + of values, which is the truth whenever nothing was lost. :returns: an :class:`InformationCriteria` whose ``log_likelihood`` is the mean, - ``replicates`` the number of values, and ``log_likelihood_standard_error`` the - standard error of that mean, or ``None`` when there is a single value. AIC, BIC - and AICc each carry twice that standard error, since each is ``-2 lnL`` plus a - constant. + ``replicates`` the number of values, ``replicates_requested`` the number of + simulations behind them, and ``log_likelihood_standard_error`` the standard error of + that mean, or ``None`` when there is a single value. AIC, BIC and AICc each carry + twice that standard error, since each is ``-2 lnL`` plus a constant. """ values = [float(v) for v in log_likelihoods] if not values: @@ -111,7 +120,8 @@ def replicated_information_criteria(log_likelihoods, k, n): mean = fmean(values) se = stdev(values) / math.sqrt(len(values)) if len(values) >= 2 else None return information_criteria(mean, k, n)._replace( - replicates=len(values), log_likelihood_standard_error=se) + replicates=len(values), log_likelihood_standard_error=se, + replicates_requested=len(values) if requested is None else int(requested)) def likelihood_information_criteria(objective, sim_data_dict, exp_data_dict, pset, k): diff --git a/tests/test_information_criteria_replicates.py b/tests/test_information_criteria_replicates.py index 923c3747..40f1ffbb 100644 --- a/tests/test_information_criteria_replicates.py +++ b/tests/test_information_criteria_replicates.py @@ -9,6 +9,13 @@ simulated ``best_fit_replicates`` times at the end of a stochastic fit, the log-likelihood is the mean over those runs, and the file says how many runs and how far they spread. +A run that fails or scores nothing is left out of that mean, and the file used to report +only the surviving count under a header that explained it as ``best_fit_replicates`` -- so a +log-likelihood averaged over 3 of 10 runs read exactly like one averaged over 3 of 3, which +matters because the comparison these criteria exist for is made between two such averages. +The count of runs made now travels beside the count used, and the file and the console say +what was lost (#741). + Three layers, in this order. * ``pybnf.objective.replicated_information_criteria``: the arithmetic. @@ -111,6 +118,21 @@ def test_a_single_value_is_the_plain_criteria_with_no_spread(): == objective.information_criteria(-7.5, k=1, n=4)) +def test_the_requested_count_rides_along_and_defaults_to_the_number_of_values(): + """An average over three of ten runs and an average over three of three are different + measurements, and the criteria have to carry which one they are (#741).""" + lost = objective.replicated_information_criteria([-10.0, -12.0, -14.0], k=2, n=10, + requested=10) + assert (lost.replicates, lost.replicates_requested) == (3, 10) + + whole = objective.replicated_information_criteria([-10.0, -12.0, -14.0], k=2, n=10) + assert (whole.replicates, whole.replicates_requested) == (3, 3) + + # Only the bookkeeping differs; the arithmetic is the same average either way. + assert lost.log_likelihood == whole.log_likelihood == -12.0 + assert lost.aic == whole.aic + + def test_the_plain_criteria_describe_one_exact_simulation(): ic = objective.information_criteria(-3.0, k=1, n=5) assert ic.replicates == 1 @@ -202,6 +224,9 @@ def test_a_run_that_cannot_be_scored_is_left_out_and_the_count_says_so(tmp_path) ic = algo._compute_information_criteria(_ps(1.0), replicates=3) assert (ic.replicates, ic.log_likelihood) == (2, -2.0) + # How many were run travels with how many were used, so the report can say what was + # lost rather than leaving the reader to notice a count below best_fit_replicates (#741). + assert ic.replicates_requested == 3 def test_a_run_with_a_non_finite_log_likelihood_is_left_out_too(tmp_path): @@ -243,6 +268,8 @@ def test_runs_that_scored_a_different_number_of_points_are_left_out(tmp_path): algo = _ic_algo(tmp_path) ic = algo._compute_information_criteria(_ps(1.0), replicates=3) + # Dropped for a different reason than a failure, and counted as lost all the same. + assert ic.replicates_requested == 3 assert (ic.n, ic.replicates, ic.log_likelihood) == (2, 2, -3.0) @@ -269,8 +296,12 @@ def test_the_file_reports_the_replicate_count_and_the_spread(tmp_path): assert kv['replicates'] == '3' assert kv['log_likelihood'] == '-12' assert float(kv['log_likelihood_standard_error']) == pytest.approx(2.0 / math.sqrt(3)) - assert list(kv) == ['k', 'n', 'replicates', 'log_likelihood', + assert list(kv) == ['k', 'n', 'replicates_requested', 'replicates', 'log_likelihood', 'log_likelihood_standard_error', 'AIC', 'BIC', 'AICc'] + # Nothing was lost, so the file says so by saying nothing (#741). + assert kv['replicates_requested'] == '3' + assert 'produced no usable log-likelihood' not in open( + algo.res_dir + '/information_criteria.txt').read() def test_the_file_for_one_exact_simulation_says_so(tmp_path): @@ -295,6 +326,93 @@ def test_the_console_line_names_the_runs_and_the_spread(tmp_path, monkeypatch, c assert 'over' not in capsys.readouterr().out +def test_the_file_says_how_many_runs_produced_nothing(tmp_path): + """``replicates`` alone reads as the number of runs that were asked for, so an average + over 3 of 10 was indistinguishable from one over 3 of 3 (#741).""" + algo = _ic_algo(tmp_path) + + algo._emit_information_criteria(objective.replicated_information_criteria( + [-10.0, -12.0, -14.0], k=2, n=10, requested=10)) + + text = open(algo.res_dir + '/information_criteria.txt').read() + kv = _kv(algo.res_dir + '/information_criteria.txt') + assert kv['replicates_requested'] == '10' + assert kv['replicates'] == '3' + assert '7 of the 10 simulations of the best fit produced no usable log-likelihood' in text + assert 'log_likelihood below is the mean over the 3 that did' in text + # The header stops telling the reader that replicates is best_fit_replicates. + assert 'replicates_requested is how many simulations of the best fit were run' in text + assert 'replicates is how many of those produced a usable log-likelihood' in text + + +def test_the_console_says_how_many_runs_produced_nothing(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(printing, 'verbosity', 1) + algo = _ic_algo(tmp_path) + + algo._emit_information_criteria(objective.replicated_information_criteria( + [-10.0, -12.0, -14.0], k=2, n=10, requested=10)) + + out = capsys.readouterr().out + assert '7 of the 10 simulations of the best fit produced no usable log-likelihood' in out + assert 'optimistic' in out + + +def test_the_console_still_says_so_when_a_single_run_survived(tmp_path, monkeypatch, capsys): + """The standard error is None with one value, so the existing ``over N runs`` clause + does not print -- the case where the most was lost is the one it would hide (#741).""" + monkeypatch.setattr(printing, 'verbosity', 1) + algo = _ic_algo(tmp_path) + + algo._emit_information_criteria(objective.replicated_information_criteria( + [-10.0], k=2, n=10, requested=10)) + + out = capsys.readouterr().out + assert 'over 1 runs' not in out # the old clause is still absent + assert '9 of the 10 simulations of the best fit produced no usable log-likelihood' in out + + +def test_a_run_that_lost_nothing_says_nothing_about_losses(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(printing, 'verbosity', 1) + algo = _ic_algo(tmp_path) + + algo._emit_information_criteria(objective.replicated_information_criteria( + [-10.0, -12.0], k=2, n=10, requested=2)) + + assert 'produced no usable log-likelihood' not in capsys.readouterr().out + assert 'produced no usable log-likelihood' not in open( + algo.res_dir + '/information_criteria.txt').read() + + +def test_a_criteria_object_with_no_request_count_is_written_without_a_loss_note(tmp_path): + """A report writer runs at the end of a finished fit, so a partly-filled criteria + object must never be the reason it raises.""" + algo = _ic_algo(tmp_path) + ic = objective.information_criteria(-12.0, k=2, n=10)._replace( + replicates=3, replicates_requested=None) + + assert algo._emit_information_criteria(ic) is True + + kv = _kv(algo.res_dir + '/information_criteria.txt') + assert kv['replicates_requested'] == '3' + assert 'produced no usable log-likelihood' not in open( + algo.res_dir + '/information_criteria.txt').read() + + +def test_the_end_to_end_stage_reports_what_it_lost(tmp_path, _sync_dask): + """Through the real path: four simulations are run, two score nothing, and the file + that a model-selection comparison reads says so (#741).""" + LOGLIK.update({5: [-1.0, -1.0], 6: None, 7: None, 8: [-3.0, -3.0]}) + algo = _ic_algo(tmp_path, candidates=1, replicates=4) + algo.trajectory.add(_ps(1.0), 1.0, 'best') + + ic = algo._compute_information_criteria(_ps(1.0), replicates=4, client=_FakeClient()) + algo._emit_information_criteria(ic) + + assert (ic.replicates, ic.replicates_requested) == (2, 4) + text = open(algo.res_dir + '/information_criteria.txt').read() + assert '2 of the 4 simulations of the best fit produced no usable log-likelihood' in text + + # --------------------------------------------------------------------------- # # The end-of-fit wiring: how many runs, and through what # --------------------------------------------------------------------------- #