From 17294c4f140c45a3391ea63d8a9c0d4bc009217f Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Tue, 22 Sep 2026 10:44:24 +0200 Subject: [PATCH] Add support for paramsets --- CHANGELOG.md | 4 + common | 2 +- .../logistic_regression/config.vsh.yaml | 26 ++++- src/methods/logistic_regression/script.py | 7 +- src/workflows/run_benchmark/config.vsh.yaml | 20 ++++ src/workflows/run_benchmark/main.nf | 95 +++++++++---------- 6 files changed, 98 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c3041e..da37338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ * Added `control_methods/random_labels` component (PR #22). +* `run_benchmark`: run parameterised methods once per named paramset from `info.variants` or the new `--paramsets` file, tag scores with `paramset_name`/`paramset`, and allow `--methods_include`/`--methods_exclude` to target `.`, using the new paramset helpers in `common/nextflow_helpers/helper.nf` (PR #23). + +* `methods/logistic_regression`: added an `--inverse_regularization_strength` argument with three `info.variants` as an example of a parameterised method (PR #23). + ## MAJOR CHANGES * Updated `api` files (PR #5). diff --git a/common b/common index 6eda5dd..d2678f2 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 6eda5dd0b9c46b43ee36fc379152e079c26f25d3 +Subproject commit d2678f23bc1ac2ec0a87574b6622bbbc00828e6f diff --git a/src/methods/logistic_regression/config.vsh.yaml b/src/methods/logistic_regression/config.vsh.yaml index e570c03..3110b32 100644 --- a/src/methods/logistic_regression/config.vsh.yaml +++ b/src/methods/logistic_regression/config.vsh.yaml @@ -43,13 +43,29 @@ links: info: # Which normalisation method this component prefers to use (required). preferred_normalization: log_cp10k + # Named parameter sets to evaluate in the benchmark (optional). The + # `run_benchmark` workflow runs the method once per variant, passing the + # listed values as arguments, and tags the scores with the variant name. + # Only the listed variants are run, so to also evaluate the default + # arguments, list them as a variant with their values spelled out (viash + # drops empty variants from the config). Without `variants`, the method is + # run once with its default arguments. + variants: + strong_regularization: + inverse_regularization_strength: 0.1 + default_regularization: + inverse_regularization_strength: 1.0 + weak_regularization: + inverse_regularization_strength: 10.0 # Component-specific parameters (optional) -# arguments: -# - name: "--n_neighbors" -# type: "integer" -# default: 5 -# description: Number of neighbors to use. +arguments: + - name: "--inverse_regularization_strength" + type: double + default: 1.0 + description: | + Inverse of the regularization strength (`C` in scikit-learn). Smaller + values specify stronger regularization. # Resources required to run the component resources: diff --git a/src/methods/logistic_regression/script.py b/src/methods/logistic_regression/script.py index 6ab5782..dd10667 100644 --- a/src/methods/logistic_regression/script.py +++ b/src/methods/logistic_regression/script.py @@ -7,7 +7,8 @@ par = { 'input_train': 'resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad', 'input_test': 'resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad', - 'output': 'output.h5ad' + 'output': 'output.h5ad', + 'inverse_regularization_strength': 1.0 } meta = { 'name': 'logistic_regression' @@ -23,7 +24,9 @@ print('Train model', flush=True) # ... train model ... -classifier = sklearn.linear_model.LogisticRegression() +classifier = sklearn.linear_model.LogisticRegression( + C=par['inverse_regularization_strength'] +) classifier.fit(input_train.obsm["X_pca"], input_train.obs["label"].astype(str)) print('Generate predictions', flush=True) diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index ef85e36..66ea08d 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -55,16 +55,36 @@ argument_groups: These arguments are mutually exclusive, so only `--methods_include` OR `--methods_exclude` can set but not both. arguments: + - name: "--paramsets" + type: file + direction: input + required: false + description: | + A yaml file mapping method ids to named parameter sets, e.g.: + + logistic_regression: + strong_regularization: + inverse_regularization_strength: 0.1 + weak_regularization: + inverse_regularization_strength: 10.0 + + Each listed method is run once per parameter set, with the given arguments + and with `paramset_name`/`paramset` attached to its scores. + If not provided, the parameter sets default to the `info.variants` + defined in the method components' configs. A method without + `info.variants` is run once with its default parameter set. - name: "--methods_include" type: string multiple: true description: | A list of method ids to include. If specified, only these methods will be run. + For parameterised methods, a single paramset can be targeted as `.`, e.g. `logistic_regression.strong_regularization`. - name: "--methods_exclude" type: string multiple: true description: | A list of method ids to exclude. If specified, all methods except the ones listed will be run. + For parameterised methods, a single paramset can be excluded as `.`, e.g. `logistic_regression.strong_regularization`. - name: Metric filtering description: | Use these arguments to filter metrics by name. By default, all metrics are diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index de2c94e..0193b4c 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -1,4 +1,4 @@ -include { checkItemAllowed } from "${meta.resources_dir}/helper.nf" +include { checkItemAllowed; paramsetsFromVariants; expandParamsets; methodMatchesParamset; checkMethodAllowed } from "${meta.resources_dir}/helper.nf" workflow auto { findStates(params, meta.config) @@ -19,6 +19,13 @@ metrics = [ accuracy ] +// serialise data to a yaml file in the temp dir +def writeYamlFile(data, String filename) { + def file = tempFile(filename) + file.write(toYamlBlob(data)) + file +} + workflow run_wf { take: input_ch @@ -49,34 +56,47 @@ workflow run_wf { ***************************/ score_ch = dataset_ch + // expand the channel so parameterised methods run once per paramset. + // the paramsets are read from the --paramsets file if provided, and + // default to the method components' info.variants. + | flatMap { id, state -> + def method_paramsets = state.paramsets + ? readYaml(state.paramsets) + : paramsetsFromVariants(methods) + expandParamsets(id, state, method_paramsets) + } + // run all methods | runEach( components: methods, - // use the 'filter' argument to only run a method on the normalisation the component is asking for + // use the 'filter' argument to only run a method on the normalisation the + // component is asking for, to match paramset-tagged states to their method, + // and to filter by --methods_include/--methods_exclude filter: { id, state, comp -> def norm = state.dataset_uns.normalization_id def pref = comp.config.info.preferred_normalization // if the preferred normalisation is none at all, // we can pass whichever dataset we want def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref - def method_check = checkItemAllowed( + def paramset_check = methodMatchesParamset(state, comp.config.name) + def method_check = checkMethodAllowed( comp.config.name, + state.paramset_name, state.methods_include, - state.methods_exclude, - "methods_include", - "methods_exclude" + state.methods_exclude ) - method_check && norm_check + norm_check && paramset_check && method_check }, - // define a new 'id' by appending the method name to the dataset id + // define a new 'id' by appending the method name and paramset name to the dataset id id: { id, state, comp -> - id + "." + comp.config.name + id + "." + comp.config.name + (state.paramset_name ? "." + state.paramset_name : "") }, - // use 'fromState' to fetch the arguments the component requires from the overall state + // use 'fromState' to fetch the arguments the component requires from the overall state, + // along with the paramset arguments (if any) fromState: { id, state, comp -> def new_args = [ input_train: state.input_train, @@ -85,7 +105,7 @@ workflow run_wf { if (comp.config.info.type == "control_method") { new_args.input_solution = state.input_solution } - new_args + new_args + (state.paramset ?: [:]) }, // use 'toState' to publish that component's outputs to the overall state @@ -129,62 +149,43 @@ workflow run_wf { } ) - // extract the scores + // extract the scores, tagged with the paramset used for the method + // (null for control methods and methods without paramsets) | extract_uns_metadata.run( key: "extract_scores", fromState: [input: "metric_output"], toState: { id, output, state -> + def uns = readYaml(output.output).uns + uns.paramset_name = state.paramset_name + uns.paramset = state.paramset state + [ - score_uns: readYaml(output.output).uns + score_uns: uns ] } ) + // store the scores in a file | joinStates { ids, states -> - // store the scores in a file - def score_uns = states.collect{it.score_uns} - def score_uns_yaml_blob = toYamlBlob(score_uns) - def score_uns_file = tempFile("score_uns.yaml") - score_uns_file.write(score_uns_yaml_blob) - - ["output", [output_scores: score_uns_file]] + ["output", [output_scores: writeYamlFile(states.collect{it.score_uns}, "score_uns.yaml")]] } /****************************** * GENERATE OUTPUT YAML FILES * ******************************/ - // TODO: can we store everything below in a separate helper function? - - // extract the dataset metadata meta_ch = dataset_ch // only keep one of the normalization methods | filter{ id, state -> state.dataset_uns.normalization_id == "log_cp10k" } | joinStates { ids, states -> - // store the dataset metadata in a file + // gather the dataset metadata, without the normalization id def dataset_uns = states.collect{state -> def uns = state.dataset_uns.clone() uns.remove("normalization_id") uns } - def dataset_uns_yaml_blob = toYamlBlob(dataset_uns) - def dataset_uns_file = tempFile("dataset_uns.yaml") - dataset_uns_file.write(dataset_uns_yaml_blob) - - // store the method configs in a file - def method_configs = methods.collect{it.config} - def method_configs_yaml_blob = toYamlBlob(method_configs) - def method_configs_file = tempFile("method_configs.yaml") - method_configs_file.write(method_configs_yaml_blob) - - // store the metric configs in a file - def metric_configs = metrics.collect{it.config} - def metric_configs_yaml_blob = toYamlBlob(metric_configs) - def metric_configs_file = tempFile("metric_configs.yaml") - metric_configs_file.write(metric_configs_yaml_blob) - - // store the task info in a file + + // gather the task info and annotate it with the commit and timestamp def task_info = readYaml(meta.resources_dir.resolve("_viash.yaml")) // commitId is null when nextflow runs from a local checkout instead of a revision if (workflow.commitId) { @@ -193,15 +194,13 @@ workflow run_wf { // the launch time -- workflow.complete is only known once the run is over task_info.timestamp = workflow.start.toInstant() .truncatedTo(java.time.temporal.ChronoUnit.SECONDS).toString() - def task_info_file = tempFile("task_info.yaml") - task_info_file.write(toYamlBlob(task_info)) - // create output state + // store the dataset metadata, component configs and task info in files def new_state = [ - output_dataset_info: dataset_uns_file, - output_method_configs: method_configs_file, - output_metric_configs: metric_configs_file, - output_task_info: task_info_file, + output_dataset_info: writeYamlFile(dataset_uns, "dataset_uns.yaml"), + output_method_configs: writeYamlFile(methods.collect{it.config}, "method_configs.yaml"), + output_metric_configs: writeYamlFile(metrics.collect{it.config}, "metric_configs.yaml"), + output_task_info: writeYamlFile(task_info, "task_info.yaml"), _meta: states[0]._meta ]