Add support for paramsets

CHANGELOG.md

@@ -16,6 +16,10 @@
 
 * Added `control_methods/random_labels` component (PR #22).
 
+* `run_benchmark`: run parameterised methods once per named paramset from `info.variants` or the new `--paramsets` file, tag scores with `paramset_name`/`paramset`, and allow `--methods_include`/`--methods_exclude` to target `<method_id>.<paramset_name>`, using the new paramset helpers in `common/nextflow_helpers/helper.nf` (PR #23).
+
+* `methods/logistic_regression`: added an `--inverse_regularization_strength` argument with three `info.variants` as an example of a parameterised method (PR #23).
+
 ## MAJOR CHANGES
 
 * Updated `api` files (PR #5).

src/methods/logistic_regression/config.vsh.yaml

@@ -43,13 +43,29 @@ links:
 info:
   # Which normalisation method this component prefers to use (required).
   preferred_normalization: log_cp10k
+  # Named parameter sets to evaluate in the benchmark (optional). The
+  # `run_benchmark` workflow runs the method once per variant, passing the
+  # listed values as arguments, and tags the scores with the variant name.
+  # Only the listed variants are run, so to also evaluate the default
+  # arguments, list them as a variant with their values spelled out (viash
+  # drops empty variants from the config). Without `variants`, the method is
+  # run once with its default arguments.
+  variants:
+    strong_regularization:
+      inverse_regularization_strength: 0.1
+    default_regularization:
+      inverse_regularization_strength: 1.0
+    weak_regularization:
+      inverse_regularization_strength: 10.0
 
 # Component-specific parameters (optional)
-# arguments:
-#   - name: "--n_neighbors"
-#     type: "integer"
-#     default: 5
-#     description: Number of neighbors to use.
+arguments:
+  - name: "--inverse_regularization_strength"
+    type: double
+    default: 1.0
+    description: |
+      Inverse of the regularization strength (`C` in scikit-learn). Smaller
+      values specify stronger regularization.
 
 # Resources required to run the component
 resources:

src/methods/logistic_regression/script.py

@@ -7,7 +7,8 @@
 par = {
   'input_train': 'resources_test/task_template/cxg_mouse_pancreas_atlas/train.h5ad',
   'input_test': 'resources_test/task_template/cxg_mouse_pancreas_atlas/test.h5ad',
-  'output': 'output.h5ad'
+  'output': 'output.h5ad',
+  'inverse_regularization_strength': 1.0
 }
 meta = {
   'name': 'logistic_regression'
@@ -23,7 +24,9 @@
 
 print('Train model', flush=True)
 # ... train model ...
-classifier = sklearn.linear_model.LogisticRegression()
+classifier = sklearn.linear_model.LogisticRegression(
+  C=par['inverse_regularization_strength']
+)
 classifier.fit(input_train.obsm["X_pca"], input_train.obs["label"].astype(str))
 
 print('Generate predictions', flush=True)

src/workflows/run_benchmark/config.vsh.yaml

@@ -55,16 +55,36 @@ argument_groups:
       These arguments are mutually exclusive, so only `--methods_include` OR
       `--methods_exclude` can set but not both.
     arguments:
+      - name: "--paramsets"
+        type: file
+        direction: input
+        required: false
+        description: |
+          A yaml file mapping method ids to named parameter sets, e.g.:
+
+            logistic_regression:
+              strong_regularization:
+                inverse_regularization_strength: 0.1
+              weak_regularization:
+                inverse_regularization_strength: 10.0
+
+          Each listed method is run once per parameter set, with the given arguments
+          and with `paramset_name`/`paramset` attached to its scores.
+          If not provided, the parameter sets default to the `info.variants`
+          defined in the method components' configs. A method without
+          `info.variants` is run once with its default parameter set.
       - name: "--methods_include"
         type: string
         multiple: true
         description: |
           A list of method ids to include. If specified, only these methods will be run.
+          For parameterised methods, a single paramset can be targeted as `<method_id>.<paramset_name>`, e.g. `logistic_regression.strong_regularization`.
       - name: "--methods_exclude"
         type: string
         multiple: true
         description: |
           A list of method ids to exclude. If specified, all methods except the ones listed will be run.
+          For parameterised methods, a single paramset can be excluded as `<method_id>.<paramset_name>`, e.g. `logistic_regression.strong_regularization`.
   - name: Metric filtering
     description: |
       Use these arguments to filter metrics by name. By default, all metrics are

src/workflows/run_benchmark/main.nf

@@ -1,4 +1,4 @@
-include { checkItemAllowed } from "${meta.resources_dir}/helper.nf"
+include { checkItemAllowed; paramsetsFromVariants; expandParamsets; methodMatchesParamset; checkMethodAllowed } from "${meta.resources_dir}/helper.nf"
 
 workflow auto {
   findStates(params, meta.config)
@@ -19,6 +19,13 @@ metrics = [
   accuracy
 ]
 
+// serialise data to a yaml file in the temp dir
+def writeYamlFile(data, String filename) {
+  def file = tempFile(filename)
+  file.write(toYamlBlob(data))
+  file
+}
+
 workflow run_wf {
   take:
   input_ch
@@ -49,34 +56,47 @@ workflow run_wf {
    ***************************/
   score_ch = dataset_ch
 
+    // expand the channel so parameterised methods run once per paramset.
+    // the paramsets are read from the --paramsets file if provided, and
+    // default to the method components' info.variants.
+    | flatMap { id, state ->
+      def method_paramsets = state.paramsets
+        ? readYaml(state.paramsets)
+        : paramsetsFromVariants(methods)
+      expandParamsets(id, state, method_paramsets)
+    }
+
     // run all methods
     | runEach(
       components: methods,
 
-      // use the 'filter' argument to only run a method on the normalisation the component is asking for
+      // use the 'filter' argument to only run a method on the normalisation the
+      // component is asking for, to match paramset-tagged states to their method,
+      // and to filter by --methods_include/--methods_exclude
       filter: { id, state, comp ->
         def norm = state.dataset_uns.normalization_id
         def pref = comp.config.info.preferred_normalization
         // if the preferred normalisation is none at all,
         // we can pass whichever dataset we want
         def norm_check = (norm == "log_cp10k" && pref == "counts") || norm == pref
-        def method_check = checkItemAllowed(
+        def paramset_check = methodMatchesParamset(state, comp.config.name)
+        def method_check = checkMethodAllowed(
           comp.config.name,
+          state.paramset_name,
           state.methods_include,
-          state.methods_exclude,
-          "methods_include",
-          "methods_exclude"
+          state.methods_exclude
         )
 
-        method_check && norm_check
+        norm_check && paramset_check && method_check
       },
 
-      // define a new 'id' by appending the method name to the dataset id
+      // define a new 'id' by appending the method name and paramset name to the dataset id
       id: { id, state, comp ->
-        id + "." + comp.config.name
+        id + "." + comp.config.name + (state.paramset_name ? "." + state.paramset_name : "")
       },
 
-      // use 'fromState' to fetch the arguments the component requires from the overall state
+      // use 'fromState' to fetch the arguments the component requires from the overall state,
+      // along with the paramset arguments (if any)
       fromState: { id, state, comp ->
         def new_args = [
           input_train: state.input_train,
@@ -85,7 +105,7 @@ workflow run_wf {
         if (comp.config.info.type == "control_method") {
           new_args.input_solution = state.input_solution
         }
-        new_args
+        new_args + (state.paramset ?: [:])
       },
 
       // use 'toState' to publish that component's outputs to the overall state
@@ -129,62 +149,43 @@ workflow run_wf {
       }
     )
 
-    // extract the scores
+    // extract the scores, tagged with the paramset used for the method
+    // (null for control methods and methods without paramsets)
     | extract_uns_metadata.run(
       key: "extract_scores",
       fromState: [input: "metric_output"],
       toState: { id, output, state ->
+        def uns = readYaml(output.output).uns
+        uns.paramset_name = state.paramset_name
+        uns.paramset = state.paramset
         state + [
-          score_uns: readYaml(output.output).uns
+          score_uns: uns
         ]
       }
     )
 
+    // store the scores in a file
     | joinStates { ids, states ->
-      // store the scores in a file
-      def score_uns = states.collect{it.score_uns}
-      def score_uns_yaml_blob = toYamlBlob(score_uns)
-      def score_uns_file = tempFile("score_uns.yaml")
-      score_uns_file.write(score_uns_yaml_blob)
-
-      ["output", [output_scores: score_uns_file]]
+      ["output", [output_scores: writeYamlFile(states.collect{it.score_uns}, "score_uns.yaml")]]
     }
 
   /******************************
    * GENERATE OUTPUT YAML FILES *
    ******************************/
-  // TODO: can we store everything below in a separate helper function?
-
-  // extract the dataset metadata
   meta_ch = dataset_ch
     // only keep one of the normalization methods
     | filter{ id, state ->
       state.dataset_uns.normalization_id == "log_cp10k"
     }
     | joinStates { ids, states ->
-      // store the dataset metadata in a file
+      // gather the dataset metadata, without the normalization id
       def dataset_uns = states.collect{state ->
         def uns = state.dataset_uns.clone()
         uns.remove("normalization_id")
         uns
       }
-      def dataset_uns_yaml_blob = toYamlBlob(dataset_uns)
-      def dataset_uns_file = tempFile("dataset_uns.yaml")
-      dataset_uns_file.write(dataset_uns_yaml_blob)
-
-      // store the method configs in a file
-      def method_configs = methods.collect{it.config}
-      def method_configs_yaml_blob = toYamlBlob(method_configs)
-      def method_configs_file = tempFile("method_configs.yaml")
-      method_configs_file.write(method_configs_yaml_blob)
-
-      // store the metric configs in a file
-      def metric_configs = metrics.collect{it.config}
-      def metric_configs_yaml_blob = toYamlBlob(metric_configs)
-      def metric_configs_file = tempFile("metric_configs.yaml")
-      metric_configs_file.write(metric_configs_yaml_blob)
-
-      // store the task info in a file
+
+      // gather the task info and annotate it with the commit and timestamp
       def task_info = readYaml(meta.resources_dir.resolve("_viash.yaml"))
       // commitId is null when nextflow runs from a local checkout instead of a revision
       if (workflow.commitId) {
@@ -193,15 +194,13 @@ workflow run_wf {
       // the launch time -- workflow.complete is only known once the run is over
       task_info.timestamp = workflow.start.toInstant()
         .truncatedTo(java.time.temporal.ChronoUnit.SECONDS).toString()
-      def task_info_file = tempFile("task_info.yaml")
-      task_info_file.write(toYamlBlob(task_info))
 
-      // create output state
+      // store the dataset metadata, component configs and task info in files
       def new_state = [
-        output_dataset_info: dataset_uns_file,
-        output_method_configs: method_configs_file,
-        output_metric_configs: metric_configs_file,
-        output_task_info: task_info_file,
+        output_dataset_info: writeYamlFile(dataset_uns, "dataset_uns.yaml"),
+        output_method_configs: writeYamlFile(methods.collect{it.config}, "method_configs.yaml"),
+        output_metric_configs: writeYamlFile(metrics.collect{it.config}, "metric_configs.yaml"),
+        output_task_info: writeYamlFile(task_info, "task_info.yaml"),
         _meta: states[0]._meta
       ]