SOLR-8003: A glob followed by a raw value transformer now writes each matching field raw

changelog/unreleased/SOLR-8003-fl-glob-raw-transformer.yml

@@ -0,0 +1,9 @@
+title: >
+  The fl parameter accepts a glob in front of a raw value transformer, for example fl=*_json:[json], which
+  writes every returned field matching the glob raw instead of failing with "Error parsing fieldname".
+type: added
+authors:
+  - name: Nick Shanin
+links:
+  - name: SOLR-8003
+    url: https://issues.apache.org/jira/browse/SOLR-8003

solr/core/src/java/org/apache/solr/response/TextResponseWriter.java

@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexableField;
@@ -31,6 +32,7 @@
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.GlobPatternUtil;
 import org.apache.solr.common.util.TextWriter;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.transform.DocTransformer;
@@ -65,6 +67,7 @@ public abstract class TextResponseWriter implements TextWriter {
 
   private final TextResponseWriter rawShim;
   private final Set<String> rawFields;
+  private final List<String> rawFieldGlobs;
   private final ReturnFields rawReturnFields;
 
   public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
@@ -79,14 +82,19 @@ public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse
     returnFields = rsp.getReturnFields();
     if (req.getParams().getBool(CommonParams.OMIT_HEADER, false)) rsp.removeResponseHeader();
     DocTransformer rootDocTransformer = returnFields.getTransformer();
-    Collection<String> rawFields;
-    if (rootDocTransformer == null || (rawFields = rootDocTransformer.getRawFields()).isEmpty()) {
+    Collection<String> rawFields =
+        rootDocTransformer == null ? Set.of() : rootDocTransformer.getRawFields();
+    Collection<String> rawGlobs =
+        rootDocTransformer == null ? Set.of() : rootDocTransformer.getRawFieldGlobs();
+    if (rawFields.isEmpty() && rawGlobs.isEmpty()) {
       this.rawFields = null;
+      this.rawFieldGlobs = List.of();
       this.rawShim = null;
       this.rawReturnFields = NO_RAW_FIELDS;
     } else {
       this.rawFields =
           rawFields.size() == 1 ? Set.of(rawFields.iterator().next()) : new HashSet<>(rawFields);
+      this.rawFieldGlobs = List.copyOf(rawGlobs);
       this.rawShim = new RawShimTextResponseWriter(this);
       this.rawReturnFields = returnFields;
     }
@@ -102,6 +110,7 @@ public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse
     this.doIndent = indent;
     this.rawShim = null;
     this.rawFields = null;
+    this.rawFieldGlobs = List.of();
     this.rawReturnFields = null;
   }
 
@@ -111,7 +120,18 @@ public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse
    */
   @SuppressWarnings("ReferenceEquality") // strict object identity is intentional, see javadoc above
   protected final boolean shouldWriteRaw(String fname, ReturnFields returnFields) {
-    return rawReturnFields == returnFields && rawFields.contains(fname);
+    if (rawReturnFields != returnFields) {
+      return false;
+    }
+    if (rawFields.contains(fname)) {
+      return true;
+    }
+    for (String glob : rawFieldGlobs) {
+      if (GlobPatternUtil.matches(glob, fname)) {
+        return true;
+      }
+    }
+    return false;
   }
 
   /** done with this ResponseWriter... make sure any buffers are flushed to writer */

solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java

@@ -71,6 +71,14 @@ public Collection<String> getRawFields() {
     return Set.of();
   }
 
+  /**
+   * Like {@link #getRawFields()}, but each entry is a glob pattern (as accepted in {@code fl}); any
+   * field whose name matches is written raw.
+   */
+  public Collection<String> getRawFieldGlobs() {
+    return Set.of();
+  }
+
   /**
    * Indicates if this transformer requires access to the underlying index to perform its functions.
    *

solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java

@@ -53,6 +53,14 @@ public Collection<String> getRawFields() {
         .collect(Collectors.toList());
   }
 
+  @Override
+  public Collection<String> getRawFieldGlobs() {
+    return children.stream()
+        .map(DocTransformer::getRawFieldGlobs)
+        .flatMap(Collection::stream)
+        .collect(Collectors.toList());
+  }
+
   public void addTransformer(DocTransformer a) {
     children.add(a);
   }

solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java

@@ -62,8 +62,6 @@ public boolean mayModifyValue() {
   }
 
   @Override
-  @SuppressWarnings(
-      "ReferenceEquality") // QueryResponseWriter identity, not equality, is what matters here
   public DocTransformer create(
       String display,
       SolrParams params,
@@ -80,22 +78,7 @@ public DocTransformer create(
     if (!copy) {
       renamedFields.put(field, display);
     }
-    // When a 'wt' is specified in the transformer, only apply it to the same wt
-    boolean apply = true;
-    if (applyToWT != null) {
-      String qwt = req.getParams().get(CommonParams.WT);
-      if (qwt == null) {
-        QueryResponseWriter qw = req.getCore().getQueryResponseWriter(req);
-        QueryResponseWriter dw = req.getCore().getQueryResponseWriter(applyToWT);
-        if (qw != dw) {
-          apply = false;
-        }
-      } else {
-        apply = applyToWT.equals(qwt);
-      }
-    }
-
-    if (apply) {
+    if (appliesTo(req)) {
       return new RawTransformer(field, display, copy);
     }
 
@@ -106,6 +89,54 @@ public DocTransformer create(
     return new RenameFieldTransformer(field, display, copy);
   }
 
+  /** When a 'wt' is specified in the transformer, it only applies to the same wt */
+  @SuppressWarnings("ReferenceEquality") // QueryResponseWriter identity is what matters here
+  private boolean appliesTo(SolrQueryRequest req) {
+    if (applyToWT == null) {
+      return true;
+    }
+    String qwt = req.getParams().get(CommonParams.WT);
+    if (qwt == null) {
+      QueryResponseWriter qw = req.getCore().getQueryResponseWriter(req);
+      QueryResponseWriter dw = req.getCore().getQueryResponseWriter(applyToWT);
+      return qw == dw;
+    }
+    return applyToWT.equals(qwt);
+  }
+
+  /**
+   * Creates the transformer for {@code glob:[name]}: every returned field whose name matches the
+   * glob is written raw, under its own name.
+   *
+   * @return null if this transformer does not apply to the request's response writer
+   */
+  public DocTransformer createForGlob(String glob, SolrQueryRequest req) {
+    return appliesTo(req) ? new GlobRawTransformer(glob) : null;
+  }
+
+  static class GlobRawTransformer extends DocTransformer {
+    final String glob;
+
+    GlobRawTransformer(String glob) {
+      this.glob = glob;
+    }
+
+    @Override
+    public String getName() {
+      return glob;
+    }
+
+    @Override
+    public Collection<String> getRawFieldGlobs() {
+      return Set.of(glob);
+    }
+
+    @Override
+    public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) {
+      // values are untouched; the response writer writes the matching fields raw
+    }
+  }
+
   static class RawTransformer extends DocTransformer {
     final String field;
     final String display;

solr/core/src/java/org/apache/solr/search/SolrReturnFields.java

@@ -38,6 +38,7 @@
 import org.apache.solr.response.transform.DocTransformer;
 import org.apache.solr.response.transform.DocTransformers;
 import org.apache.solr.response.transform.OriginalScoreAugmenter;
+import org.apache.solr.response.transform.RawValueTransformerFactory;
 import org.apache.solr.response.transform.RenameFieldTransformer;
 import org.apache.solr.response.transform.ScoreAugmenter;
 import org.apache.solr.response.transform.TransformerFactory;
@@ -314,6 +315,36 @@ private void add(
               globs.add(field);
             }
             continue;
+          } else if (field != null && ch == ':' && sp.val.startsWith("[", sp.pos + 1)) {
+            // "glob:[json]" applies a raw value transformer to every matching field
+            if ("*".equals(field)) {
+              throw new SolrException(
+                  SolrException.ErrorCode.BAD_REQUEST,
+                  "A raw value transformer cannot follow a bare *. "
+                      + "Use a named glob such as *_json:[json] instead.");
+            }
+            sp.pos++;
+            ModifiableSolrParams augmenterParams = new ModifiableSolrParams();
+            sp.pos +=
+                QueryParsing.parseLocalParams(
+                    sp.val.substring(sp.pos), 0, augmenterParams, req.getParams(), "[", ']');
+            String augmenterName = augmenterParams.get("type");
+            TransformerFactory factory = req.getCore().getTransformerFactory(augmenterName);
+            if (!(factory instanceof RawValueTransformerFactory rawFactory)) {
+              throw new SolrException(
+                  SolrException.ErrorCode.BAD_REQUEST,
+                  "A glob can only be combined with a raw value transformer such as [json] or [xml]: "
+                      + field
+                      + ":["
+                      + augmenterName
+                      + "]");
+            }
+            globs.add(field);
+            DocTransformer t = rawFactory.createForGlob(field, req);
+            if (t != null) {
+              augmenters.addTransformer(t);
+            }
+            continue;
           } else if (ORIGINAL_SCORE_NAME.equals(field) && sp.opt("(") && sp.opt(")")) {
             // TODO: Remove this in https://issues.apache.org/jira/browse/SOLR-17784 when
             // originalScore() becomes a true function