From a8dabd6ead54e9d1c01aecde47e2777a06b23d51 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 13 Aug 2026 17:56:30 +0100 Subject: [PATCH 1/5] Fix missing newline after YAML block-scalar indicator in Bloblang highlighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extractBloblangFromBlock() trims a mapping/mutation/etc. token's raw text before tokenizing it as Bloblang, which strips the newline + indentation that visually separates the YAML block-scalar indicator (| or >) from the first line of content. processMultilineMapping() then discards the token's original content entirely (token.innerHTML = '') and replaces it with only the tokenized (already-trimmed) Bloblang, so that leading newline never makes it back into the rendered output — collapsing "mapping: |" and the first line of code onto one visual line, e.g.: mapping: |let jokes = [ instead of: mapping: | let jokes = [ Fix: capture the leading newline+indentation from the raw token text before it gets trimmed, and prepend it (HTML-escaped) to the wrapper's innerHTML alongside the tokenized Bloblang content, so the original line break survives. Verified against the real production code path (prism-core.js + prism-bloblang.js + this file, unmodified) in a real headless Chrome instance via Puppeteer: reproduced the exact bug from a real page (docs.redpanda.com's Connect quickstart), confirmed the fix restores the newline in both textContent and the rendered screenshot, using the same mapping: | example currently live on that page. Fixes a bug reported against docs.redpanda.com/cloud-data-platform (the example YAML rendering as `mapping: |let jokes = [` on one line) - not a docs-content bug, since the source YAML has always had the correct line break; this highlighting extension was the one dropping it. Co-Authored-By: Claude Sonnet 5 --- src/js/17-bloblang-yaml.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/js/17-bloblang-yaml.js b/src/js/17-bloblang-yaml.js index 538b21d3..ccb6ab21 100644 --- a/src/js/17-bloblang-yaml.js +++ b/src/js/17-bloblang-yaml.js @@ -303,7 +303,18 @@ * Also handles continuation content after blank lines */ function processMultilineMapping(token) { - var bloblangCode = extractBloblangFromBlock(token.textContent) + var rawText = token.textContent + var bloblangCode = extractBloblangFromBlock(rawText) + + // extractBloblangFromBlock() trims rawText before tokenizing, which + // strips the newline + indentation that visually separates the YAML + // block-scalar indicator (| or >) from the first line of content. + // Capture it here so it can be restored below — otherwise the token's + // original whitespace is discarded when its innerHTML is replaced, and + // "mapping: |" collapses onto the same line as the first Bloblang + // statement (e.g. "mapping: |let jokes = ["). + var leadingWhitespaceMatch = rawText.match(/^(\r?\n[ \t]*)/) + var leadingWhitespace = leadingWhitespaceMatch ? leadingWhitespaceMatch[1] : '' // Check for continuation content after this token var continuationNodes = collectLiteralBlockContinuation(token) @@ -323,7 +334,7 @@ var wrapper = document.createElement('span') wrapper.className = 'bloblang-embedded' - wrapper.innerHTML = highlighted + wrapper.innerHTML = escapeHtml(leadingWhitespace) + highlighted token.innerHTML = '' token.appendChild(wrapper) From 6cea1dff9269bcdd937575db0db5627415b9157d Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 13 Aug 2026 20:25:56 +0100 Subject: [PATCH 2/5] Add regression test case for the mapping: | newline bug (DOC-2433) Uses the exact producer-pipeline example from the Connect quickstart that was reported broken, so a future regression in processMultilineMapping()'s leading-whitespace preservation shows up immediately when previewing this page - mapping: | must stay on its own line, not collapse onto the same line as the first Bloblang statement. Co-Authored-By: Claude Sonnet 5 --- preview-src/bloblang-syntax-test.adoc | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/preview-src/bloblang-syntax-test.adoc b/preview-src/bloblang-syntax-test.adoc index 57e53b02..846373e4 100644 --- a/preview-src/bloblang-syntax-test.adoc +++ b/preview-src/bloblang-syntax-test.adoc @@ -367,4 +367,38 @@ spec: env: - name: DATABASE_URL value: "postgres://localhost:5432/mydb" +---- + +== Regression: Block Scalar Indicator Must Stay on Its Own Line + +Reported against docs.redpanda.com as +https://redpandadata.atlassian.net/browse/DOC-2433[DOC-2433]: this exact +example (the Connect quickstart's producer pipeline) rendered as +`mapping: |let jokes = [` on one line instead of `|` followed by a line +break. Root cause was `extractBloblangFromBlock()` in +`src/js/17-bloblang-yaml.js` trimming away the newline + indentation +between the block-scalar indicator and the first line of Bloblang before +`processMultilineMapping()` discarded the token's original content +entirely. + +**Check visually**: `mapping: |` must end its own line, with `let jokes` +starting the next line, indented — not glued onto the same line as `|`. +If this collapses again, the leading-whitespace preservation in +`processMultilineMapping()` has regressed. + +[source,yaml] +---- +input: + generate: + interval: 5s + count: 0 + mapping: | + let jokes = [ + "Why don't scientists trust atoms? Because they make up everything!", + "I'm reading a book about anti-gravity. It's impossible to put down!" + ] + root = jokes.index(random_int(seed: timestamp_unix_nano(), max: jokes.length() - 1)) +output: + redpanda: + topic: dad-jokes ---- \ No newline at end of file From 7154eb837e889035cb926c5420751c2d37949a65 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Fri, 14 Aug 2026 07:06:48 +0100 Subject: [PATCH 3/5] Update src/js/17-bloblang-yaml.js Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/js/17-bloblang-yaml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/17-bloblang-yaml.js b/src/js/17-bloblang-yaml.js index ccb6ab21..c65cd328 100644 --- a/src/js/17-bloblang-yaml.js +++ b/src/js/17-bloblang-yaml.js @@ -313,7 +313,7 @@ // original whitespace is discarded when its innerHTML is replaced, and // "mapping: |" collapses onto the same line as the first Bloblang // statement (e.g. "mapping: |let jokes = ["). - var leadingWhitespaceMatch = rawText.match(/^(\r?\n[ \t]*)/) + var leadingWhitespaceMatch = rawText.match(/^((?:\r?\n[ \t]*)+)/) var leadingWhitespace = leadingWhitespaceMatch ? leadingWhitespaceMatch[1] : '' // Check for continuation content after this token From 16bf784c725ed359e39034bf2859f33e079ff183 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 18 Aug 2026 16:11:06 +0100 Subject: [PATCH 4/5] Scope quote/escape rewriting to quoted scalars; capture whitespace exactly Review feedback from Feediver1 and DOC-2441: - Whitespace preservation now captures exactly what trim() strips (trimStart complement) instead of approximating it with a regex, so a trailing space after the | indicator no longer defeats the DOC-2433 fix. The capture lives inside extractBloblangFromBlock next to the trim it compensates for, and the whitespace-only string is concatenated without the no-op escapeHtml call. - DOC-2441 issue 1: escape rewriting no longer runs on literal/folded block scalars (YAML passes those through verbatim). Quoted flow scalars keep it, now as a correct single pass per quote style: the old sequential chain turned a literal \\n into backslash+newline, and single-quoted scalars only process ''. - DOC-2441 issue 2: quote-stripping no longer fires on block scalars whose first and last characters are coincidentally the same quote; it applies only to tokens Prism classified as quoted strings, where the grammar guarantees the quotes enclose the token. - Block scalars are detected by the Prism "scalar" token class ("string" is only an alias on those tokens). Adds 5 regression tests (mini-playground-test.html fixtures 8-11 plus an assertion on fixture 6) - the three new-behavior ones fail against the previous code - and matching visual sections in preview-src/bloblang-syntax-test.adoc, which now ends with a newline. Co-Authored-By: Claude Fable 5 --- preview-src/bloblang-syntax-test.adoc | 47 +++++- src/js/17-bloblang-yaml.js | 79 ++++++---- .../mini-playground-test.html | 136 ++++++++++++++++++ 3 files changed, 234 insertions(+), 28 deletions(-) diff --git a/preview-src/bloblang-syntax-test.adoc b/preview-src/bloblang-syntax-test.adoc index 846373e4..f6860f75 100644 --- a/preview-src/bloblang-syntax-test.adoc +++ b/preview-src/bloblang-syntax-test.adoc @@ -401,4 +401,49 @@ input: output: redpanda: topic: dad-jokes ----- \ No newline at end of file +---- + +== Regression: Literal Block Scalars Are Not Rewritten (DOC-2441) + +https://redpandadata.atlassian.net/browse/DOC-2441[DOC-2441]: YAML performs +no quote or escape processing on literal block scalars, so the highlighter +must not either. + +**Check visually and with the copy button**: `join("\n")` and `join("\t")` +must render (and copy) as two characters each - a backslash followed by a +letter - never as a real line break or tab inside the string literal. + +[source,yaml] +---- +pipeline: + processors: + - mapping: | + root.lines = this.values.join("\n") + root.tabbed = this.cols.join("\t") +---- + +**Check visually**: the first line's opening quote on `"prefix-"` and the +last line's closing quote on `"-suffix"` must both render. They belong to +two different string literals, and must not be stripped as if they enclosed +the whole block. + +[source,yaml] +---- +pipeline: + processors: + - mapping: | + "prefix-" + this.id + "-suffix" +---- + +**Check visually**: a trailing space after the `|` indicator must not glue +the first Bloblang line onto the `mapping:` line (the space after `|` below +is intentional). + +[source,yaml] +---- +pipeline: + processors: + - mapping: | + let doubled = this.value * 2 + root.result = $doubled +---- diff --git a/src/js/17-bloblang-yaml.js b/src/js/17-bloblang-yaml.js index c65cd328..febfd09e 100644 --- a/src/js/17-bloblang-yaml.js +++ b/src/js/17-bloblang-yaml.js @@ -121,25 +121,50 @@ } /** - * Extract Bloblang code from a YAML literal block string - * Handles | and > block scalars + * Extract Bloblang code from a YAML scalar token's text. + * + * Returns { leading, code }: + * - leading: the whitespace trim() strips from the front of the token. + * For block scalars this is the newline + indentation that separates + * the | or > indicator from the first content line (the token can also + * begin with spaces or tabs, because the [ \t]* after the indicator is + * part of the scalar token in Prism's YAML grammar). It must be + * restored when the token is re-rendered, or "mapping: |" collapses + * onto the same line as the first Bloblang statement (DOC-2433). + * - code: the Bloblang source to tokenize. + * + * Block scalars (mapping: | ...) are literal source text: YAML performs + * no quote or escape processing on them, so neither does this function - + * rewriting \n inside them, or stripping a coincidental leading/trailing + * quote pair, corrupts the rendered and copied code (DOC-2441). + * + * Quoted flow scalars (mapping: "root = ...") are unwrapped and + * unescaped according to their quote style: double quotes process + * backslash escapes, single quotes only the '' escape. */ - function extractBloblangFromBlock(tokenText) { - // Remove leading/trailing quotes if present + function extractBloblangFromBlock(tokenText, isBlockScalar) { + var leading = tokenText.slice(0, tokenText.length - tokenText.trimStart().length) var text = tokenText.trim() - if ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'"))) { - text = text.slice(1, -1) - } - // Handle escape sequences - text = text - .replace(/\\n/g, '\n') - .replace(/\\t/g, '\t') - .replace(/\\"/g, '"') - .replace(/\\'/g, "'") - .replace(/\\\\/g, '\\') + if (!isBlockScalar) { + var quote = text.charAt(0) + if ((quote === '"' || quote === "'") && text.length >= 2 && text.endsWith(quote)) { + text = text.slice(1, -1) + if (quote === '"') { + // Single pass so an escaped backslash can't feed a later rule + // (the old sequential replace chain turned \\n into \) + text = text.replace(/\\(.)/g, function (match, ch) { + if (ch === 'n') return '\n' + if (ch === 't') return '\t' + return ch + }) + } else { + text = text.replace(/''/g, "'") + } + } + } - return text + return { leading: leading, code: text } } /** @@ -304,17 +329,13 @@ */ function processMultilineMapping(token) { var rawText = token.textContent - var bloblangCode = extractBloblangFromBlock(rawText) - - // extractBloblangFromBlock() trims rawText before tokenizing, which - // strips the newline + indentation that visually separates the YAML - // block-scalar indicator (| or >) from the first line of content. - // Capture it here so it can be restored below — otherwise the token's - // original whitespace is discarded when its innerHTML is replaced, and - // "mapping: |" collapses onto the same line as the first Bloblang - // statement (e.g. "mapping: |let jokes = ["). - var leadingWhitespaceMatch = rawText.match(/^((?:\r?\n[ \t]*)+)/) - var leadingWhitespace = leadingWhitespaceMatch ? leadingWhitespaceMatch[1] : '' + // Prism's YAML grammar gives block scalars the "scalar" class ("string" + // is only an alias on them); a token with "string" alone is a quoted + // flow scalar, whose quotes/escapes YAML actually processes. + var isBlockScalar = token.classList.contains('scalar') + var extracted = extractBloblangFromBlock(rawText, isBlockScalar) + var bloblangCode = extracted.code + var leadingWhitespace = extracted.leading // Check for continuation content after this token var continuationNodes = collectLiteralBlockContinuation(token) @@ -334,7 +355,11 @@ var wrapper = document.createElement('span') wrapper.className = 'bloblang-embedded' - wrapper.innerHTML = escapeHtml(leadingWhitespace) + highlighted + // leadingWhitespace is whitespace-only (what trim() stripped), so it is + // HTML-inert and safe to concatenate unescaped. Asymmetry note: trailing + // whitespace on the scalar's last content line is still dropped by the + // trim - invisible in the render, observable only via the copy button. + wrapper.innerHTML = leadingWhitespace + highlighted token.innerHTML = '' token.appendChild(wrapper) diff --git a/tests/bloblang-interactive/mini-playground-test.html b/tests/bloblang-interactive/mini-playground-test.html index 29dd5890..48b6ebf9 100644 --- a/tests/bloblang-interactive/mini-playground-test.html +++ b/tests/bloblang-interactive/mini-playground-test.html @@ -170,6 +170,47 @@

Test Summary

- name: app image: myapp:latest + + +
+
+ 
pipeline:
+  processors:
+    - mapping: | 
+        let doubled = this.value * 2
+        root.result = $doubled
+
+ + +
+
+ 
pipeline:
+  processors:
+    - mapping: |
+        root.lines = this.values.join("\n")
+        root.tabbed = this.cols.join("\t")
+
+ + +
+
+ 
pipeline:
+  processors:
+    - mapping: |
+        "prefix-" + this.id + "-suffix"
+
+ + +
+
+ 
pipeline:
+  processors:
+    - mapping: "root = this.name.uppercase()"
+