diff --git a/.changelog/5929-fix-liveregion-card-text-format.md b/.changelog/5929-fix-liveregion-card-text-format.md new file mode 100644 index 0000000000..76444827c6 --- /dev/null +++ b/.changelog/5929-fix-liveregion-card-text-format.md @@ -0,0 +1,3 @@ +### Fixed + +- Fixed screen readers reading Adaptive Card text with its Markdown and `{{DATE()}}` syntax, in PR [#5929](https://github.com/microsoft/BotFramework-WebChat/pull/5929), by [@OEvgeny](https://github.com/OEvgeny) diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html new file mode 100644 index 0000000000..39af46ff3c --- /dev/null +++ b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html @@ -0,0 +1,91 @@ + + + + + + + + +
+ + + + diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html.snap-1.png b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html.snap-1.png new file mode 100644 index 0000000000..d02e7faa07 Binary files /dev/null and b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html.snap-1.png differ diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html new file mode 100644 index 0000000000..2dad3ea518 --- /dev/null +++ b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html @@ -0,0 +1,92 @@ + + + + + + + + +
+ + + + diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html.snap-1.png b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html.snap-1.png new file mode 100644 index 0000000000..f2a916ac38 Binary files /dev/null and b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html.snap-1.png differ diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html new file mode 100644 index 0000000000..c469d239d7 --- /dev/null +++ b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html @@ -0,0 +1,96 @@ + + + + + + + + +
+ + + + diff --git a/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html.snap-1.png b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html.snap-1.png new file mode 100644 index 0000000000..f330909696 Binary files /dev/null and b/__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html.snap-1.png differ diff --git a/packages/bundle/src/adaptiveCards/Attachment/private/renderAdaptiveCard.ts b/packages/bundle/src/adaptiveCards/Attachment/private/renderAdaptiveCard.ts index a41cc32e71..8ea39fadc4 100644 --- a/packages/bundle/src/adaptiveCards/Attachment/private/renderAdaptiveCard.ts +++ b/packages/bundle/src/adaptiveCards/Attachment/private/renderAdaptiveCard.ts @@ -1,9 +1,6 @@ -import type { - AdaptiveCard, - GlobalSettings as GlobalSettingsType, - HostConfig as HostConfigType, - IMarkdownProcessingResult -} from 'adaptivecards'; +import type { AdaptiveCard, GlobalSettings as GlobalSettingsType, HostConfig as HostConfigType } from 'adaptivecards'; + +import setMarkdownEngine from './setMarkdownEngine'; /** * Returns `true`, if the object is a plain object and not a class, otherwise, `false`. @@ -29,19 +26,7 @@ export default function renderAdaptiveCard( setTabIndexAtCardRoot: boolean; } ): { element: HTMLElement } | { errors: Error[] } { - // Currently, the only way to set the Markdown engine is to set it thru static member of AdaptiveCard class - - // TODO: [P3] Checks if we could make the "renderMarkdownAsHTML" per card - // This could be limitations from Adaptive Cards package (not supported as of 1.2.5) - // Because there could be timing difference between .parse and .render, we could be using wrong Markdown engine - - // "onProcessMarkdown" is a static function but we are trying to scope it to the current object instead. - adaptiveCard.constructor['onProcessMarkdown'] = (text: string, result: IMarkdownProcessingResult) => { - if (renderMarkdownAsHTML) { - result.outputHtml = renderMarkdownAsHTML(text); - result.didProcess = true; - } - }; + setMarkdownEngine(adaptiveCard, renderMarkdownAsHTML); if (adaptiveCardsHostConfig) { adaptiveCard.hostConfig = isPlainObject(adaptiveCardsHostConfig) diff --git a/packages/bundle/src/adaptiveCards/Attachment/private/setMarkdownEngine.ts b/packages/bundle/src/adaptiveCards/Attachment/private/setMarkdownEngine.ts new file mode 100644 index 0000000000..2f4d2e325b --- /dev/null +++ b/packages/bundle/src/adaptiveCards/Attachment/private/setMarkdownEngine.ts @@ -0,0 +1,20 @@ +import type { AdaptiveCard, IMarkdownProcessingResult } from 'adaptivecards'; + +export default function setMarkdownEngine( + adaptiveCard: AdaptiveCard, + renderMarkdownAsHTML: ((markdown: string) => string) | undefined +): void { + // Currently, the only way to set the Markdown engine is to set it thru static member of AdaptiveCard class + + // TODO: [P3] Checks if we could make the "renderMarkdownAsHTML" per card + // This could be limitations from Adaptive Cards package (not supported as of 1.2.5) + // Because there could be timing difference between .parse and .render, we could be using wrong Markdown engine + + // "onProcessMarkdown" is a static function but we are trying to scope it to the current object instead. + adaptiveCard.constructor['onProcessMarkdown'] = (text: string, result: IMarkdownProcessingResult) => { + if (renderMarkdownAsHTML) { + result.outputHtml = renderMarkdownAsHTML(text); + result.didProcess = true; + } + }; +} diff --git a/packages/bundle/src/adaptiveCards/AttachmentForScreenReader/AdaptiveCardAttachment.js b/packages/bundle/src/adaptiveCards/AttachmentForScreenReader/AdaptiveCardAttachment.js index d52b17eabf..f58abbaa7e 100644 --- a/packages/bundle/src/adaptiveCards/AttachmentForScreenReader/AdaptiveCardAttachment.js +++ b/packages/bundle/src/adaptiveCards/AttachmentForScreenReader/AdaptiveCardAttachment.js @@ -1,14 +1,44 @@ /* eslint-disable react/no-array-index-key */ /* eslint-disable react/forbid-dom-props */ import { hooks } from 'botframework-webchat-component'; +import { htmlTextAlternatives } from 'botframework-webchat-component/internal.js'; +import { onErrorResumeNext } from 'botframework-webchat-core'; import PropTypes from 'prop-types'; import React, { useMemo } from 'react'; +import setMarkdownEngine from '../Attachment/private/setMarkdownEngine'; import useAdaptiveCardsPackage from '../hooks/useAdaptiveCardsPackage'; import useParseAdaptiveCardJSON from '../hooks/internal/useParseAdaptiveCardJSON'; import useUniqueId from '../hooks/internal/useUniqueId'; -const { useLocalizer } = hooks; +const { useLocalizer, useRenderMarkdownAsHTML } = hooks; + +// Returns the text as the visible card shows it: Adaptive Cards formats {{DATE()}} and {{TIME()}}, then renders the Markdown. +// Images are read by their alt text and blocks are separated, as in the live region of message text. +// If the Markdown engine or Adaptive Cards throws, the visible card shows an error, and the text is read as written. +function getRenderedText(element) { + return ( + onErrorResumeNext(() => { + const renderedElement = element.render(); + + // Plain text, such as a TextRun, is set with innerText, which renders line breaks as
; htmlTextAlternatives() reads
as nothing. + renderedElement?.querySelectorAll('br').forEach(br => br.replaceWith('\n')); + + return renderedElement ? htmlTextAlternatives(renderedElement).join('') : ''; + }, element.asString()) || '' + ) + .replace(/\s+/gu, ' ') + .trim(); +} + +// Facts are rendered as text blocks, as Adaptive Cards does. +function getFactText(TextBlock, factSet, text) { + const textBlock = new TextBlock(text); + + textBlock.setParent(factSet); + + return getRenderedText(textBlock); +} // Perform a depth-first search of the visible elements of the Adaptive Card tree, skipping hidden subtrees. // Actions are visited but not walked into, so the card of Action.ShowCard, hidden until opened, is not visited. @@ -69,6 +99,7 @@ AdaptiveCardChoiceSetInput.propTypes = { const AdaptiveCardAttachment = ({ content }) => { const localize = useLocalizer(); const parseAdaptiveCardJSON = useParseAdaptiveCardJSON(); + const renderMarkdownAsHTML = useRenderMarkdownAsHTML('adaptive cards'); const [ { ChoiceSetInput, @@ -127,19 +158,24 @@ const AdaptiveCardAttachment = ({ content }) => { walkAllItems(card, node => nodes.push(node)); + // Same Markdown engine as the visible card. This sets a static shared by all cards while rendering, which is safe because every render sets it first. + setMarkdownEngine(card, renderMarkdownAsHTML); + return nodes .flatMap(node => node instanceof FactSet - ? node.facts.map(({ name, value }) => [name, value].filter(Boolean).join(': ')) + ? node.facts.map(({ name, value }) => + [getFactText(TextBlock, node, name), getFactText(TextBlock, node, value)].filter(Boolean).join(': ') + ) : node instanceof Image ? [node.altText] : node instanceof RichTextBlock || node instanceof TextBlock - ? [node.asString()] + ? [getRenderedText(node)] : [] ) .filter(Boolean) .join(' '); - }, [card, FactSet, Image, RichTextBlock, TextBlock]); + }, [card, FactSet, Image, renderMarkdownAsHTML, RichTextBlock, TextBlock]); const cardLabel = localize('ATTACHMENT_CARD', card.speak || text, '', ''); diff --git a/packages/component/src/Utils/activityAltText.ts b/packages/component/src/Utils/activityAltText.ts index ddf5e4e175..bb7131f8e2 100644 --- a/packages/component/src/Utils/activityAltText.ts +++ b/packages/component/src/Utils/activityAltText.ts @@ -1,7 +1,7 @@ import textFormatToContentType from './textFormatToContentType'; -function walk(document: Document, walker: (node: T) => string[]): string[] { - const nodes: T[] = [].slice.call(document.childNodes); +function walk(root: Node, walker: (node: T) => string[]): string[] { + const nodes: T[] = [].slice.call(root.childNodes); const results: string[] = []; while (nodes.length) { @@ -75,15 +75,15 @@ const HTML_INLINE_TAGS = [ ]; /** - * Computes all text from a given HTML document as flattened array. This is best-effort. + * Computes all text from the descendants of a given HTML node, such as a document, as flattened array. This is best-effort. * - * @param {Document} document - HTML document to computes texts from. + * @param {Node} root - HTML node to compute texts from. */ -function htmlTextAlternatives(document: Document): string[] { +function htmlTextAlternatives(root: Node): string[] { // TODO: [P2] #3923 Revisit this logic with W3C standard, we could do a better text alternatives computation. // For example, is not computed. // https://www.w3.org/TR/accname-1.1/#mapping_additional_nd_name - return walk(document, node => { + return walk(root, node => { const { nodeType, tagName, textContent } = node; if (nodeType === Node.TEXT_NODE) { @@ -133,3 +133,5 @@ export default function activityAltText( return text; } + +export { htmlTextAlternatives }; diff --git a/packages/component/src/boot/internal.ts b/packages/component/src/boot/internal.ts index cada18e9f3..674de1fa16 100644 --- a/packages/component/src/boot/internal.ts +++ b/packages/component/src/boot/internal.ts @@ -15,6 +15,7 @@ export { export { default as PartGrouping } from '../Middleware/ActivityGrouping/ui/PartGrouping/PartGrouping'; export { useLiveRegion } from '../providers/LiveRegionTwin/index'; export { default as ScreenReaderText } from '../ScreenReaderText'; +export { htmlTextAlternatives } from '../Utils/activityAltText'; export { default as createIconComponent } from '../Utils/createIconComponent'; export { default as FixedWidthImage } from '../Utils/FixedWidthImage';