Read Adaptive Card text in the live region when the card has no speak

CHANGELOG.md

@@ -60,6 +60,7 @@ Legends:
 - Fixed GIFs in animation cards looping with no way to stop them: they now have a pause button, and do not play when the user prefers reduced motion or the card has `autostart: false`; the icons of the Copy and View code buttons are now visible in forced colors, fixes [#5734](https://github.com/microsoft/BotFramework-WebChat/issues/5734), in PR [#5897](https://github.com/microsoft/BotFramework-WebChat/pull/5897), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed the play, mute, and "more options" icons of audio players being invisible in dark high contrast themes, such as "Aquatic" on Windows, fixes [#5710](https://github.com/microsoft/BotFramework-WebChat/issues/5710), in PR [#5906](https://github.com/microsoft/BotFramework-WebChat/pull/5906), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed the "Positive" card action button having less than 4.5:1 color contrast, fixes [#5713](https://github.com/microsoft/BotFramework-WebChat/issues/5713), in PR [#5905](https://github.com/microsoft/BotFramework-WebChat/pull/5905), by [@OEvgeny](https://github.com/OEvgeny)
+- Fixed screen readers announcing Adaptive Cards without `speak` as just "A card", they now read the card text, in PR [#5922](https://github.com/microsoft/BotFramework-WebChat/pull/5922), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed screen readers announcing a single audio, video, or animation in a card as a list item, such as "1 of 1", fixes [#5722](https://github.com/microsoft/BotFramework-WebChat/issues/5722), in PR [#5903](https://github.com/microsoft/BotFramework-WebChat/pull/5903), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed video and animation cards disappearing in production builds when the video URL is relative or invalid, in PR [#5889](https://github.com/microsoft/BotFramework-WebChat/pull/5889), by [@OEvgeny](https://github.com/OEvgeny)
 

__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.hiddenText.html

@@ -0,0 +1,102 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
+    <link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="importmap">
+      {
+        "imports": {
+          "botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
+          "botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
+          "react": "/__dist__/packages/bundle/static/react.js",
+          "react-dom": "/__dist__/packages/bundle/static/react-dom.js",
+          "react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
+        }
+      }
+    </script>
+    <script type="module">
+      import '/test-harness.mjs';
+      import '/test-page-object.mjs';
+
+      import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
+      import { createElement } from 'react';
+      import { createRoot } from 'react-dom/client';
+
+      // TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
+      window.WebChat = { createStoreWithOptions };
+
+      run(async function () {
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
+        createRoot(document.getElementById('webchat')).render(
+          createElement(ReactWebChat, {
+            directLine,
+            store,
+            styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
+          })
+        );
+
+        await pageConditions.uiConnected();
+
+        // WHEN: The bot sends an Adaptive Card without "speak", with a hidden text block, a hidden input, and a collapsed "show card" action.
+        await directLine.emulateIncomingActivity({
+          attachments: [
+            {
+              content: {
+                $schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
+                actions: [
+                  {
+                    card: {
+                      body: [{ text: 'Hourly forecast', type: 'TextBlock' }], // Only shown after the action is clicked.
+                      type: 'AdaptiveCard'
+                    },
+                    title: 'Show more',
+                    type: 'Action.ShowCard'
+                  }
+                ],
+                body: [
+                  { text: 'Seattle', type: 'TextBlock' },
+                  { isVisible: false, text: 'Air quality: good', type: 'TextBlock' },
+                  { id: 'note', isVisible: false, label: 'Note', type: 'Input.Text' },
+                  {
+                    isVisible: false, // Hidden subtree: its text block is hidden too.
+                    items: [{ text: 'UV index: 3', type: 'TextBlock' }],
+                    type: 'Container'
+                  }
+                ],
+                type: 'AdaptiveCard',
+                version: '1.5'
+              },
+              contentType: 'application/vnd.microsoft.card.adaptive'
+            }
+          ],
+          from: { role: 'bot' },
+          type: 'message'
+        });
+
+        await pageConditions.numActivitiesShown(1);
+        await pageConditions.liveRegionStabilized();
+
+        const liveRegionActivity = pageElements.liveRegion().querySelector('.webchat__live-region-activity');
+
+        // THEN: The live region copy reads the visible text only, followed by the action button.
+        expect(testHelpers.getAllTextContents(liveRegionActivity).map(text => text.trim())).toEqual([
+          'Bot said:',
+          'A card: Seattle',
+          'Show more'
+        ]);
+
+        // THEN: The hidden input is not listed.
+        expect(liveRegionActivity.querySelectorAll('input')).toHaveLength(0);
+
+        // THEN: The live region shows "A card: Seattle" and the "Show more" button, without the hidden text and input.
+        await host.snapshot('local');
+      });
+    </script>
+  </body>
+</html>

__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.speakOverText.html

@@ -0,0 +1,82 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
+    <link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="importmap">
+      {
+        "imports": {
+          "botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
+          "botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
+          "react": "/__dist__/packages/bundle/static/react.js",
+          "react-dom": "/__dist__/packages/bundle/static/react-dom.js",
+          "react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
+        }
+      }
+    </script>
+    <script type="module">
+      import '/test-harness.mjs';
+      import '/test-page-object.mjs';
+
+      import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
+      import { createElement } from 'react';
+      import { createRoot } from 'react-dom/client';
+
+      // TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
+      window.WebChat = { createStoreWithOptions };
+
+      run(async function () {
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
+        createRoot(document.getElementById('webchat')).render(
+          createElement(ReactWebChat, {
+            directLine,
+            store,
+            styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
+          })
+        );
+
+        await pageConditions.uiConnected();
+
+        // WHEN: The bot sends an Adaptive Card with text and "speak".
+        await directLine.emulateIncomingActivity({
+          attachments: [
+            {
+              content: {
+                $schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
+                body: [
+                  { style: 'heading', text: 'Seattle', type: 'TextBlock' },
+                  { text: 'Sunny, 21°C', type: 'TextBlock' }
+                ],
+                speak: 'Sunny and 21 degrees in Seattle',
+                type: 'AdaptiveCard',
+                version: '1.5'
+              },
+              contentType: 'application/vnd.microsoft.card.adaptive'
+            }
+          ],
+          from: { role: 'bot' },
+          type: 'message'
+        });
+
+        await pageConditions.numActivitiesShown(1);
+        await pageConditions.liveRegionStabilized();
+
+        // THEN: The live region copy describes the card by "speak" only, the text written by the bot for screen readers.
+        expect(
+          testHelpers
+            .getAllTextContents(pageElements.liveRegion().querySelector('.webchat__live-region-activity'))
+            .map(text => text.trim())
+        ).toEqual(['Bot said:', 'A card: Sunny and 21 degrees in Seattle']);
+
+        // THEN: The live region shows "A card:" followed by "speak", not the card text.
+        await host.snapshot('local');
+      });
+    </script>
+  </body>
+</html>

__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.text.html

@@ -0,0 +1,100 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+    <!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
+    <link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="importmap">
+      {
+        "imports": {
+          "botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
+          "botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
+          "react": "/__dist__/packages/bundle/static/react.js",
+          "react-dom": "/__dist__/packages/bundle/static/react-dom.js",
+          "react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
+        }
+      }
+    </script>
+    <script type="module">
+      import '/test-harness.mjs';
+      import '/test-page-object.mjs';
+
+      import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
+      import { createElement } from 'react';
+      import { createRoot } from 'react-dom/client';
+
+      // TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
+      window.WebChat = { createStoreWithOptions };
+
+      run(async function () {
+        const { directLine, store } = testHelpers.createDirectLineEmulator();
+
+        // GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
+        createRoot(document.getElementById('webchat')).render(
+          createElement(ReactWebChat, {
+            directLine,
+            store,
+            styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
+          })
+        );
+
+        await pageConditions.uiConnected();
+
+        // WHEN: The bot sends an Adaptive Card with text only, and without "speak".
+        await directLine.emulateIncomingActivity({
+          attachments: [
+            {
+              content: {
+                $schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
+                body: [
+                  { style: 'heading', text: 'Seattle', type: 'TextBlock' },
+                  { text: 'Sunny, 21°C', type: 'TextBlock' },
+                  { altText: 'Bot avatar', type: 'Image', url: '/assets/bot-avatar.jpg' },
+                  {
+                    facts: [
+                      { title: 'Humidity', value: '40%' },
+                      { title: 'Wind', value: '8 km/h' }
+                    ],
+                    type: 'FactSet'
+                  },
+                  {
+                    inlines: [
+                      { text: 'Updated ', type: 'TextRun' },
+                      { italic: true, text: '5 minutes ago', type: 'TextRun' }
+                    ],
+                    type: 'RichTextBlock'
+                  }
+                ],
+                type: 'AdaptiveCard',
+                version: '1.5'
+              },
+              contentType: 'application/vnd.microsoft.card.adaptive'
+            }
+          ],
+          from: { role: 'bot' },
+          type: 'message'
+        });
+
+        await pageConditions.numActivitiesShown(1);
+        await pageConditions.allImagesLoaded();
+        await pageConditions.liveRegionStabilized();
+
+        // THEN: The live region copy describes the card by its text, in document order: text blocks, image alt text, facts as "title: value", and rich text.
+        expect(
+          testHelpers
+            .getAllTextContents(pageElements.liveRegion().querySelector('.webchat__live-region-activity'))
+            .map(text => text.trim())
+        ).toEqual([
+          'Bot said:',
+          'A card: Seattle Sunny, 21°C Bot avatar Humidity: 40% Wind: 8 km/h Updated 5 minutes ago'
+        ]);
+
+        // THEN: The live region shows "A card:" followed by the card text.
+        await host.snapshot('local');
+      });
+    </script>
+  </body>
+</html>

packages/bundle/src/adaptiveCards/AttachmentForScreenReader/AdaptiveCardAttachment.js

@@ -10,19 +10,22 @@ import useUniqueId from '../hooks/internal/useUniqueId';
 
 const { useLocalizer } = hooks;
 
-// Perform a depth-first search of the Adaptive Card tree.
+// Perform a depth-first search of the visible elements of the Adaptive Card tree, skipping hidden subtrees.
+// Actions are visited but not walked into, so the card of Action.ShowCard, hidden until opened, is not visited.
 function walkAllItems(node, fn) {
-  fn(node);
+  if (node.isVisible) {
+    fn(node);
 
-  if (node.getItemAt && node.getItemCount) {
-    for (let count = node.getItemCount(), index = 0; index < count; index++) {
-      walkAllItems(node.getItemAt(index), fn);
+    if (node.getItemAt && node.getItemCount) {
+      for (let count = node.getItemCount(), index = 0; index < count; index++) {
+        walkAllItems(node.getItemAt(index), fn);
+      }
     }
-  }
 
-  if (node.getActionAt && node.getActionCount) {
-    for (let count = node.getActionCount(), index = 0; index < count; index++) {
-      fn(node.getActionAt(index));
+    if (node.getActionAt && node.getActionCount) {
+      for (let count = node.getActionCount(), index = 0; index < count; index++) {
+        fn(node.getActionAt(index));
+      }
     }
   }
 }
@@ -70,10 +73,14 @@ const AdaptiveCardAttachment = ({ content }) => {
     {
       ChoiceSetInput,
       DateInput,
+      FactSet,
+      Image,
       NumberInput,
       OpenUrlAction,
+      RichTextBlock,
       ShowCardAction,
       SubmitAction,
+      TextBlock,
       TextInput,
       TimeInput,
       ToggleInput
@@ -114,7 +121,27 @@ const AdaptiveCardAttachment = ({ content }) => {
     ToggleInput
   ]);
 
-  const cardLabel = localize('ATTACHMENT_CARD', card.speak || '', '', '');
+  // Without `speak`, the card is described by its visible text, in document order. Inputs and actions are listed separately below.
+  const text = useMemo(() => {
+    const nodes = [];
+
+    walkAllItems(card, node => nodes.push(node));
+
+    return nodes
+      .flatMap(node =>
+        node instanceof FactSet
+          ? node.facts.map(({ name, value }) => [name, value].filter(Boolean).join(': '))
+          : node instanceof Image
+            ? [node.altText]
+            : node instanceof RichTextBlock || node instanceof TextBlock
+              ? [node.asString()]
+              : []
+      )
+      .filter(Boolean)
+      .join(' ');
+  }, [card, FactSet, Image, RichTextBlock, TextBlock]);
+
+  const cardLabel = localize('ATTACHMENT_CARD', card.speak || text, '', '');
 
   return (
     <article>