Handle A-labels case-insensitively in the IDNA module

src/core/dns/hostname.cc

@@ -2,7 +2,6 @@
 #include <sourcemeta/core/idna.h>
 #include <sourcemeta/core/text.h>
 
-#include <string>      // std::string
 #include <string_view> // std::string_view
 
 namespace sourcemeta::core {
@@ -56,18 +55,14 @@ auto is_hostname(const std::string_view value) -> bool {
       return false;
     }
 
-    // RFC 5890 §2.3.2.1: the ACE prefix "xn--" is case-insensitive. A-labels
-    // must also satisfy RFC 5891 §4.2.3 and RFC 5892 (Punycode round-trip,
-    // IDNA 2008 derived properties, contextual rules)
-    if (label_length >= 4 && ((value[label_start] | 0x20) == 'x') &&
-        ((value[label_start + 1] | 0x20) == 'n') &&
-        value[label_start + 2] == '-' && value[label_start + 3] == '-') {
-      std::string canonical{value.substr(label_start, label_length)};
-      canonical[0] = 'x';
-      canonical[1] = 'n';
-      if (!idna_is_valid_a_label(canonical)) {
-        return false;
-      }
+    // RFC 5891 §5.3: an A-label starts in "xn--", interpreted
+    // case-insensitively. A-labels must also satisfy RFC 5891 §4.2.3 and
+    // RFC 5892 (Punycode round-trip, IDNA 2008 derived properties,
+    // contextual rules)
+    const auto label{value.substr(label_start, label_length)};
+    if (starts_with_ignore_case(label, "xn--") &&
+        !idna_is_valid_a_label(label)) {
+      return false;
     }
 
     if (position < value.size()) {

src/core/idna/CMakeLists.txt

@@ -39,3 +39,4 @@ endif()
 
 target_link_libraries(sourcemeta_core_idna PRIVATE sourcemeta::core::unicode)
 target_link_libraries(sourcemeta_core_idna PRIVATE sourcemeta::core::punycode)
+target_link_libraries(sourcemeta_core_idna PRIVATE sourcemeta::core::text)

src/core/idna/idna.cc

@@ -1,6 +1,7 @@
 #include <sourcemeta/core/idna.h>
 
 #include <sourcemeta/core/punycode.h>
+#include <sourcemeta/core/text.h>
 #include <sourcemeta/core/unicode.h>
 
 #include <algorithm>   // std::ranges::lower_bound
@@ -27,8 +28,21 @@ auto validate_a_label_body(const std::string_view encoded,
     return false;
   }
 
+  // RFC 5891 §5.3: "first ensuring that the A-label is entirely in lowercase
+  // (converting it to lowercase if necessary)", which RFC 5891 §4.2.1 also
+  // requires ahead of the canonical form check below. Discarding case loses
+  // nothing, as RFC 3492 §5 assigns A-Z and a-z the same digit values and
+  // RFC 5890 §2.3.2.4 rules out mixed-case annotation
+  std::string lowercased;
+  std::string_view body{encoded};
+  if (!is_lowercase(encoded)) {
+    lowercased.assign(encoded);
+    to_lowercase(lowercased);
+    body = lowercased;
+  }
+
   try {
-    decoded = punycode_to_utf32(encoded);
+    decoded = punycode_to_utf32(body);
   } catch (const PunycodeError &) {
     return false;
   }
@@ -50,10 +64,11 @@ auto validate_a_label_body(const std::string_view encoded,
     return false;
   }
 
-  // RFC 5891 §4.2: A-labels must be in canonical Punycode form, so
-  // re-encoding the decoded U-label must yield the original bytes.
+  // RFC 5891 §4.2.1: "verify that the A-label produced by the step in
+  // Section 4.4 matches the one provided as input", so re-encoding the decoded
+  // U-label must yield the lowercased bytes we decoded from
   try {
-    return utf32_to_punycode(decoded) == encoded;
+    return utf32_to_punycode(decoded) == body;
   } catch (const PunycodeError &) {
     return false;
   }
@@ -93,11 +108,6 @@ auto idna_classify_label(const std::u32string_view label,
       for (const auto codepoint : label) {
         ascii.push_back(static_cast<char>(codepoint));
       }
-      // Normalise the prefix to canonical lowercase before validating, so
-      // the round-trip equality does not reject input that only differs in
-      // the case of the prefix
-      ascii[0] = 'x';
-      ascii[1] = 'n';
       if (!validate_a_label_body(
               std::string_view{ascii.data() + 4, ascii.size() - 4}, decoded)) {
         return std::nullopt;
@@ -418,8 +428,10 @@ auto idna_passes_bidi_rule(const std::u32string_view label) noexcept -> bool {
 }
 
 auto idna_is_valid_a_label(const std::string_view label) -> bool {
+  // RFC 5891 §5.3: a label is treated as an A-label when it starts in
+  // "xn--", interpreted case-insensitively
   constexpr std::string_view PREFIX{"xn--"};
-  if (!label.starts_with(PREFIX)) {
+  if (!starts_with_ignore_case(label, PREFIX)) {
     return false;
   }
 

src/core/idna/include/sourcemeta/core/idna.h

@@ -39,8 +39,9 @@ enum class IDNALabelKind : std::uint8_t {
 /// @ingroup idna
 /// Classify `label` as an Ascii / A-label / U-label per RFC 5890 §2.3.2,
 /// validate the A-label and U-label cases per RFC 5891 §4, and write the
-/// U-label codepoint form to `decoded`. Detection of the ACE prefix "xn--"
-/// is case-insensitive per RFC 5890 §2.3.2.1. For example:
+/// U-label codepoint form to `decoded`. Per RFC 5891 §4.2.1 an A-label is
+/// lowercased before it is decoded, so the ACE prefix "xn--" and the Punycode
+/// body are both matched case-insensitively. For example:
 ///
 /// ```cpp
 /// #include <sourcemeta/core/idna.h>
@@ -161,20 +162,21 @@ auto idna_is_valid_u_label(const std::u32string_view label) -> bool;
 /// @ingroup idna
 /// Return whether the given label is a valid A-label per RFC 5891 §4. See
 /// https://www.rfc-editor.org/rfc/rfc5891#section-4 for the criteria.
-/// A valid A-label starts with the lowercase ACE prefix "xn--", is pure
-/// ASCII, is at most 63 octets, has a non-empty Punycode body that decodes to
-/// a U-label containing at least one non-ASCII codepoint, and round-trips
-/// through Punycode in its canonical form. Both the prefix and the Punycode
-/// body are matched case-sensitively, so an uppercase prefix or a mixed-case
-/// body is rejected. This is intended for registration-side validation rather
-/// than case-folding lookup. For example:
+/// A valid A-label starts with the ACE prefix "xn--", is pure ASCII, is at
+/// most 63 octets, has a non-empty Punycode body that decodes to a U-label
+/// containing at least one non-ASCII codepoint, and round-trips through
+/// Punycode in its canonical form. Per RFC 5891 §4.2.1 the label is lowercased
+/// before it is decoded, so the prefix and the Punycode body are both matched
+/// case-insensitively. For example:
 ///
 /// ```cpp
 /// #include <sourcemeta/core/idna.h>
 /// #include <cassert>
 ///
 /// // xn--mnchen-3ya decodes to "München"
 /// assert(sourcemeta::core::idna_is_valid_a_label("xn--mnchen-3ya"));
+/// // The same label, lowercased before it is decoded
+/// assert(sourcemeta::core::idna_is_valid_a_label("XN--MNCHEN-3YA"));
 /// // Missing "xn--" prefix
 /// assert(!sourcemeta::core::idna_is_valid_a_label("abc"));
 /// // Decodes to "abc" (no non-ASCII codepoint)

test/dns/hostname_test.cc

@@ -2520,3 +2520,47 @@ TEST(valid_single_digit_nine) {
   // the top of the digit range
   EXPECT_TRUE(sourcemeta::core::is_hostname("9"));
 }
+
+// RFC 5891 §5.3: "first ensuring that the A-label is entirely in lowercase
+// (converting it to lowercase if necessary)". RFC 5890 §2.3.2.4 notes an
+// A-label matches "other (mixed-case or uppercase) potential labels in the DNS"
+TEST(xn_uppercase_body) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("xn--NXASMQ6B"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("xn--NXASMQ6B"));
+}
+
+TEST(xn_uppercase_body_with_tld) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("xn--NXASMQ6B.com"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("xn--NXASMQ6B.com"));
+}
+
+TEST(xn_uppercase_prefix_and_body) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("XN--NXASMQ6B.COM"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("XN--NXASMQ6B.COM"));
+}
+
+TEST(xn_mixed_case_body) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("xn--NxAsMq6B.com"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("xn--NxAsMq6B.com"));
+}
+
+TEST(xn_uppercase_basic_portion) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("XN--MNCHEN-3YA"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("XN--MNCHEN-3YA"));
+}
+
+TEST(xn_uppercase_multiple_a_labels) {
+  EXPECT_TRUE(sourcemeta::core::is_hostname("XN--4GBWDL.XN--WGBH1C"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("XN--4GBWDL.XN--WGBH1C"));
+}
+
+// Case folding must not rescue a label that is invalid on its own merits
+TEST(xn_uppercase_leading_spacing_combining_mark) {
+  EXPECT_FALSE(sourcemeta::core::is_hostname("XN--HELLO-TXK"));
+  EXPECT_FALSE(sourcemeta::core::is_idn_hostname("XN--HELLO-TXK"));
+}
+
+TEST(xn_uppercase_leading_nonspacing_mark) {
+  EXPECT_FALSE(sourcemeta::core::is_hostname("XN--HELLO-ZED"));
+  EXPECT_FALSE(sourcemeta::core::is_idn_hostname("XN--HELLO-ZED"));
+}

test/dns/idn_hostname_uts46_test.cc

@@ -46,11 +46,11 @@ TEST(valid_uppercase_non_ascii_is_folded) {
   EXPECT_FALSE(sourcemeta::core::is_idn_hostname("M\xc3\x9cNCHEN"));
 }
 
-// An uppercase A-label is lowercased by the mapping and then validates. The
-// strict validator rejects the uppercase Punycode body
+// An uppercase A-label is lowercased by the mapping and then validates. RFC
+// 5891 §5.3 has the strict validator lowercase it too, so both accept it
 TEST(valid_uppercase_a_label_is_folded) {
   EXPECT_TRUE(sourcemeta::core::is_idn_hostname_uts46("XN--MNCHEN-3YA"));
-  EXPECT_FALSE(sourcemeta::core::is_idn_hostname("XN--MNCHEN-3YA"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_hostname("XN--MNCHEN-3YA"));
 }
 
 // UTS #46 step 1: SOFT HYPHEN (U+00AD) is ignored, so "a<SHY>b" maps to "ab".

test/email/email_test.cc

@@ -1900,3 +1900,14 @@ TEST(invalid_general_address_literal_no_dcontent) {
   EXPECT_FALSE(sourcemeta::core::is_email("a@[x:]"));
   EXPECT_FALSE(sourcemeta::core::is_idn_email("a@[x:]"));
 }
+
+// RFC 5891 §5.3: an A-label domain is lowercased before validation, so the
+// uppercase form of a valid A-label is equally acceptable here
+TEST(valid_uppercase_a_label_domain) {
+  EXPECT_TRUE(sourcemeta::core::is_email("user@xn--NXASMQ6B.com"));
+  EXPECT_TRUE(sourcemeta::core::is_email("user@XN--NXASMQ6B.COM"));
+}
+
+TEST(invalid_uppercase_a_label_domain) {
+  EXPECT_FALSE(sourcemeta::core::is_email("user@XN--HELLO-TXK"));
+}

test/email/idn_email_test.cc

@@ -757,3 +757,14 @@ TEST(valid_quoted_local_escaped_dquote) {
   EXPECT_TRUE(sourcemeta::core::is_idn_email("\"\\\"\"@example.com"));
   EXPECT_TRUE(sourcemeta::core::is_email("\"\\\"\"@example.com"));
 }
+
+// RFC 5891 §5.3: an A-label domain is lowercased before validation, so the
+// uppercase form of a valid A-label is equally acceptable here
+TEST(valid_uppercase_a_label_domain) {
+  EXPECT_TRUE(sourcemeta::core::is_idn_email("\xce\xb1@xn--NXASMQ6B.com"));
+  EXPECT_TRUE(sourcemeta::core::is_idn_email("\xce\xb1@XN--NXASMQ6B.COM"));
+}
+
+TEST(invalid_uppercase_a_label_domain) {
+  EXPECT_FALSE(sourcemeta::core::is_idn_email("\xce\xb1@XN--HELLO-TXK"));
+}

test/idna/idna_classify_label_test.cc

@@ -125,3 +125,56 @@ TEST(invalid_u_label_disallowed_codepoint) {
   EXPECT_EQ(sourcemeta::core::idna_classify_label(U"a\u302Eb", decoded),
             std::nullopt);
 }
+
+// RFC 5891 §5.3: "first ensuring that the A-label is entirely in lowercase
+// (converting it to lowercase if necessary)"
+TEST(valid_a_label_uppercase_body) {
+  std::u32string decoded;
+  const auto kind{
+      sourcemeta::core::idna_classify_label(U"xn--NXASMQ6B", decoded)};
+  EXPECT_TRUE(kind.has_value());
+  EXPECT_EQ(*kind, sourcemeta::core::IDNALabelKind::ALabel);
+  EXPECT_EQ(decoded, U"\U000003B2\U000003CC\U000003BB\U000003BF\U000003C3");
+}
+
+TEST(valid_a_label_uppercase_prefix_and_body) {
+  std::u32string decoded;
+  const auto kind{
+      sourcemeta::core::idna_classify_label(U"XN--NXASMQ6B", decoded)};
+  EXPECT_TRUE(kind.has_value());
+  EXPECT_EQ(*kind, sourcemeta::core::IDNALabelKind::ALabel);
+  EXPECT_EQ(decoded, U"\U000003B2\U000003CC\U000003BB\U000003BF\U000003C3");
+}
+
+TEST(valid_a_label_mixed_case_body) {
+  std::u32string decoded;
+  const auto kind{
+      sourcemeta::core::idna_classify_label(U"xn--NxAsMq6B", decoded)};
+  EXPECT_TRUE(kind.has_value());
+  EXPECT_EQ(*kind, sourcemeta::core::IDNALabelKind::ALabel);
+  EXPECT_EQ(decoded, U"\U000003B2\U000003CC\U000003BB\U000003BF\U000003C3");
+}
+
+// RFC 3492 §5: the decoder copies basic code points verbatim, so the decoded
+// U-label comes from the lowercased body rather than the input as given
+TEST(valid_a_label_uppercase_basic_portion) {
+  std::u32string decoded;
+  const auto kind{
+      sourcemeta::core::idna_classify_label(U"XN--MNCHEN-3YA", decoded)};
+  EXPECT_TRUE(kind.has_value());
+  EXPECT_EQ(*kind, sourcemeta::core::IDNALabelKind::ALabel);
+  EXPECT_EQ(decoded, U"m\U000000FCnchen");
+}
+
+// Case folding must not rescue a label that is invalid on its own merits
+TEST(invalid_a_label_uppercase_decodes_to_pure_ascii) {
+  std::u32string decoded;
+  EXPECT_EQ(sourcemeta::core::idna_classify_label(U"XN--ABC-", decoded),
+            std::nullopt);
+}
+
+TEST(invalid_a_label_uppercase_leading_combining_mark) {
+  std::u32string decoded;
+  EXPECT_EQ(sourcemeta::core::idna_classify_label(U"XN--HELLO-TXK", decoded),
+            std::nullopt);
+}

test/idna/idna_is_valid_a_label_test.cc

@@ -9,7 +9,7 @@ TEST(munich_german) {
 }
 
 TEST(volos_greek) {
-  // xn--nxasmq6b decodes to "\u03B2\u03CC\u03BB\u03BF\u03C2"
+  // xn--nxasmq6b decodes to "\u03B2\u03CC\u03BB\u03BF\u03C3"
   EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--nxasmq6b"));
 }
 
@@ -47,11 +47,11 @@ TEST(non_ascii_byte_in_input) {
   EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("xn--\u00E4"));
 }
 
+// RFC 5890 §2.3.2.4: A-labels "should be produced only in lowercase, despite
+// matching other (mixed-case or uppercase) potential labels in the DNS", so
+// lowercase is a producer convention rather than grounds for rejection
 TEST(uppercase_in_body) {
-  // Punycode is case-insensitive but A-labels are conventionally lowercase.
-  // An uppercase letter in the Punycode body is not the canonical
-  // representation, so the round-trip check rejects it.
-  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("xn--MNCHEN-3ya"));
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--MNCHEN-3ya"));
 }
 
 // RFC 5890 §2.3.2.1: a label in A-label form is at most 63 octets. A 64-octet
@@ -61,3 +61,66 @@ TEST(exceeds_63_octets) {
   EXPECT_EQ(label.size(), 64);
   EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label(label));
 }
+
+// RFC 3492 §5: "A decoder MUST recognize the letters in both uppercase and
+// lowercase forms (including mixtures of both forms)"
+TEST(uppercase_body) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--NXASMQ6B"));
+}
+
+TEST(mixed_case_body) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--NxAsMq6B"));
+}
+
+// RFC 5891 §5.3: the ACE prefix is "interpreted case-insensitively"
+TEST(uppercase_prefix) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("XN--nxasmq6b"));
+}
+
+TEST(mixed_case_prefix) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("Xn--nxasmq6b"));
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xN--nxasmq6b"));
+}
+
+TEST(uppercase_prefix_and_body) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("XN--NXASMQ6B"));
+}
+
+// RFC 3492 §5: basic code points are copied verbatim by the decoder, so the
+// literal portion of the body is lowercased before decoding too
+TEST(uppercase_basic_portion) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--Mnchen-3ya"));
+}
+
+TEST(uppercase_extended_portion) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("xn--mnchen-3YA"));
+}
+
+TEST(uppercase_whole_label_with_basic_portion) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("XN--MNCHEN-3YA"));
+}
+
+TEST(uppercase_accented_label) {
+  EXPECT_TRUE(sourcemeta::core::idna_is_valid_a_label("XN--DJ-KIA8A"));
+}
+
+// Case folding must not make an otherwise invalid label pass
+TEST(uppercase_empty_body) {
+  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("XN--"));
+}
+
+TEST(uppercase_pure_ascii_decode) {
+  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("XN--ABC-"));
+}
+
+TEST(uppercase_leading_spacing_combining_mark) {
+  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("XN--HELLO-TXK"));
+}
+
+TEST(uppercase_non_ascii_body) {
+  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("XN--Ä"));
+}
+
+TEST(uppercase_partial_prefix) {
+  EXPECT_FALSE(sourcemeta::core::idna_is_valid_a_label("XN-ABC"));
+}