wit-parser: distinguish doc comments from non-doc comments

crates/wit-component/tests/interfaces/doc-comments.wat

@@ -66,7 +66,7 @@
     )
   )
   (export (;5;) "coverage-world" (type 4))
-  (@custom "package-docs" "\01{\22docs\22:\22package docs;\22,\22worlds\22:{\22coverage-world\22:{\22docs\22:\22world docs\22,\22types\22:{\22t\22:{\22docs\22:\22world typedef docs\22}},\22funcs\22:{\22imp\22:{\22docs\22:\22world func import docs\22}},\22interface_exports\22:{\22i\22:{\22docs\22:\22world inline interface docs\22,\22funcs\22:{\22f\22:{\22docs\22:\22inline interface func docs\22}},\22types\22:{\22t\22:{\22docs\22:\22inline interface typedef docs\22}}}},\22func_exports\22:{\22exp\22:{\22docs\22:\22world func export docs\22}}}},\22interfaces\22:{\22coverage-iface\22:{\22docs\22:\22interface docs\22,\22funcs\22:{\22[constructor]res\22:{\22docs\22:\22constructor docs\22},\22[method]res.m\22:{\22docs\22:\22method docs\22},\22[static]res.s\22:{\22docs\22:\22static func docs\22},\22f\22:{\22docs\22:\22interface func docs\22}},\22types\22:{\22t\22:{\22docs\22:\22basic typedef docs\22},\22r\22:{\22docs\22:\22record typedef docs\22,\22items\22:{\22f1\22:\22record field docs\22}},\22fl\22:{\22items\22:{\22f1\22:\22flag docs\22}},\22v\22:{\22items\22:{\22c1\22:\22variant case docs\22}},\22e\22:{\22items\22:{\22c1\22:\22enum case docs\22}}}},\22other-comment-forms\22:{\22docs\22:\22other comment forms\5cn    multi-line block\22,\22funcs\22:{\22multiple-lines-split\22:{\22docs\22:\22one doc line\5cnnon-doc in the middle\5cnanother doc line\22},\22mixed-forms\22:{\22docs\22:\22mixed forms; line doc\5cnplus block doc\5cn      multi-line\22}}}}}")
+  (@custom "package-docs" "\01{\22docs\22:\22package docs;\22,\22worlds\22:{\22coverage-world\22:{\22docs\22:\22world docs\22,\22types\22:{\22t\22:{\22docs\22:\22world typedef docs\22}},\22funcs\22:{\22imp\22:{\22docs\22:\22world func import docs\22}},\22interface_exports\22:{\22i\22:{\22docs\22:\22world inline interface docs\22,\22funcs\22:{\22f\22:{\22docs\22:\22inline interface func docs\22}},\22types\22:{\22t\22:{\22docs\22:\22inline interface typedef docs\22}}}},\22func_exports\22:{\22exp\22:{\22docs\22:\22world func export docs\22}}}},\22interfaces\22:{\22coverage-iface\22:{\22docs\22:\22interface docs\22,\22funcs\22:{\22[constructor]res\22:{\22docs\22:\22constructor docs\22},\22[method]res.m\22:{\22docs\22:\22method docs\22},\22[static]res.s\22:{\22docs\22:\22static func docs\22},\22f\22:{\22docs\22:\22interface func docs\22}},\22types\22:{\22t\22:{\22docs\22:\22basic typedef docs\22},\22r\22:{\22docs\22:\22record typedef docs\22,\22items\22:{\22f1\22:\22record field docs\22}},\22fl\22:{\22items\22:{\22f1\22:\22flag docs\22}},\22v\22:{\22items\22:{\22c1\22:\22variant case docs\22}},\22e\22:{\22items\22:{\22c1\22:\22enum case docs\22}}}},\22other-comment-forms\22:{\22docs\22:\22other comment forms\5cn    multi-line block\22,\22funcs\22:{\22multiple-lines-split\22:{\22docs\22:\22one doc line\5cnanother doc line\22},\22mixed-forms\22:{\22docs\22:\22mixed forms; line doc\5cnplus block doc\5cn      multi-line\22}}}}}")
   (@producers
     (processed-by "wit-component" "$CARGO_PKG_VERSION")
   )

crates/wit-component/tests/interfaces/doc-comments/foo.wit.print

@@ -48,7 +48,6 @@ interface coverage-iface {
 ///     multi-line block
 interface other-comment-forms {
   /// one doc line
-  /// non-doc in the middle
   /// another doc line
   multiple-lines-split: func();
 

crates/wit-component/tests/interfaces/maps.wat

@@ -179,7 +179,7 @@
     )
   )
   (export (;3;) "maps-test-world" (type 2))
-  (@custom "package-docs" "\01{\22interfaces\22:{\22maps-interface\22:{\22funcs\22:{\22map-param\22:{\22docs\22:\22Functions\22}},\22types\22:{\22bool-map\22:{\22docs\22:\22Test all primitive key types\22},\22string-to-bool\22:{\22docs\22:\22Test all value types\22},\22map-of-maps\22:{\22docs\22:\22Nested structures\22}}}}}")
+  (@custom "package-docs" "\01{}")
   (@producers
     (processed-by "wit-component" "$CARGO_PKG_VERSION")
   )

crates/wit-component/tests/interfaces/maps.wit.print

@@ -1,7 +1,6 @@
 package foo:maps-test;
 
 interface maps-interface {
-  /// Test all primitive key types
   type bool-map = map<bool, string>;
 
   type u8-map = map<u8, string>;
@@ -24,7 +23,6 @@ interface maps-interface {
 
   type string-map = map<string, u32>;
 
-  /// Test all value types
   type string-to-bool = map<string, bool>;
 
   type string-to-list = map<string, list<u8>>;
@@ -35,14 +33,12 @@ interface maps-interface {
 
   type string-to-tuple = map<string, tuple<u32, string>>;
 
-  /// Nested structures
   type map-of-maps = map<string, map<string, u32>>;
 
   type list-of-maps = list<map<string, u32>>;
 
   type option-of-map = option<map<string, u32>>;
 
-  /// Functions
   map-param: func(x: map<string, u32>);
 
   map-result: func() -> map<string, u32>;

crates/wit-parser/src/ast.rs

@@ -1379,8 +1379,12 @@ fn parse_docs<'a>(tokens: &mut Tokenizer<'a>) -> Result<Docs<'a>, lex::Error> {
     let mut started = false;
     while let Some((span, token)) = clone.next_raw()? {
         match token {
-            Token::Whitespace => {}
-            Token::Comment => {
+            // Whitespace and non-doc comments are transparent for the
+            // purposes of collecting doc comments: they neither contribute
+            // to `docs` nor stop us from attaching a later `///`/`/** */`
+            // block to the following item.
+            Token::Whitespace | Token::Comment => {}
+            Token::DocComment => {
                 let comment = tokens.get_span(span);
                 if !started {
                     docs.span.set_start(span.start());

crates/wit-parser/src/ast/lex.rs

@@ -92,7 +92,14 @@ impl Span {
 #[derive(Eq, PartialEq, Debug, Copy, Clone)]
 pub enum Token {
     Whitespace,
+    /// A non-documentation comment: a line comment beginning with `//` (but
+    /// not `///`) or a block comment beginning with `/*` (but not `/**`).
     Comment,
+    /// A documentation comment: a line comment beginning with `///` or a
+    /// block comment beginning with `/**` (matching Rust's rustdoc rules,
+    /// so `////`-prefixed lines and `/**/` are treated as non-doc
+    /// [`Token::Comment`]s instead).
+    DocComment,
 
     Equals,
     Comma,
@@ -234,7 +241,9 @@ impl<'a> Tokenizer<'a> {
     pub fn next(&mut self) -> Result<Option<(Span, Token)>, Error> {
         loop {
             match self.next_raw()? {
-                Some((_, Token::Whitespace)) | Some((_, Token::Comment)) => {}
+                Some((_, Token::Whitespace))
+                | Some((_, Token::Comment))
+                | Some((_, Token::DocComment)) => {}
                 other => break Ok(other),
             }
         }
@@ -258,14 +267,50 @@ impl<'a> Tokenizer<'a> {
             '/' => {
                 // Eat a line comment if it's `//...`
                 if self.eatc('/') {
+                    // Distinguish doc line comments (`///...`) from regular
+                    // line comments (`//...`) and from `////...` which
+                    // rustdoc also treats as non-doc.
+                    let is_doc = {
+                        let mut peek = self.chars.clone();
+                        match (peek.next(), peek.next()) {
+                            (Some((_, '/')), Some((_, ch2))) if ch2 != '/' => true,
+                            (Some((_, '/')), None) => true,
+                            _ => false,
+                        }
+                    };
+                    if is_doc {
+                        // consume the third `/`
+                        self.chars.next();
+                    }
                     for (_, ch) in &mut self.chars {
                         if ch == '\n' {
                             break;
                         }
                     }
-                    Comment
+                    if is_doc { DocComment } else { Comment }
                 // eat a block comment if it's `/*...`
                 } else if self.eatc('*') {
+                    // Distinguish doc block comments (`/** ... */`) from
+                    // regular block comments. Following rustdoc: `/**/`,
+                    // `/***/` (and `/*** ... */` where after the first two
+                    // `*`s the next char is `*` or `/`) are treated as
+                    // non-doc.
+                    let is_doc = {
+                        let mut peek = self.chars.clone();
+                        match peek.next() {
+                            Some((_, '*')) => {
+                                // Look at the char after `/**`; if it is
+                                // `/` we have the empty `/**/` block, and
+                                // if it is `*` we have `/***...` which is
+                                // not a doc block.
+                                match peek.next() {
+                                    Some((_, '/')) | Some((_, '*')) => false,
+                                    _ => true,
+                                }
+                            }
+                            _ => false,
+                        }
+                    };
                     let mut depth = 1;
                     while depth > 0 {
                         let (_, ch) = match self.chars.next() {
@@ -278,7 +323,7 @@ impl<'a> Tokenizer<'a> {
                             _ => {}
                         }
                     }
-                    Comment
+                    if is_doc { DocComment } else { Comment }
                 } else {
                     Slash
                 }
@@ -681,6 +726,7 @@ impl Token {
         match self {
             Whitespace => "whitespace",
             Comment => "a comment",
+            DocComment => "a doc comment",
             Equals => "'='",
             Comma => "','",
             Colon => "':'",
@@ -982,6 +1028,47 @@ fn test_tokenizer() {
     assert!(collect("\u{85}").is_err(), "control code");
 }
 
+#[test]
+fn test_comment_kinds() {
+    fn collect_raw(s: &str) -> Result<Vec<Token>, Error> {
+        let mut t = Tokenizer::new(s, 0)?;
+        let mut tokens = Vec::new();
+        while let Some((_, token)) = t.next_raw()? {
+            match token {
+                Token::Whitespace => continue,
+                _ => tokens.push(token),
+            }
+        }
+        Ok(tokens)
+    }
+
+    // Line comments: `//` is non-doc, `///` is doc, `////`+ is non-doc.
+    assert_eq!(collect_raw("//\n").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("// hi\n").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("///\n").unwrap(), vec![Token::DocComment]);
+    assert_eq!(collect_raw("/// doc\n").unwrap(), vec![Token::DocComment]);
+    assert_eq!(collect_raw("////\n").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("//// not-doc\n").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("///////\n").unwrap(), vec![Token::Comment]);
+
+    // Block comments: `/* */` is non-doc, `/** */` is doc,
+    // `/**/` and `/*** */` are non-doc.
+    assert_eq!(collect_raw("/* hi */").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("/** hi */").unwrap(), vec![Token::DocComment]);
+    assert_eq!(collect_raw("/**/").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("/***/").unwrap(), vec![Token::Comment]);
+    assert_eq!(collect_raw("/*** hi */").unwrap(), vec![Token::Comment]);
+    assert_eq!(
+        collect_raw("/** /* nested */ */").unwrap(),
+        vec![Token::DocComment]
+    );
+
+    // `next` (as opposed to `next_raw`) skips both comment kinds.
+    let mut t = Tokenizer::new("// non-doc\n/// doc\ntype", 0).unwrap();
+    assert!(matches!(t.next().unwrap(), Some((_, Token::Type))));
+    assert!(t.next().unwrap().is_none());
+}
+
 #[test]
 fn test_strings() {
     #[track_caller]

crates/wit-parser/tests/ui/comments.wit

@@ -7,6 +7,8 @@ package foo:comments;
 /* this too */ /* is a comment */
 /* this /* is /* a */ nested */ comment */
 
+/// doc comment for `foo`
+/// second doc line
 interface foo {
   type x = u32;
 
@@ -17,9 +19,26 @@ interface foo {
 
 
 
-
   x
 
   >;
 
+  /// doc comment for `docd`
+  // this line comment must NOT appear in docs
+  /* neither does this block */
+  /** doc block for `docd`, continued */
+  type docd = u32;
+
+  //// four-slash lines are NOT doc comments
+  type undocd = u32;
+
+  /**/
+  /***/
+  type also-undocd = u32;
+}
+
+/// docs for bar interface
+interface bar {
+  /// docs for the-fn
+  the-fn: func();
 }

crates/wit-parser/tests/ui/comments.wit.json

@@ -5,11 +5,32 @@
       "name": "foo",
       "types": {
         "x": 0,
-        "bar": 1
+        "bar": 1,
+        "docd": 2,
+        "undocd": 3,
+        "also-undocd": 4
       },
       "functions": {},
       "docs": {
-        "contents": " hello\n world\n why, yes\n this is a comment\n* this too */\n* is a comment */\n* this /* is /* a */ nested */ comment */"
+        "contents": "doc comment for `foo`\nsecond doc line"
+      },
+      "package": 0
+    },
+    {
+      "name": "bar",
+      "types": {},
+      "functions": {
+        "the-fn": {
+          "name": "the-fn",
+          "kind": "freestanding",
+          "params": [],
+          "docs": {
+            "contents": "docs for the-fn"
+          }
+        }
+      },
+      "docs": {
+        "contents": "docs for bar interface"
       },
       "package": 0
     }
@@ -32,13 +53,44 @@
       "owner": {
         "interface": 0
       }
+    },
+    {
+      "name": "docd",
+      "kind": {
+        "type": "u32"
+      },
+      "owner": {
+        "interface": 0
+      },
+      "docs": {
+        "contents": "doc comment for `docd`\ndoc block for `docd`, continued"
+      }
+    },
+    {
+      "name": "undocd",
+      "kind": {
+        "type": "u32"
+      },
+      "owner": {
+        "interface": 0
+      }
+    },
+    {
+      "name": "also-undocd",
+      "kind": {
+        "type": "u32"
+      },
+      "owner": {
+        "interface": 0
+      }
     }
   ],
   "packages": [
     {
       "name": "foo:comments",
       "interfaces": {
-        "foo": 0
+        "foo": 0,
+        "bar": 1
       },
       "worlds": {}
     }

crates/wit-parser/tests/ui/maps.wit.json

@@ -46,10 +46,7 @@
               "type": "string"
             }
           ],
-          "result": 11,
-          "docs": {
-            "contents": "Functions using maps"
-          }
+          "result": 11
         },
         "set-value": {
           "name": "set-value",
@@ -100,9 +97,6 @@
       },
       "owner": {
         "interface": 0
-      },
-      "docs": {
-        "contents": "Basic maps"
       }
     },
     {
@@ -137,9 +131,6 @@
       },
       "owner": {
         "interface": 0
-      },
-      "docs": {
-        "contents": "Nested maps"
       }
     },
     {
@@ -160,9 +151,6 @@
       },
       "owner": {
         "interface": 0
-      },
-      "docs": {
-        "contents": "Maps with complex types"
       }
     },
     {
@@ -201,9 +189,6 @@
       },
       "owner": {
         "interface": 0
-      },
-      "docs": {
-        "contents": "Maps with tuples, lists, options"
       }
     },
     {
@@ -216,9 +201,6 @@
       },
       "owner": {
         "interface": 0
-      },
-      "docs": {
-        "contents": "Map with various key types"
       }
     },
     {