Rollup merge of rust-lang#134260 - GuillaumeGomez:doctest-attrs, r=notriddle

Correctly handle comments in attributes in doctests source code

Fixes rust-lang#134221.

The problem was that attributes are "inlined" (backlines are stripped), then when there is an inline comment inside it, the attribute is never considered valid (since unclosed). Fix was to simply put back backlines in case it's a multiline attribute.

r? ``@notriddle``

Author: matthiaskrgr

src/librustdoc/doctest/make.rs

@@ -51,8 +51,17 @@ impl DocTestBuilder {
                 !lang_str.compile_fail && !lang_str.test_harness && !lang_str.standalone_crate
             });
 
-        let SourceInfo { crate_attrs, maybe_crate_attrs, crates, everything_else } =
-            partition_source(source, edition);
+        let Some(SourceInfo { crate_attrs, maybe_crate_attrs, crates, everything_else }) =
+            partition_source(source, edition)
+        else {
+            return Self::invalid(
+                String::new(),
+                String::new(),
+                String::new(),
+                source.to_string(),
+                test_id,
+            );
+        };
 
         // Uses librustc_ast to parse the doctest and find if there's a main fn and the extern
         // crate already is included.
@@ -77,18 +86,7 @@ impl DocTestBuilder {
         else {
             // If the parser panicked due to a fatal error, pass the test code through unchanged.
             // The error will be reported during compilation.
-            return Self {
-                supports_color: false,
-                has_main_fn: false,
-                crate_attrs,
-                maybe_crate_attrs,
-                crates,
-                everything_else,
-                already_has_extern_crate: false,
-                test_id,
-                failed_ast: true,
-                can_be_merged: false,
-            };
+            return Self::invalid(crate_attrs, maybe_crate_attrs, crates, everything_else, test_id);
         };
         // If the AST returned an error, we don't want this doctest to be merged with the
         // others. Same if it contains `#[feature]` or `#[no_std]`.
@@ -113,6 +111,27 @@ impl DocTestBuilder {
         }
     }
 
+    fn invalid(
+        crate_attrs: String,
+        maybe_crate_attrs: String,
+        crates: String,
+        everything_else: String,
+        test_id: Option<String>,
+    ) -> Self {
+        Self {
+            supports_color: false,
+            has_main_fn: false,
+            crate_attrs,
+            maybe_crate_attrs,
+            crates,
+            everything_else,
+            already_has_extern_crate: false,
+            test_id,
+            failed_ast: true,
+            can_be_merged: false,
+        }
+    }
+
     /// Transforms a test into code that can be compiled into a Rust binary, and returns the number of
     /// lines before the test code begins.
     pub(crate) fn generate_unique_doctest(
@@ -518,8 +537,8 @@ fn handle_attr(mod_attr_pending: &mut String, source_info: &mut SourceInfo, edit
         push_to.push('\n');
         // If it's complete, then we can clear the pending content.
         mod_attr_pending.clear();
-    } else if mod_attr_pending.ends_with('\\') {
-        mod_attr_pending.push('n');
+    } else {
+        mod_attr_pending.push_str("\n");
     }
 }
 
@@ -531,7 +550,7 @@ struct SourceInfo {
     everything_else: String,
 }
 
-fn partition_source(s: &str, edition: Edition) -> SourceInfo {
+fn partition_source(s: &str, edition: Edition) -> Option<SourceInfo> {
     #[derive(Copy, Clone, PartialEq)]
     enum PartitionState {
         Attrs,
@@ -606,11 +625,16 @@ fn partition_source(s: &str, edition: Edition) -> SourceInfo {
         }
     }
 
+    if !mod_attr_pending.is_empty() {
+        debug!("invalid doctest code: {s:?}");
+        return None;
+    }
+
     source_info.everything_else = source_info.everything_else.trim().to_string();
 
     debug!("crate_attrs:\n{}{}", source_info.crate_attrs, source_info.maybe_crate_attrs);
     debug!("crates:\n{}", source_info.crates);
     debug!("after:\n{}", source_info.everything_else);
 
-    source_info
+    Some(source_info)
 }

tests/rustdoc-ui/doctest/comment-in-attr-134221-2.rs

@@ -0,0 +1,15 @@
+//@ compile-flags:--test --test-args --test-threads=1
+//@ failure-status: 101
+//@ normalize-stdout-test: "tests/rustdoc-ui/doctest" -> "$$DIR"
+//@ normalize-stdout-test: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ normalize-stdout-test: ".rs:\d+:\d+" -> ".rs:$$LINE:$$COL"
+
+//! ```
+#![doc = "#![all\
+ow(unused)]"]
+//! ```
+//!
+//! ```
+#![doc = r#"#![all\
+ow(unused)]"#]
+//! ```

tests/rustdoc-ui/doctest/comment-in-attr-134221-2.stdout

@@ -0,0 +1,31 @@
+
+running 2 tests
+test $DIR/comment-in-attr-134221-2.rs - (line 11) ... FAILED
+test $DIR/comment-in-attr-134221-2.rs - (line 7) ... ok
+
+failures:
+
+---- $DIR/comment-in-attr-134221-2.rs - (line 11) stdout ----
+error: unknown start of token: \
+  --> $DIR/comment-in-attr-134221-2.rs:$LINE:$COL
+   |
+LL | #![all\
+   |       ^
+
+error: expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `ow`
+  --> $DIR/comment-in-attr-134221-2.rs:$LINE:$COL
+   |
+LL | #![all\
+   |       - expected one of `(`, `::`, `=`, `[`, `]`, or `{`
+LL | ow(unused)]
+   | ^^ unexpected token
+
+error: aborting due to 2 previous errors
+
+Couldn't compile the test.
+
+failures:
+    $DIR/comment-in-attr-134221-2.rs - (line 11)
+
+test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+

tests/rustdoc-ui/doctest/comment-in-attr-134221.rs

@@ -0,0 +1,27 @@
+// Regression test for <https://github.com/rust-lang/rust/issues/134221>.
+// It checks that even if there are comments in the attributes, the attributes
+// will still be generated correctly (and therefore fail in this test).
+
+//@ compile-flags:--test --test-args --test-threads=1
+//@ failure-status: 101
+//@ normalize-stdout-test: "tests/rustdoc-ui/doctest" -> "$$DIR"
+//@ normalize-stdout-test: "finished in \d+\.\d+s" -> "finished in $$TIME"
+//@ normalize-stdout-test: ".rs:\d+:\d+" -> ".rs:$$LINE:$$COL"
+
+/*!
+```rust
+#![feature(
+  foo, //
+)]
+```
+
+```rust
+#![feature(
+  foo,
+)]
+```
+
+```rust
+#![
+```
+*/

tests/rustdoc-ui/doctest/comment-in-attr-134221.stdout

@@ -0,0 +1,50 @@
+
+running 3 tests
+test $DIR/comment-in-attr-134221.rs - (line 11) ... FAILED
+test $DIR/comment-in-attr-134221.rs - (line 17) ... FAILED
+test $DIR/comment-in-attr-134221.rs - (line 23) ... FAILED
+
+failures:
+
+---- $DIR/comment-in-attr-134221.rs - (line 11) stdout ----
+error[E0635]: unknown feature `foo`
+  --> $DIR/comment-in-attr-134221.rs:$LINE:$COL
+   |
+LL |   foo, //
+   |   ^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0635`.
+Couldn't compile the test.
+---- $DIR/comment-in-attr-134221.rs - (line 17) stdout ----
+error[E0635]: unknown feature `foo`
+  --> $DIR/comment-in-attr-134221.rs:$LINE:$COL
+   |
+LL |   foo,
+   |   ^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0635`.
+Couldn't compile the test.
+---- $DIR/comment-in-attr-134221.rs - (line 23) stdout ----
+error: this file contains an unclosed delimiter
+  --> $DIR/comment-in-attr-134221.rs:$LINE:$COL
+   |
+LL | #![
+   |   -^
+   |   |
+   |   unclosed delimiter
+
+error: aborting due to 1 previous error
+
+Couldn't compile the test.
+
+failures:
+    $DIR/comment-in-attr-134221.rs - (line 11)
+    $DIR/comment-in-attr-134221.rs - (line 17)
+    $DIR/comment-in-attr-134221.rs - (line 23)
+
+test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+