Rollup merge of rust-lang#69838 - Centril:expand-module, r=petrochenkov,eddyb

Expansion-driven outline module parsing

After this PR, the parser will not do any conditional compilation or loading of external module files when `mod foo;` is encountered. Instead, the parser only leaves `mod foo;` in place in the AST, with no items filled in. Expansion later kicks in and will load the actual files and do the parsing. This entails that the following is now valid:

```rust
#[cfg(FALSE)]
mod foo {
    mod bar {
        mod baz; // `foo/bar/baz.rs` doesn't exist, but no error!
    }
}
```

Fixes rust-lang#64197.

r? @petrochenkov

Author: Centril

src/librustc_ast/ast.rs

@@ -2153,7 +2153,7 @@ impl FnRetTy {
 /// Module declaration.
 ///
 /// E.g., `mod foo;` or `mod foo { .. }`.
-#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Default)]
 pub struct Mod {
     /// A span from the first token past `{` to the last token until `}`.
     /// For `mod foo;`, the inner span ranges from the first token

src/librustc_builtin_macros/source_util.rs

@@ -5,7 +5,7 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
 use rustc_expand::panictry;
-use rustc_parse::{self, new_sub_parser_from_file, parser::Parser, DirectoryOwnership};
+use rustc_parse::{self, new_sub_parser_from_file, parser::Parser};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, Pos, Span};
@@ -108,8 +108,7 @@ pub fn expand_include<'cx>(
             return DummyResult::any(sp);
         }
     };
-    let directory_ownership = DirectoryOwnership::Owned { relative: None };
-    let p = new_sub_parser_from_file(cx.parse_sess(), &file, directory_ownership, None, sp);
+    let p = new_sub_parser_from_file(cx.parse_sess(), &file, None, sp);
 
     struct ExpandResult<'a> {
         p: Parser<'a>,

src/librustc_expand/base.rs

@@ -1,4 +1,5 @@
 use crate::expand::{self, AstFragment, Invocation};
+use crate::module::DirectoryOwnership;
 
 use rustc_ast::ast::{self, Attribute, Name, NodeId, PatKind};
 use rustc_ast::mut_visit::{self, MutVisitor};
@@ -10,7 +11,7 @@ use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, DiagnosticId};
-use rustc_parse::{self, parser, DirectoryOwnership, MACRO_ARGUMENTS};
+use rustc_parse::{self, parser, MACRO_ARGUMENTS};
 use rustc_session::parse::ParseSess;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::{AstPass, ExpnData, ExpnId, ExpnKind};

src/librustc_parse/config.rs renamed to src/librustc_expand/config.rs

@@ -1,14 +1,5 @@
-//! Process the potential `cfg` attributes on a module.
-//! Also determine if the module should be included in this configuration.
-//!
-//! This module properly belongs in rustc_expand, but for now it's tied into
-//! parsing, so we leave it here to avoid complicated out-of-line dependencies.
-//!
-//! A principled solution to this wrong location would be to implement [#64197].
-//!
-//! [#64197]: https://github.com/rust-lang/rust/issues/64197
-
-use crate::{parse_in, validate_attr};
+//! Conditional compilation stripping.
+
 use rustc_ast::ast::{self, AttrItem, Attribute, MetaItem};
 use rustc_ast::attr::HasAttrs;
 use rustc_ast::mut_visit::*;
@@ -21,6 +12,7 @@ use rustc_feature::{Feature, Features, State as FeatureState};
 use rustc_feature::{
     ACCEPTED_FEATURES, ACTIVE_FEATURES, REMOVED_FEATURES, STABLE_REMOVED_FEATURES,
 };
+use rustc_parse::{parse_in, validate_attr};
 use rustc_session::parse::{feature_err, ParseSess};
 use rustc_span::edition::{Edition, ALL_EDITIONS};
 use rustc_span::symbol::{sym, Symbol};
@@ -538,12 +530,3 @@ impl<'a> MutVisitor for StripUnconfigured<'a> {
 fn is_cfg(attr: &Attribute) -> bool {
     attr.check_name(sym::cfg)
 }
-
-/// Process the potential `cfg` attributes on a module.
-/// Also determine if the module should be included in this configuration.
-pub fn process_configure_mod(sess: &ParseSess, cfg_mods: bool, attrs: &mut Vec<Attribute>) -> bool {
-    // Don't perform gated feature checking.
-    let mut strip_unconfigured = StripUnconfigured { sess, features: None };
-    strip_unconfigured.process_cfg_attrs(attrs);
-    !cfg_mods || strip_unconfigured.in_cfg(&attrs)
-}

src/librustc_expand/expand.rs

@@ -1,7 +1,9 @@
 use crate::base::*;
 use crate::config::StripUnconfigured;
+use crate::configure;
 use crate::hygiene::{ExpnData, ExpnId, ExpnKind, SyntaxContext};
 use crate::mbe::macro_rules::annotate_err_with_kind;
+use crate::module::{parse_external_mod, push_directory, Directory, DirectoryOwnership};
 use crate::placeholders::{placeholder, PlaceholderExpander};
 use crate::proc_macro::collect_derives;
 
@@ -17,10 +19,8 @@ use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, is_builtin_attr, HasAttrs};
 use rustc_errors::{Applicability, FatalError, PResult};
 use rustc_feature::Features;
-use rustc_parse::configure;
 use rustc_parse::parser::Parser;
 use rustc_parse::validate_attr;
-use rustc_parse::DirectoryOwnership;
 use rustc_session::lint::builtin::UNUSED_DOC_COMMENTS;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::parse::{feature_err, ParseSess};
@@ -1359,59 +1359,72 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 .make_items();
         }
 
+        let mut attrs = mem::take(&mut item.attrs); // We do this to please borrowck.
+        let ident = item.ident;
+        let span = item.span;
+
         match item.kind {
             ast::ItemKind::Mac(..) => {
+                item.attrs = attrs;
                 self.check_attributes(&item.attrs);
                 item.and_then(|item| match item.kind {
                     ItemKind::Mac(mac) => self
-                        .collect(
-                            AstFragmentKind::Items,
-                            InvocationKind::Bang { mac, span: item.span },
-                        )
+                        .collect(AstFragmentKind::Items, InvocationKind::Bang { mac, span })
                         .make_items(),
                     _ => unreachable!(),
                 })
             }
-            ast::ItemKind::Mod(ast::Mod { inner, inline, .. })
-                if item.ident != Ident::invalid() =>
-            {
-                let orig_directory_ownership = self.cx.current_expansion.directory_ownership;
+            ast::ItemKind::Mod(ref mut old_mod @ ast::Mod { .. }) if ident != Ident::invalid() => {
+                let sess = self.cx.parse_sess;
+                let orig_ownership = self.cx.current_expansion.directory_ownership;
                 let mut module = (*self.cx.current_expansion.module).clone();
-                module.mod_path.push(item.ident);
 
-                if inline {
-                    if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) {
-                        self.cx.current_expansion.directory_ownership =
-                            DirectoryOwnership::Owned { relative: None };
-                        module.directory.push(&*path.as_str());
-                    } else {
-                        module.directory.push(&*item.ident.as_str());
-                    }
+                let pushed = &mut false; // Record `parse_external_mod` pushing so we can pop.
+                let dir = Directory { ownership: orig_ownership, path: module.directory };
+                let Directory { ownership, path } = if old_mod.inline {
+                    // Inline `mod foo { ... }`, but we still need to push directories.
+                    item.attrs = attrs;
+                    push_directory(ident, &item.attrs, dir)
                 } else {
-                    let path = self.cx.parse_sess.source_map().span_to_unmapped_path(inner);
-                    let mut path = match path {
-                        FileName::Real(path) => path,
-                        other => PathBuf::from(other.to_string()),
-                    };
-                    let directory_ownership = match path.file_name().unwrap().to_str() {
-                        Some("mod.rs") => DirectoryOwnership::Owned { relative: None },
-                        Some(_) => DirectoryOwnership::Owned { relative: Some(item.ident) },
-                        None => DirectoryOwnership::UnownedViaMod,
+                    // We have an outline `mod foo;` so we need to parse the file.
+                    let (new_mod, dir) =
+                        parse_external_mod(sess, ident, span, dir, &mut attrs, pushed);
+                    *old_mod = new_mod;
+                    item.attrs = attrs;
+                    // File can have inline attributes, e.g., `#![cfg(...)]` & co. => Reconfigure.
+                    item = match self.configure(item) {
+                        Some(node) => node,
+                        None => {
+                            if *pushed {
+                                sess.included_mod_stack.borrow_mut().pop();
+                            }
+                            return Default::default();
+                        }
                     };
-                    path.pop();
-                    module.directory = path;
-                    self.cx.current_expansion.directory_ownership = directory_ownership;
-                }
+                    dir
+                };
 
+                // Set the module info before we flat map.
+                self.cx.current_expansion.directory_ownership = ownership;
+                module.directory = path;
+                module.mod_path.push(ident);
                 let orig_module =
                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
+
                 let result = noop_flat_map_item(item, self);
+
+                // Restore the module info.
                 self.cx.current_expansion.module = orig_module;
-                self.cx.current_expansion.directory_ownership = orig_directory_ownership;
+                self.cx.current_expansion.directory_ownership = orig_ownership;
+                if *pushed {
+                    sess.included_mod_stack.borrow_mut().pop();
+                }
                 result
             }
-
-            _ => noop_flat_map_item(item, self),
+            _ => {
+                item.attrs = attrs;
+                noop_flat_map_item(item, self)
+            }
         }
     }
 

src/librustc_expand/lib.rs

@@ -1,9 +1,11 @@
+#![feature(bool_to_option)]
 #![feature(cow_is_borrowed)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_span)]
+#![feature(try_blocks)]
 
 extern crate proc_macro as pm;
 
@@ -33,8 +35,10 @@ pub use mbe::macro_rules::compile_declarative_macro;
 crate use rustc_span::hygiene;
 pub mod base;
 pub mod build;
+#[macro_use]
+pub mod config;
 pub mod expand;
-pub use rustc_parse::config;
+pub mod module;
 pub mod proc_macro;
 
 crate mod mbe;

src/librustc_expand/mbe/macro_rules.rs

@@ -1,4 +1,4 @@
-use crate::base::{DummyResult, ExpansionData, ExtCtxt, MacResult, TTMacroExpander};
+use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander};
 use crate::base::{SyntaxExtension, SyntaxExtensionKind};
 use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind};
 use crate::mbe;
@@ -18,7 +18,6 @@ use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, DiagnosticBuilder, FatalError};
 use rustc_feature::Features;
 use rustc_parse::parser::Parser;
-use rustc_parse::Directory;
 use rustc_session::parse::ParseSess;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::Transparency;
@@ -182,6 +181,8 @@ fn generic_extension<'cx>(
     lhses: &[mbe::TokenTree],
     rhses: &[mbe::TokenTree],
 ) -> Box<dyn MacResult + 'cx> {
+    let sess = cx.parse_sess;
+
     if cx.trace_macros() {
         let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone()));
         trace_macros_note(&mut cx.expansions, sp, msg);
@@ -209,7 +210,7 @@ fn generic_extension<'cx>(
     // hacky, but speeds up the `html5ever` benchmark significantly. (Issue
     // 68836 suggests a more comprehensive but more complex change to deal with
     // this situation.)
-    let parser = parser_from_cx(&cx.current_expansion, &cx.parse_sess, arg.clone());
+    let parser = parser_from_cx(sess, arg.clone());
 
     for (i, lhs) in lhses.iter().enumerate() {
         // try each arm's matchers
@@ -222,14 +223,13 @@ fn generic_extension<'cx>(
         // This is used so that if a matcher is not `Success(..)`ful,
         // then the spans which became gated when parsing the unsuccessful matcher
         // are not recorded. On the first `Success(..)`ful matcher, the spans are merged.
-        let mut gated_spans_snapshot =
-            mem::take(&mut *cx.parse_sess.gated_spans.spans.borrow_mut());
+        let mut gated_spans_snapshot = mem::take(&mut *sess.gated_spans.spans.borrow_mut());
 
         match parse_tt(&mut Cow::Borrowed(&parser), lhs_tt) {
             Success(named_matches) => {
                 // The matcher was `Success(..)`ful.
                 // Merge the gated spans from parsing the matcher with the pre-existing ones.
-                cx.parse_sess.gated_spans.merge(gated_spans_snapshot);
+                sess.gated_spans.merge(gated_spans_snapshot);
 
                 let rhs = match rhses[i] {
                     // ignore delimiters
@@ -258,11 +258,7 @@ fn generic_extension<'cx>(
                     trace_macros_note(&mut cx.expansions, sp, msg);
                 }
 
-                let directory = Directory {
-                    path: cx.current_expansion.module.directory.clone(),
-                    ownership: cx.current_expansion.directory_ownership,
-                };
-                let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None);
+                let mut p = Parser::new(sess, tts, false, None);
                 p.root_module_name =
                     cx.current_expansion.module.mod_path.last().map(|id| id.to_string());
                 p.last_type_ascription = cx.current_expansion.prior_type_ascription;
@@ -289,7 +285,7 @@ fn generic_extension<'cx>(
 
         // The matcher was not `Success(..)`ful.
         // Restore to the state before snapshotting and maybe try again.
-        mem::swap(&mut gated_spans_snapshot, &mut cx.parse_sess.gated_spans.spans.borrow_mut());
+        mem::swap(&mut gated_spans_snapshot, &mut sess.gated_spans.spans.borrow_mut());
     }
     drop(parser);
 
@@ -309,8 +305,7 @@ fn generic_extension<'cx>(
                 mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
                 _ => continue,
             };
-            let parser = parser_from_cx(&cx.current_expansion, &cx.parse_sess, arg.clone());
-            match parse_tt(&mut Cow::Borrowed(&parser), lhs_tt) {
+            match parse_tt(&mut Cow::Borrowed(&parser_from_cx(sess, arg.clone())), lhs_tt) {
                 Success(_) => {
                     if comma_span.is_dummy() {
                         err.note("you might be missing a comma");
@@ -392,7 +387,7 @@ pub fn compile_declarative_macro(
         ),
     ];
 
-    let parser = Parser::new(sess, body, None, true, true, rustc_parse::MACRO_ARGUMENTS);
+    let parser = Parser::new(sess, body, true, rustc_parse::MACRO_ARGUMENTS);
     let argument_map = match parse_tt(&mut Cow::Borrowed(&parser), &argument_gram) {
         Success(m) => m,
         Failure(token, msg) => {
@@ -1209,16 +1204,8 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     }
 }
 
-fn parser_from_cx<'cx>(
-    current_expansion: &'cx ExpansionData,
-    sess: &'cx ParseSess,
-    tts: TokenStream,
-) -> Parser<'cx> {
-    let directory = Directory {
-        path: current_expansion.module.directory.clone(),
-        ownership: current_expansion.directory_ownership,
-    };
-    Parser::new(sess, tts, Some(directory), true, true, rustc_parse::MACRO_ARGUMENTS)
+fn parser_from_cx(sess: &ParseSess, tts: TokenStream) -> Parser<'_> {
+    Parser::new(sess, tts, true, rustc_parse::MACRO_ARGUMENTS)
 }
 
 /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For