Rollup merge of #33041 - petrochenkov:path, r=nrc,Manishearth

 Paths are mostly parsed without taking whitespaces into account, e.g. `std :: vec :: Vec :: new ()` parses successfully, however, there are some special cases involving keywords `super`, `self` and `Self`. For example, `self::` is considered a path start only if there are no spaces between `self` and `::`. These restrictions probably made sense when `self` and friends weren't keywords, but now they are unnecessary.

The first two commits remove this special treatment of whitespaces by removing `token::IdentStyle` entirely and therefore fix #14109.
This change also affects naked `self` and `super` (which are not tightly followed by `::`, obviously) they can now be parsed as paths, however they are still not resolved correctly in imports (cc @jseyfried, see `compile-fail/use-keyword.rs`), so #29036 is not completely fixed.

The third commit also makes `super`, `self`, `Self` and `static` keywords nominally (before this they acted as keywords for all purposes) and removes most of remaining \"special idents\".

The last commit (before tests) contains some small improvements - some qualified paths with type parameters are parsed correctly, `parse_path` is not used for parsing single identifiers, imports are sanity checked for absence of type parameters - such type parameters can be generated by syntax extensions or by macros when #10415 is fixed (~~soon!~~already!).

This patch changes some pretty basic things in `libsyntax`, like `token::Token` and the keyword list, so it's a plugin-[breaking-change].

r? @eddyb

Author: Manishearth

src/etc/generate-keyword-tests.py

@@ -34,15 +34,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// compile-flags: -Z parse-only
+
 // This file was auto-generated using 'src/etc/generate-keyword-tests.py %s'
 
 fn main() {
-    let %s = "foo"; //~ error: ident
+    let %s = "foo"; //~ error: expected pattern, found keyword `%s`
 }
 """
 
 test_dir = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), '../test/compile-fail')
+    os.path.join(os.path.dirname(__file__), '../test/parse-fail')
 )
 
 for kw in sys.argv[1:]:
@@ -53,7 +55,7 @@
         os.chmod(test_file, stat.S_IWUSR)
 
     with open(test_file, 'wt') as f:
-        f.write(template % (datetime.datetime.now().year, kw, kw))
+        f.write(template % (datetime.datetime.now().year, kw, kw, kw))
 
     # mark file read-only
     os.chmod(test_file, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

src/librustc/hir/fold.rs

@@ -18,7 +18,7 @@ use syntax::attr::ThinAttributesExt;
 use hir;
 use syntax::codemap::{respan, Span, Spanned};
 use syntax::ptr::P;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 use syntax::util::move_map::MoveMap;
 
 pub trait Folder : Sized {
@@ -867,7 +867,7 @@ pub fn noop_fold_crate<T: Folder>(Crate { module, attrs, config, span,
     let config = folder.fold_meta_items(config);
 
     let crate_mod = folder.fold_item(hir::Item {
-        name: token::special_idents::invalid.name,
+        name: keywords::Invalid.name(),
         attrs: attrs,
         id: DUMMY_NODE_ID,
         vis: hir::Public,

src/librustc/hir/print.rs

@@ -14,9 +14,8 @@ use syntax::abi::Abi;
 use syntax::ast;
 use syntax::codemap::{self, CodeMap, BytePos, Spanned};
 use syntax::errors;
-use syntax::parse::token::{self, BinOpToken};
+use syntax::parse::token::{self, keywords, BinOpToken};
 use syntax::parse::lexer::comments;
-use syntax::parse;
 use syntax::print::pp::{self, break_offset, word, space, hardbreak};
 use syntax::print::pp::{Breaks, eof};
 use syntax::print::pp::Breaks::{Consistent, Inconsistent};
@@ -2209,9 +2208,8 @@ impl<'a> State<'a> {
             hir::TyInfer if is_closure => self.print_pat(&input.pat)?,
             _ => {
                 match input.pat.node {
-                    PatKind::Ident(_, ref path1, _) if
-                        path1.node.name ==
-                            parse::token::special_idents::invalid.name => {
+                    PatKind::Ident(_, ref path1, _)
+                            if path1.node.name == keywords::Invalid.name() => {
                         // Do nothing.
                     }
                     _ => {

src/librustc/middle/liveness.rs

@@ -125,7 +125,7 @@ use std::io;
 use std::rc::Rc;
 use syntax::ast::{self, NodeId};
 use syntax::codemap::{BytePos, original_sp, Span};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use syntax::ptr::P;
 
 use hir::Expr;
@@ -1578,7 +1578,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 let var = self.variable(p_id, sp);
                 // Ignore unused self.
                 let name = path1.node;
-                if name != special_idents::self_.name {
+                if name != keywords::SelfValue.name() {
                     if !self.warn_about_unused(sp, p_id, entry_ln, var) {
                         if self.live_on_entry(entry_ln, var).is_none() {
                             self.report_dead_assign(p_id, sp, var, true);

src/librustc/middle/resolve_lifetime.rs

@@ -29,7 +29,7 @@ use std::fmt;
 use std::mem::replace;
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 use util::nodemap::NodeMap;
 
 use hir;
@@ -245,7 +245,7 @@ impl<'a, 'v> Visitor<'v> for LifetimeContext<'a> {
     }
 
     fn visit_lifetime(&mut self, lifetime_ref: &hir::Lifetime) {
-        if lifetime_ref.name == special_idents::static_lifetime.name {
+        if lifetime_ref.name == keywords::StaticLifetime.name() {
             self.insert_lifetime(lifetime_ref, DefStaticRegion);
             return;
         }
@@ -672,9 +672,8 @@ impl<'a> LifetimeContext<'a> {
         for i in 0..lifetimes.len() {
             let lifetime_i = &lifetimes[i];
 
-            let special_idents = [special_idents::static_lifetime];
             for lifetime in lifetimes {
-                if special_idents.iter().any(|&i| i.name == lifetime.lifetime.name) {
+                if lifetime.lifetime.name == keywords::StaticLifetime.name() {
                     span_err!(self.sess, lifetime.lifetime.span, E0262,
                         "invalid lifetime parameter name: `{}`", lifetime.lifetime.name);
                 }

src/librustc/mir/repr.rs

@@ -200,7 +200,7 @@ pub struct ArgDecl<'tcx> {
     /// and has to be collected from multiple actual arguments.
     pub spread: bool,
 
-    /// Either special_idents::invalid or the name of a single-binding
+    /// Either keywords::Invalid or the name of a single-binding
     /// pattern associated with this argument. Useful for debuginfo.
     pub debug_name: Name
 }

src/librustc/ty/context.rs

@@ -44,7 +44,7 @@ use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
-use syntax::parse::token::{self, special_idents};
+use syntax::parse::token::{self, keywords};
 
 use hir;
 
@@ -1069,7 +1069,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn mk_self_type(&self) -> Ty<'tcx> {
-        self.mk_param(subst::SelfSpace, 0, special_idents::type_self.name)
+        self.mk_param(subst::SelfSpace, 0, keywords::SelfType.name())
     }
 
     pub fn mk_param_from_def(&self, def: &ty::TypeParameterDef) -> Ty<'tcx> {

src/librustc/ty/sty.rs

@@ -24,7 +24,7 @@ use std::ops;
 use std::mem;
 use syntax::abi;
 use syntax::ast::{self, Name};
-use syntax::parse::token::special_idents;
+use syntax::parse::token::keywords;
 
 use serialize::{Decodable, Decoder};
 
@@ -533,7 +533,7 @@ impl ParamTy {
     }
 
     pub fn for_self() -> ParamTy {
-        ParamTy::new(subst::SelfSpace, 0, special_idents::type_self.name)
+        ParamTy::new(subst::SelfSpace, 0, keywords::SelfType.name())
     }
 
     pub fn for_def(def: &ty::TypeParameterDef) -> ParamTy {

src/librustc_mir/build/mod.rs

@@ -18,7 +18,7 @@ use rustc::hir::pat_util::pat_is_binding;
 use std::ops::{Index, IndexMut};
 use syntax::ast;
 use syntax::codemap::Span;
-use syntax::parse::token;
+use syntax::parse::token::keywords;
 
 pub struct Builder<'a, 'tcx: 'a> {
     hir: Cx<'a, 'tcx>,
@@ -238,7 +238,7 @@ pub fn construct<'a,'tcx>(hir: Cx<'a,'tcx>,
                 ty::UpvarCapture::ByRef(..) => true
             });
             let mut decl = UpvarDecl {
-                debug_name: token::special_idents::invalid.name,
+                debug_name: keywords::Invalid.name(),
                 by_ref: by_ref
             };
             if let Some(hir::map::NodeLocal(pat)) = tcx.map.find(fv.def.var_id()) {
@@ -296,7 +296,7 @@ impl<'a,'tcx> Builder<'a,'tcx> {
                 self.schedule_drop(pattern.as_ref().map_or(ast_block.span, |pat| pat.span),
                                    argument_extent, &lvalue, ty);
 
-                let mut name = token::special_idents::invalid.name;
+                let mut name = keywords::Invalid.name();
                 if let Some(pat) = pattern {
                     if let hir::PatKind::Ident(_, ref ident, _) = pat.node {
                         if pat_is_binding(&self.hir.tcx().def_map.borrow(), pat) {

src/librustc_resolve/build_reduced_graph.rs

@@ -112,15 +112,14 @@ impl<'b, 'tcx:'b> Resolver<'b, 'tcx> {
             !segment.parameters.bindings().is_empty()
         });
         if found_param {
-            self.session.span_err(path.span,
-                                  "type or lifetime parameter is found in import path");
+            self.session.span_err(path.span, "type or lifetime parameters in import path");
         }
 
         // Checking for special identifiers in path
         // prevent `self` or `super` at beginning of global path
         if path.global && path.segments.len() > 0 {
             let first = path.segments[0].identifier.name;
-            if first == keywords::Super.to_name() || first == keywords::SelfValue.to_name() {
+            if first == keywords::Super.name() || first == keywords::SelfValue.name() {
                 self.session.add_lint(
                     lint::builtin::SUPER_OR_SELF_IN_GLOBAL_PATH, id, path.span,
                     format!("expected identifier, found keyword `{}`", first)