remove XID and Pattern_White_Space unicode tables from libcore

They are only used by rustc_lexer, and are not needed elsewhere.

So we move the relevant definitions into rustc_lexer (while the actual
unicode data comes from the unicode-xid crate) and make the rest of
the compiler use it.

Author: matklad

Cargo.lock

@@ -1011,6 +1011,7 @@ dependencies = [
 name = "fmt_macros"
 version = "0.0.0"
 dependencies = [
+ "rustc_lexer",
  "syntax_pos",
 ]
 
@@ -2372,7 +2373,7 @@ version = "0.4.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
 dependencies = [
- "unicode-xid",
+ "unicode-xid 0.1.0",
 ]
 
 [[package]]
@@ -3290,7 +3291,7 @@ dependencies = [
 name = "rustc_lexer"
 version = "0.1.0"
 dependencies = [
- "unicode-xid",
+ "unicode-xid 0.2.0",
 ]
 
 [[package]]
@@ -3368,6 +3369,7 @@ dependencies = [
  "rustc_apfloat",
  "rustc_data_structures",
  "rustc_errors",
+ "rustc_lexer",
  "rustc_target",
  "serialize",
  "smallvec",
@@ -3976,7 +3978,7 @@ checksum = "641e117d55514d6d918490e47102f7e08d096fdde360247e4a10f7a91a8478d3"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-xid 0.1.0",
 ]
 
 [[package]]
@@ -3988,7 +3990,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn",
- "unicode-xid",
+ "unicode-xid 0.1.0",
 ]
 
 [[package]]
@@ -4017,6 +4019,7 @@ dependencies = [
  "log",
  "rustc_data_structures",
  "rustc_errors",
+ "rustc_lexer",
  "rustc_target",
  "smallvec",
  "syntax",
@@ -4532,6 +4535,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
+
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"

src/libcore/char/methods.rs

@@ -547,29 +547,6 @@ impl char {
         }
     }
 
-    /// Returns `true` if this `char` satisfies the `XID_Start` Unicode property, and false
-    /// otherwise.
-    ///
-    /// `XID_Start` is a Unicode Derived Property specified in
-    /// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
-    /// mostly similar to `ID_Start` but modified for closure under `NFKx`.
-    #[unstable(feature = "unicode_internals", issue = "0")]
-    pub fn is_xid_start(self) -> bool {
-        derived_property::XID_Start(self)
-    }
-
-    /// Returns `true` if this `char` satisfies the `XID_Continue` Unicode property, and false
-    /// otherwise.
-    ///
-    /// `XID_Continue` is a Unicode Derived Property specified in
-    /// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
-    /// mostly similar to `ID_Continue` but modified for closure under NFKx.
-    #[unstable(feature = "unicode_internals", issue = "0")]
-    #[inline]
-    pub fn is_xid_continue(self) -> bool {
-        derived_property::XID_Continue(self)
-    }
-
     /// Returns `true` if this `char` is lowercase.
     ///
     /// 'Lowercase' is defined according to the terms of the Unicode Derived Core

src/libcore/unicode/mod.rs

@@ -13,8 +13,3 @@ pub mod derived_property {
 pub mod conversions {
     pub use crate::unicode::tables::conversions::{to_lower, to_upper};
 }
-
-// For use in libsyntax
-pub mod property {
-    pub use crate::unicode::tables::property::Pattern_White_Space;
-}

src/libcore/unicode/tables.rs

@@ -728,7 +728,7 @@ def generate_property_module(mod, grouped_categories, category_subset):
 
     yield "pub(crate) mod %s {\n" % mod
     for cat in sorted(category_subset):
-        if cat in ("Cc", "White_Space", "Pattern_White_Space"):
+        if cat in ("Cc", "White_Space"):
             generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat])
         else:
             generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat])
@@ -841,19 +841,18 @@ def main():
     unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA))
     load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data)
 
-    want_derived = {"XID_Start", "XID_Continue", "Alphabetic", "Lowercase", "Uppercase",
+    want_derived = {"Alphabetic", "Lowercase", "Uppercase",
                     "Cased", "Case_Ignorable", "Grapheme_Extend"}
     derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived)
 
     props = load_properties(get_path(UnicodeFiles.PROPS),
-                            {"White_Space", "Join_Control", "Noncharacter_Code_Point",
-                             "Pattern_White_Space"})
+                            {"White_Space", "Join_Control", "Noncharacter_Code_Point"})
 
     # Category tables
     for (name, categories, category_subset) in (
             ("general_category", unicode_data.general_categories, ["N", "Cc"]),
             ("derived_property", derived, want_derived),
-            ("property", props, ["White_Space", "Pattern_White_Space"])
+            ("property", props, ["White_Space"])
     ):
         for fragment in generate_property_module(name, categories, category_subset):
             buf.write(fragment)

src/libcore/unicode/unicode.py

@@ -10,4 +10,4 @@ path = "lib.rs"
 
 [dependencies]
 syntax_pos = { path = "../libsyntax_pos" }
-
+rustc_lexer = { path = "../librustc_lexer" }

src/libfmt_macros/Cargo.toml

@@ -23,6 +23,7 @@ use std::string;
 use std::iter;
 
 use syntax_pos::{InnerSpan, Symbol};
+use rustc_lexer::character_properties::{is_id_start, is_id_continue};
 
 #[derive(Copy, Clone)]
 struct InnerOffset(usize);
@@ -597,12 +598,11 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Parses a word starting at the current position. A word is considered to
-    /// be an alphabetic character followed by any number of alphanumeric
-    /// characters.
+    /// Parses a word starting at the current position. A word is the same as
+    /// Rust identifier, except that it can't start with `_` character.
     fn word(&mut self) -> &'a str {
         let start = match self.cur.peek() {
-            Some(&(pos, c)) if c.is_xid_start() => {
+            Some(&(pos, c)) if c != '_' && is_id_start(c) => {
                 self.cur.next();
                 pos
             }
@@ -611,7 +611,7 @@ impl<'a> Parser<'a> {
             }
         };
         while let Some(&(pos, c)) = self.cur.peek() {
-            if c.is_xid_continue() {
+            if is_id_continue(c) {
                 self.cur.next();
             } else {
                 return &self.input[start..pos];

src/libfmt_macros/lib.rs

@@ -4,12 +4,12 @@ name = "rustc_lexer"
 version = "0.1.0"
 edition = "2018"
 
-# Note that this crate purposefully does not depend on other rustc crates
-[dependencies]
-unicode-xid = { version = "0.1.0", optional = true }
-
 # Note: do not remove this blank `[lib]` section.
 # This will be used when publishing this crate as `rustc-ap-rustc_lexer`.
 [lib]
 doctest = false
 name = "rustc_lexer"
+
+# Note that this crate purposefully does not depend on other rustc crates
+[dependencies]
+unicode-xid = "0.2.0"

src/librustc_lexer/Cargo.toml

@@ -1,6 +1,5 @@
-// We want to be able to build this crate with a stable compiler, so feature
-// flags should be optional.
-#![cfg_attr(not(feature = "unicode-xid"), feature(unicode_internals))]
+// We want to be able to build this crate with a stable compiler, so no
+// `#![feature]` attributes should be added.
 
 mod cursor;
 pub mod unescape;
@@ -507,54 +506,52 @@ impl Cursor<'_> {
 }
 
 pub mod character_properties {
-    // this is Pattern_White_Space
-    #[cfg(feature = "unicode-xid")]
+    // See [UAX #31](http://unicode.org/reports/tr31) for definitions of these
+    // classes.
+
+    // This is Pattern_White_Space.
+    //
+    // Note that this set is stable (ie, it doesn't change with different
+    // Unicode versions), so it's ok to just hard-code the values.
     pub fn is_whitespace(c: char) -> bool {
         match c {
-            '\u{0009}' | '\u{000A}' | '\u{000B}' | '\u{000C}' | '\u{000D}' | '\u{0020}'
-            | '\u{0085}' | '\u{200E}' | '\u{200F}' | '\u{2028}' | '\u{2029}' => true,
+            // Usual ASCII suspects
+            | '\u{0009}' // \t
+            | '\u{000A}' // \n
+            | '\u{000B}' // vertical tab
+            | '\u{000C}' // form feed
+            | '\u{000D}' // \r
+            | '\u{0020}' // space
+
+            // NEXT LINE from latin1
+            | '\u{0085}'
+
+            // Bidi markers
+            | '\u{200E}' // LEFT-TO-RIGHT MARK
+            | '\u{200F}' // RIGHT-TO-LEFT MARK
+
+            // Dedicated whitespace characters from Unicode
+            | '\u{2028}' // LINE SEPARATOR
+            | '\u{2029}' // PARAGRAPH SEPARATOR
+              => true,
             _ => false,
         }
     }
 
-    #[cfg(not(feature = "unicode-xid"))]
-    pub fn is_whitespace(c: char) -> bool {
-        core::unicode::property::Pattern_White_Space(c)
-    }
-
-    // this is XID_Start OR '_' (which formally is not a XID_Start)
-    #[cfg(feature = "unicode-xid")]
+    // This is XID_Start OR '_' (which formally is not a XID_Start).
     pub fn is_id_start(c: char) -> bool {
         ('a' <= c && c <= 'z')
             || ('A' <= c && c <= 'Z')
             || c == '_'
             || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_start(c))
     }
 
-    #[cfg(not(feature = "unicode-xid"))]
-    pub fn is_id_start(c: char) -> bool {
-        ('a' <= c && c <= 'z')
-            || ('A' <= c && c <= 'Z')
-            || c == '_'
-            || (c > '\x7f' && c.is_xid_start())
-    }
-
-    // this is XID_Continue
-    #[cfg(feature = "unicode-xid")]
+    // This is XID_Continue.
     pub fn is_id_continue(c: char) -> bool {
         ('a' <= c && c <= 'z')
             || ('A' <= c && c <= 'Z')
             || ('0' <= c && c <= '9')
             || c == '_'
             || (c > '\x7f' && unicode_xid::UnicodeXID::is_xid_continue(c))
     }
-
-    #[cfg(not(feature = "unicode-xid"))]
-    pub fn is_id_continue(c: char) -> bool {
-        ('a' <= c && c <= 'z')
-            || ('A' <= c && c <= 'Z')
-            || ('0' <= c && c <= '9')
-            || c == '_'
-            || (c > '\x7f' && c.is_xid_continue())
-    }
 }

src/librustc_lexer/src/lib.rs

@@ -20,6 +20,7 @@ rustc = { path = "../librustc" }
 rustc_target = { path = "../librustc_target" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 rustc_errors = { path = "../librustc_errors" }
+rustc_lexer = { path = "../librustc_lexer" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }