Translate CRLF -> LF in raw (byte) strings

Author: Xanewok

src/libsyntax/parse/lexer/mod.rs

@@ -1346,7 +1346,7 @@ impl<'a> StringReader<'a> {
 
     fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) {
         self.with_str_from_to(content_start, content_end, |lit: &str| {
-            unescape::unescape_raw_str(lit, &mut |range, c| {
+            unescape::unescape_raw_str(lit, unescape::Mode::Str, &mut |range, c| {
                 if let Err(err) = c {
                     emit_unescape_error(
                         &self.sess.span_diagnostic,
@@ -1363,7 +1363,7 @@ impl<'a> StringReader<'a> {
 
     fn validate_raw_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) {
         self.with_str_from_to(content_start, content_end, |lit: &str| {
-            unescape::unescape_raw_byte_str(lit, &mut |range, c| {
+            unescape::unescape_raw_str(lit, unescape::Mode::ByteStr, &mut |range, c| {
                 if let Err(err) = c {
                     emit_unescape_error(
                         &self.sess.span_diagnostic,

src/libsyntax/parse/literal.rs

@@ -4,7 +4,7 @@ use crate::ast::{self, Lit, LitKind};
 use crate::parse::parser::Parser;
 use crate::parse::PResult;
 use crate::parse::token::{self, Token, TokenKind};
-use crate::parse::unescape::{unescape_str, unescape_byte_str, unescape_raw_str};
+use crate::parse::unescape::{self, unescape_str, unescape_byte_str, unescape_raw_str};
 use crate::parse::unescape::{unescape_char, unescape_byte};
 use crate::print::pprust;
 use crate::symbol::{kw, sym, Symbol};
@@ -144,7 +144,7 @@ impl LitKind {
                 let symbol = if s.contains('\r') {
                     let mut buf = String::with_capacity(s.len());
                     let mut error = Ok(());
-                    unescape_raw_str(&s, &mut |_, unescaped_char| {
+                    unescape_raw_str(&s, unescape::Mode::Str, &mut |_, unescaped_char| {
                         match unescaped_char {
                             Ok(c) => buf.push(c),
                             Err(_) => error = Err(LitError::LexerError),

src/libsyntax/parse/unescape.rs

@@ -71,44 +71,28 @@ where
 /// sequence of characters or errors.
 /// NOTE: Raw strings do not perform any explicit character escaping, here we
 /// only translate CRLF to LF and produce errors on bare CR.
-pub(crate) fn unescape_raw_str<F>(literal_text: &str, callback: &mut F)
+pub(crate) fn unescape_raw_str<F>(literal_text: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     let mut byte_offset: usize = 0;
 
     let mut chars = literal_text.chars().peekable();
     while let Some(curr) = chars.next() {
-        let result = match (curr, chars.peek()) {
-            ('\r', Some('\n')) => Ok(curr),
-            ('\r', _) => Err(EscapeError::BareCarriageReturn),
-            _ => Ok(curr),
+        let (result, scanned) = match (curr, chars.peek()) {
+            ('\r', Some('\n')) => {
+                chars.next();
+                (Ok('\n'), [Some('\r'), Some('\n')])
+            },
+            ('\r', _) =>
+                (Err(EscapeError::BareCarriageReturn), [Some('\r'), None]),
+            (c, _) if mode.is_bytes() && c > '\x7F' =>
+                (Err(EscapeError::NonAsciiCharInByteString), [Some(c), None]),
+            (c, _) => (Ok(c), [Some(c), None]),
         };
-        callback(byte_offset..(byte_offset + curr.len_utf8()), result);
-        byte_offset += curr.len_utf8();
-    }
-}
-
-/// Takes a contents of a string literal (without quotes) and produces a
-/// sequence of characters or errors.
-/// NOTE: Raw strings do not perform any explicit character escaping, here we
-/// only translate CRLF to LF and produce errors on bare CR.
-pub(crate) fn unescape_raw_byte_str<F>(literal_text: &str, callback: &mut F)
-where
-    F: FnMut(Range<usize>, Result<char, EscapeError>),
-{
-    let mut byte_offset: usize = 0;
-
-    let mut chars = literal_text.chars().peekable();
-    while let Some(curr) = chars.next() {
-        let result = match (curr, chars.peek()) {
-            ('\r', Some('\n')) => Ok(curr),
-            ('\r', _) => Err(EscapeError::BareCarriageReturn),
-            (c, _) if c > '\x7F' => Err(EscapeError::NonAsciiCharInByteString),
-            _ => Ok(curr),
-        };
-        callback(byte_offset..(byte_offset + curr.len_utf8()), result);
-        byte_offset += curr.len_utf8();
+        let len_utf8: usize = scanned.iter().filter_map(|&x| x).map(char::len_utf8).sum();
+        callback(byte_offset..(byte_offset + len_utf8), result);
+        byte_offset += len_utf8;
     }
 }