find fluent resource from hash slug

Author: chenyukang

Cargo.lock

@@ -2338,10 +2338,11 @@ dependencies = [
 
 [[package]]
 name = "md-5"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
+checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
 dependencies = [
+ "cfg-if",
  "digest",
 ]
 
@@ -3812,6 +3813,7 @@ dependencies = [
  "annotate-snippets",
  "derive_setters",
  "fluent",
+ "md-5",
  "rustc_ast",
  "rustc_ast_pretty",
  "rustc_data_structures",

compiler/rustc_errors/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2021"
 annotate-snippets = "0.9"
 derive_setters = "0.1.6"
 fluent = "0.16.0"
+md-5 = "0.10.6"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }

compiler/rustc_errors/src/translation.rs

@@ -3,6 +3,7 @@ use crate::fluent_bundle::FluentResource;
 use crate::snippet::Style;
 use crate::{DiagnosticArg, DiagnosticMessage, FluentBundle};
 use fluent::FluentBundle as RawFluentBundle;
+use md5::{Digest, Md5};
 use rustc_data_structures::sync::Lrc;
 use rustc_error_messages::FluentArgs;
 use std::borrow::Cow;
@@ -69,23 +70,68 @@ pub trait Translate {
             }
             DiagnosticMessage::FluentRaw(msg) => {
                 // FIXME(yukang): A hack for raw fluent content for new diagnostics proc format
-                let fluent_text = format!("dummy = {}", msg);
-                if let Ok(resource) = FluentResource::try_new(fluent_text) {
-                    let mut bundle = RawFluentBundle::new(vec![langid!("en-US")]);
-                    bundle.add_resource(resource).unwrap();
-                    let mut errors = vec![];
-                    let pattern = bundle.get_message("dummy").unwrap().value().unwrap();
-                    let res = bundle.format_pattern(&pattern, Some(args), &mut errors);
-                    return Ok(Cow::Owned(
-                        res.to_string().replace("\u{2068}", "").replace("\u{2069}", ""),
-                    ));
-                }
+                // calculate the `slug` from the raw fluent content
+                let mut hasher = Md5::new();
+                hasher.update(msg.to_string());
+                let digest = hasher.finalize();
+                let id = format!("{:x}", digest);
+                let identifier = format!("slug-{}", id[0..8].to_string());
+                let id = Cow::Borrowed(identifier.as_str());
 
-                // If the message is not a valid Fluent resource, just return the original
-                return Ok(Cow::Borrowed(msg));
+                //eprintln!("trying id: {}", identifier);
+                let translate_with_bundle =
+                    |bundle: &'a FluentBundle| -> Result<Cow<'_, str>, TranslateError<'_>> {
+                        let message = bundle
+                            .get_message(&identifier)
+                            .ok_or(TranslateError::message(&id, args))?;
+                        let value = message.value().ok_or(TranslateError::value(&id, args))?;
+                        debug!(?message, ?value);
+
+                        let mut errs = vec![];
+                        let translated = bundle.format_pattern(value, Some(args), &mut errs);
+                        debug!(?translated, ?errs);
+                        if errs.is_empty() {
+                            Ok(translated)
+                        } else {
+                            Err(TranslateError::fluent(&id, args, errs))
+                        }
+                    };
+
+                return {
+                    match self.fluent_bundle().map(|b| translate_with_bundle(b)) {
+                        // The primary bundle was present and translation succeeded
+                        Some(Ok(t)) => {
+                            // eprintln!("translated id OK: {} => {}", identifier, t);
+                            Ok(t)
+                        }
+
+                        // If `translate_with_bundle` returns `Err` with the primary bundle, this is likely
+                        // just that the primary bundle doesn't contain the message being translated, so
+                        // proceed to the fallback bundle.
+                        _ => {
+                            // eprintln!("fallback to en.... for id: {}", identifier);
+                            // fallback to en-US, we don't need fluent bundle for raw fluent content in English
+                            // here we just interpret the variables in the fluent content.
+                            let fluent_text = format!("dummy = {}", msg);
+                            if let Ok(resource) = FluentResource::try_new(fluent_text) {
+                                let mut bundle = RawFluentBundle::new(vec![langid!("en-US")]);
+                                bundle.add_resource(resource).unwrap();
+                                let mut errors = vec![];
+                                let pattern = bundle.get_message("dummy").unwrap().value().unwrap();
+                                let res = bundle.format_pattern(&pattern, Some(args), &mut errors);
+                                Ok(Cow::Owned(
+                                    res.to_string().replace("\u{2068}", "").replace("\u{2069}", ""),
+                                ))
+                            } else {
+                                Ok(Cow::Owned(msg.to_string()))
+                            }
+                        }
+                    }
+                };
             }
             DiagnosticMessage::FluentIdentifier(identifier, attr) => (identifier, attr),
         };
+        // FIXME(yukang): remove this part for fluent resource id after all diagnostics are migrated to Fluent
         let translate_with_bundle =
             |bundle: &'a FluentBundle| -> Result<Cow<'_, str>, TranslateError<'_>> {
                 let message = bundle

tests/run-make/translation/Makefile

@@ -9,16 +9,14 @@ FAKEROOT=$(TMPDIR)/fakeroot
 RUSTC_LOG:=rustc_error_messages
 export RUSTC_TRANSLATION_NO_DEBUG_ASSERT:=1
 
-all: normal missing broken sysroot-invalid sysroot-missing
+all: normal custom missing broken sysroot sysroot-invalid sysroot-missing
 
 # Check that the test works normally, using the built-in fallback bundle.
 normal: test.rs
 	$(RUSTC) $< 2>&1 | $(CGREP) "struct literal body without path"
 
 # Check that a primary bundle can be loaded and will be preferentially used
 # where possible.
-# FIXME(yukang): This test is broken because the compiler doesn't look for the fluent slugs now
-# we need to fix it after we have implemented the new way to find the fluent resources
 custom: test.rs working.ftl
 	$(RUSTC) $< -Ztranslate-additional-ftl=$(CURDIR)/working.ftl 2>&1 | $(CGREP) "this is a test message"
 
@@ -36,8 +34,6 @@ broken: test.rs broken.ftl
 # identifier by making a local copy of the sysroot and adding the custom locale
 # to it.
 
-# FIXME(yukang): This test is broken because the compiler doesn't look for the fluent slugs now
-# we need to fix it after we have implemented the new way to find the fluent resources
 sysroot: test.rs working.ftl
 	rm -rf $(FAKEROOT)
 	mkdir $(FAKEROOT)

tests/run-make/translation/working.ftl

@@ -1,2 +1,2 @@
-parse_struct_literal_body_without_path = this is a test message
-    .suggestion = this is a test suggestion
+slug-687d6246 = this is a test message
+slug-7d40154a = this is a test suggestion