impl watcher

Author: klensy

Cargo.lock

@@ -5367,6 +5367,7 @@ dependencies = [
  "cargo_metadata 0.19.1",
  "fluent-syntax",
  "ignore",
+ "md-5",
  "miropt-test-tools",
  "regex",
  "rustc-hash 2.1.0",

src/tools/tidy/Cargo.toml

@@ -16,6 +16,7 @@ termcolor = "1.1.3"
 rustc-hash = "2.0.0"
 fluent-syntax = "0.11.1"
 similar = "2.5.0"
+md-5 = "0.10"
 
 [[bin]]
 name = "rust-tidy"

src/tools/tidy/src/lib.rs

@@ -93,4 +93,5 @@ pub mod unit_tests;
 pub mod unknown_revision;
 pub mod unstable_book;
 pub mod walk;
+pub mod watcher;
 pub mod x_version;

src/tools/tidy/src/main.rs

@@ -146,6 +146,8 @@ fn main() {
 
         check!(x_version, &root_path, &cargo);
 
+        check!(watcher, &root_path);
+
         let collected = {
             drain_handles(&mut handles);
 

src/tools/tidy/src/watcher.rs

@@ -0,0 +1,188 @@
+//! Checks that text between tags unchanged, emitting warning otherwise,
+//! allowing asserting that code in different places over codebase is in sync.
+//!
+//! This works via hashing text between tags and saving hash in tidy.
+//!
+//! Usage:
+//!
+//! some.rs:
+//! // tidy-ticket-foo
+//! const FOO: usize = 42;
+//! // tidy-ticket-foo
+//!
+//! some.sh:
+//! # tidy-ticket-foo
+//! export FOO=42
+//! # tidy-ticket-foo
+use md5::{Digest, Md5};
+use std::fs;
+use std::path::Path;
+
+#[cfg(test)]
+mod tests;
+
+/// Return hash for source text between 2 tag occurrence,
+/// ignoring lines where tag written
+///
+/// Expecting:
+/// tag is not multiline
+/// source always have at least 2 occurrence of tag (>2 ignored)
+fn span_hash(source: &str, tag: &str, bad: &mut bool) -> Result<String, ()> {
+    let start_idx = match source.find(tag) {
+        Some(idx) => idx,
+        None => return Err(tidy_error!(bad, "tag {} should exist in provided text", tag)),
+    };
+    let end_idx = {
+        let end = match source[start_idx + tag.len()..].find(tag) {
+            // index from source start
+            Some(idx) => start_idx + tag.len() + idx,
+            None => return Err(tidy_error!(bad, "tag end {} should exist in provided text", tag)),
+        };
+        // second line with tag can contain some other text before tag, ignore it
+        // by finding position of previous line ending
+        //
+        // FIXME: what if line ending is \r\n? In that case \r will be hashed too
+        let offset = source[start_idx..end].rfind('\n').unwrap();
+        start_idx + offset
+    };
+
+    let mut hasher = Md5::new();
+
+    source[start_idx..end_idx]
+        .lines()
+        // skip first line with tag
+        .skip(1)
+        // hash next lines, ignoring end trailing whitespaces
+        .for_each(|line| {
+            let trimmed = line.trim_end();
+            hasher.update(trimmed);
+        });
+    Ok(format!("{:x}", hasher.finalize()))
+}
+
+fn check_entry(entry: &ListEntry<'_>, group_idx: usize, bad: &mut bool, root_path: &Path) {
+    let file = fs::read_to_string(root_path.join(Path::new(entry.0)))
+        .unwrap_or_else(|e| panic!("{:?}, path: {}", e, entry.0));
+    let actual_hash = span_hash(&file, entry.2, bad).unwrap();
+    if actual_hash != entry.1 {
+        // Write tidy error description for wather only once.
+        // Will not work if there was previous errors of other types.
+        if *bad == false {
+            tidy_error!(
+                bad,
+                "The code blocks tagged with tidy watcher has changed.\n\
+                 It's likely that code blocks with the following tags need to be changed too. Check src/tools/tidy/src/watcher.rs, find tag/hash in TIDY_WATCH_LIST list \
+                and verify that sources for provided group of tags in sync. Once that done, run tidy again and update hashes in TIDY_WATCH_LIST with provided actual hashes."
+            )
+        }
+        tidy_error!(
+            bad,
+            "hash for tag `{}` in path `{}` mismatch:\n  actual: `{}`, expected: `{}`\n  \
+            Verify that tags `{:?}` in sync.",
+            entry.2,
+            entry.0,
+            actual_hash,
+            entry.1,
+            TIDY_WATCH_LIST[group_idx].iter().map(|e| e.2).collect::<Vec<&str>>()
+        );
+    }
+}
+
+macro_rules! add_group {
+    ($($entry:expr),*) => {
+        &[$($entry),*]
+    };
+}
+
+/// (path, hash, tag)
+type ListEntry<'a> = (&'a str, &'a str, &'a str);
+
+/// List of tags to watch, along with paths and hashes
+#[rustfmt::skip]
+const TIDY_WATCH_LIST: &[&[ListEntry<'_>]] = &[
+    // sync perf commit across dockerfile and opt-dist
+    add_group!(
+        ("src/tools/opt-dist/src/main.rs", "f99844fda86216628167cd3391844ed1", "tidy-ticket-perf-commit"),
+        ("src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile", "fa9a88e552735e9f93934501e9c8d87a", "tidy-ticket-perf-commit")
+    ),
+
+    add_group!(
+        ("compiler/rustc_ast/src/token.rs", "0f6ba78f1007a398f01cc6217838cadc", "tidy-ticket-ast-from_token"),
+        ("compiler/rustc_ast/src/token.rs", "9a78008a2377486eadf19d67ee4fdce2", "tidy-ticket-ast-can_begin_literal_maybe_minus"),
+        ("compiler/rustc_parse/src/parser/expr.rs", "500240cdc80690209060fdce10ce065a", "tidy-ticket-rustc_parse-can_begin_literal_maybe_minus")
+    ),
+
+    add_group!(
+        ("compiler/rustc_builtin_macros/src/assert/context.rs", "de6cc928308947a05a373a355e036f68", "tidy-ticket-all-expr-kinds"),
+        ("tests/ui/macros/rfc-2011-nicer-assert-messages/all-expr-kinds.rs", "78ce54cc25baeac3ae07c876db25180c", "tidy-ticket-all-expr-kinds")
+    ),
+
+    add_group!(
+        ("compiler/rustc_const_eval/src/interpret/validity.rs", "91c69e391741f64b7624e1bda4b31bc3", "tidy-ticket-try_visit_primitive"),
+        ("compiler/rustc_const_eval/src/interpret/validity.rs", "763a70aa04279a7b1cd184b75f79751d", "tidy-ticket-visit_value")
+    ),
+
+    // sync self-profile-events help mesage with actual list of events
+    add_group!(
+        ("compiler/rustc_data_structures/src/profiling.rs", "881e7899c7d6904af1bc000594ee0418", "tidy-ticket-self-profile-events"),
+        ("compiler/rustc_session/src/options.rs", "012ee5a3b61ee1377744e5c6913fa00a", "tidy-ticket-self-profile-events")
+    ),
+
+    add_group!(
+        ("compiler/rustc_errors/src/json.rs", "5907da5c0476785fe2aae4d0d62f7171", "tidy-ticket-UnusedExterns"),
+        ("src/librustdoc/doctest.rs", "b5bb5128abb4a2dbb47bb1a1a083ba9b", "tidy-ticket-UnusedExterns")
+    ),
+
+    add_group!(
+        ("compiler/rustc_middle/src/ty/util.rs", "cae64b1bc854e7ee81894212facb5bfa", "tidy-ticket-static_ptr_ty"),
+        ("compiler/rustc_middle/src/ty/util.rs", "6f5ead08474b4d3e358db5d3c7aef970", "tidy-ticket-thread_local_ptr_ty")
+    ),
+
+    add_group!(
+        ("compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs", "c17706947fc814aa5648972a5b3dc143", "tidy-ticket-arity"),
+        ("compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs", "7ce77b84c142c22530b047703ef209f0", "tidy-ticket-wildcards")
+    ),
+
+    add_group!(
+        ("compiler/rustc_monomorphize/src/partitioning.rs", "f4f33e9c14f4e0c3a20b5240ae36a7c8", "tidy-ticket-short_description"),
+        ("compiler/rustc_codegen_ssa/src/back/write.rs", "5286f7f76fcf564c98d7a8eaeec39b18", "tidy-ticket-short_description")
+    ),
+
+    add_group!(
+        ("compiler/rustc_session/src/config/sigpipe.rs", "8d765a5c613d931852c0f59ed1997dcd", "tidy-ticket-sigpipe"),
+        ("library/std/src/sys/unix/mod.rs", "2cdc37081831cdcf44f3331efbe440af", "tidy-ticket-sigpipe")
+    ),
+
+    add_group!(
+        ("compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs", "6b4ce7c9aa0e799618d53926fb3e9684", "tidy-ticket-extract_tupled_inputs_and_output_from_callable"),
+        ("compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs", "f1085622f189fc5ec2786e4abff67915", "tidy-ticket-assemble_fn_pointer_candidates")
+    ),
+
+    add_group!(
+        ("compiler/rustc_trait_selection/src/solve/eval_ctxt/select.rs", "d0c807d90501f3f63dffc3e7ec046c20", "tidy-ticket-rematch_unsize"),
+        ("compiler/rustc_trait_selection/src/solve/trait_goals.rs", "f1b0ce28128b5d5a5b545af3f3cf55f4", "tidy-ticket-consider_builtin_unsize_candidate")
+    ),
+
+    add_group!(
+        ("compiler/rustc_trait_selection/src/traits/project.rs", "66585f93352fe56a5be6cc5a63bcc756", "tidy-ticket-assemble_candidates_from_impls-UserDefined"),
+        ("compiler/rustc_ty_utils/src/instance.rs", "5ad30b96493636ba3357448f0b0a4d76", "tidy-ticket-resolve_associated_item-UserDefined")
+    ),
+
+    add_group!(
+        ("compiler/rustc_hir_analysis/src/lib.rs", "842e23fb65caf3a96681686131093316", "tidy-ticket-sess-time-item_types_checking"),
+        ("src/librustdoc/core.rs", "85d9dd0cbb94fd521e2d15a8ed38a75f", "tidy-ticket-sess-time-item_types_checking")
+    ),
+
+    add_group!(
+        ("library/core/src/ptr/metadata.rs", "57fc0e05c177c042c9766cc1134ae240", "tidy-ticket-static_assert_expected_bounds_for_metadata"),
+        ("library/core/tests/ptr.rs", "13ecb32e2a0db0998ff94f33a30f5cfd", "tidy-ticket-static_assert_expected_bounds_for_metadata")
+    ),
+];
+
+pub fn check(root_path: &Path, bad: &mut bool) {
+    for (group_idx, group) in TIDY_WATCH_LIST.iter().enumerate() {
+        for entry in group.iter() {
+            check_entry(entry, group_idx, bad, root_path);
+        }
+    }
+}

src/tools/tidy/src/watcher/tests.rs

@@ -0,0 +1,39 @@
+use super::*;
+
+#[test]
+fn test_span_hash_one_line() {
+    let source = "some text\ntidy-tag\ncheckme=42\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("42258eba764c3f94a24de379e5715dc8", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_multiple_lines() {
+    let source = "some text\ntidy-tag\ncheckme=42\nother line\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("49cb23dc2032ceea671ca48092750a1c", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_has_some_text_in_line_with_tag() {
+    let source = "some text\ntidy-tag ignore me\ncheckme=42\nother line\ntidy-tag\n";
+    let tag = "tidy-tag";
+    assert_eq!("49cb23dc2032ceea671ca48092750a1c", span_hash(source, tag, &mut true).unwrap());
+}
+
+#[test]
+fn test_span_hash_has_some_text_in_line_before_second_tag() {
+    let source = r#"
+RUN ./build-clang.sh
+ENV CC=clang CXX=clang++
+# tidy-ticket-perf-commit
+# rustc-perf version from 2023-05-30
+ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1
+# tidy-ticket-perf-commit
+RUN curl -LS -o perf.zip https://github.com/rust-lang/rustc-perf/archive/$PERF_COMMIT.zip && \
+    unzip perf.zip && \
+    mv rustc-perf-$PERF_COMMIT rustc-perf && \
+    rm perf.zip"#;
+    let tag = "tidy-ticket-perf-commit";
+    assert_eq!("76c8d9783e38e25a461355f82fcd7955", span_hash(source, tag, &mut true).unwrap());
+}