Make try_mark_previous_green lock-free.

Author: cjgillot

compiler/rustc_data_structures/src/sync.rs

@@ -40,7 +40,7 @@ cfg_if! {
             }
         }
 
-        use std::ops::Add;
+        use std::ops::{Add, BitOr};
         use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe};
 
         /// This is a single threaded variant of AtomicCell provided by crossbeam.
@@ -147,6 +147,15 @@ cfg_if! {
             }
         }
 
+        impl<T: BitOr<Output=T> + Copy> Atomic<T> {
+            #[inline]
+            pub fn fetch_or(&self, val: T, _: Ordering) -> T {
+                let old = self.0.get();
+                self.0.set(old | val);
+                old
+            }
+        }
+
         pub type AtomicUsize = Atomic<usize>;
         pub type AtomicBool = Atomic<bool>;
         pub type AtomicU32 = Atomic<u32>;

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -46,6 +46,10 @@ impl std::convert::From<DepNodeIndex> for QueryInvocationId {
 }
 
 struct DepGraphData<K: DepKind> {
+    /// The dep-graph from the previous compilation session. It contains all
+    /// nodes and edges as well as all fingerprints of nodes that have them.
+    serialized: SerializedDepGraph<K>,
+
     /// The dep-graph from the compilation session. It contains all
     /// nodes and edges as well as all fingerprints of nodes that have them.
     graph: RwLock<CurrentDepGraph<K>>,
@@ -118,9 +122,11 @@ impl<K: DepKind> DepGraph<K> {
             Err(_) => None,
         };
 
+        let graph = RwLock::new(CurrentDepGraph::new(&prev_graph));
         DepGraph {
             data: Some(Lrc::new(DepGraphData {
-                graph: RwLock::new(CurrentDepGraph::new(prev_graph)),
+                serialized: prev_graph,
+                graph,
                 emitting_diagnostics: Default::default(),
                 previous_work_products: prev_work_products,
                 dep_node_debug: Default::default(),
@@ -148,7 +154,7 @@ impl<K: DepKind> DepGraph<K> {
 
     pub fn query(&self) -> DepGraphQuery<K> {
         let data = self.data.as_ref().unwrap();
-        data.graph.read().query()
+        data.graph.read().query(&data.serialized)
     }
 
     pub fn assert_ignored(&self) {
@@ -239,6 +245,7 @@ impl<K: DepKind> DepGraph<K> {
 
             // Intern the new `DepNode`.
             let dep_node_index = data.graph.write().intern_task_node(
+                &data.serialized,
                 key,
                 &edges[..],
                 current_fingerprint,
@@ -284,8 +291,11 @@ impl<K: DepKind> DepGraph<K> {
                 hash: data.anon_id_seed.combine(hasher.finish()).into(),
             };
 
-            let dep_node_index =
-                data.graph.write().intern_anon_node(target_dep_node, &task_deps.reads[..]);
+            let dep_node_index = data.graph.write().intern_anon_node(
+                &data.serialized,
+                target_dep_node,
+                &task_deps.reads[..],
+            );
 
             (result, dep_node_index)
         } else {
@@ -334,7 +344,10 @@ impl<K: DepKind> DepGraph<K> {
                         #[cfg(debug_assertions)]
                         if let Some(target) = task_deps.node {
                             if let Some(ref forbidden_edge) = _data.forbidden_edge {
-                                let src = self.dep_node_of(dep_node_index);
+                                let src = _data
+                                    .graph
+                                    .read()
+                                    .index_to_node(&_data.serialized, dep_node_index);
                                 if forbidden_edge.test(&src, &target) {
                                     panic!("forbidden edge {:?} -> {:?} created", src, target)
                                 }
@@ -352,24 +365,18 @@ impl<K: DepKind> DepGraph<K> {
     #[inline]
     fn dep_node_index_of_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
         let data = self.data.as_ref()?;
-        data.graph.read().dep_node_index_of_opt(dep_node)
+        data.graph.read().dep_node_index_of_opt(&data.serialized, dep_node)
     }
 
     #[inline]
     pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
         self.dep_node_index_of_opt(dep_node).is_some()
     }
 
-    #[inline]
-    fn dep_node_of(&self, dep_node_index: DepNodeIndex) -> DepNode<K> {
-        let data = self.data.as_ref().unwrap();
-        data.graph.read().dep_node_of(dep_node_index)
-    }
-
     #[inline]
     pub(crate) fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
         let data = self.data.as_ref().unwrap();
-        data.graph.read().fingerprint_of(dep_node_index)
+        data.graph.read().fingerprint_of(&data.serialized, dep_node_index)
     }
 
     /// Checks whether a previous work product exists for `v` and, if
@@ -405,8 +412,8 @@ impl<K: DepKind> DepGraph<K> {
     pub fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
         if let Some(ref data) = self.data {
             let graph = data.graph.read();
-            if let Some(prev_index) = graph.node_to_index_opt(dep_node) {
-                return graph.color(prev_index);
+            if let Some(prev_index) = graph.node_to_index_opt(&data.serialized, dep_node) {
+                return data.serialized.color(prev_index);
             } else {
                 // This is a node that did not exist in the previous compilation
                 // session, so we consider it to be red.
@@ -444,8 +451,8 @@ impl<K: DepKind> DepGraph<K> {
         let data = self.data.as_ref()?;
 
         // Return None if the dep node didn't exist in the previous session
-        let prev_index = data.graph.read().node_to_index_opt(dep_node)?;
-        let prev_deps = data.graph.read().color_or_edges(prev_index);
+        let prev_index = data.graph.read().node_to_index_opt(&data.serialized, dep_node)?;
+        let prev_deps = data.serialized.color_or_edges(prev_index);
         let prev_deps = match prev_deps {
             Err(prev_deps) => prev_deps,
             Ok(DepNodeColor::Green) => return Some((prev_index, prev_index.rejuvenate())),
@@ -468,7 +475,7 @@ impl<K: DepKind> DepGraph<K> {
         parent_dep_node_index: SerializedDepNodeIndex,
         dep_node: &DepNode<K>,
     ) -> Option<()> {
-        let dep_dep_node_color = data.graph.read().color_or_edges(parent_dep_node_index);
+        let dep_dep_node_color = data.serialized.color_or_edges(parent_dep_node_index);
         let prev_deps = match dep_dep_node_color {
             Ok(DepNodeColor::Green) => {
                 // This dependency has been marked as green before, we are
@@ -477,7 +484,7 @@ impl<K: DepKind> DepGraph<K> {
                 debug!(
                     "try_mark_parent_green({:?}) --- found dependency {:?} to be immediately green",
                     dep_node,
-                    data.graph.read().index_to_node(parent_dep_node_index)
+                    data.serialized.index_to_node(parent_dep_node_index)
                 );
                 return Some(());
             }
@@ -489,7 +496,7 @@ impl<K: DepKind> DepGraph<K> {
                 debug!(
                     "try_mark_parent_green({:?}) - END - dependency {:?} was immediately red",
                     dep_node,
-                    data.graph.read().index_to_node(parent_dep_node_index)
+                    data.serialized.index_to_node(parent_dep_node_index)
                 );
                 return None;
             }
@@ -503,12 +510,12 @@ impl<K: DepKind> DepGraph<K> {
                                  is unknown, trying to mark it green",
             dep_node,
             {
-                let dep_dep_node = data.graph.read().index_to_node(parent_dep_node_index);
+                let dep_dep_node = data.serialized.index_to_node(parent_dep_node_index);
                 (dep_dep_node, dep_dep_node.hash)
             }
         );
 
-        let dep_dep_node = &data.graph.read().index_to_node(parent_dep_node_index);
+        let dep_dep_node = &data.serialized.index_to_node(parent_dep_node_index);
         let node_index =
             self.try_mark_previous_green(tcx, data, parent_dep_node_index, prev_deps, dep_dep_node);
         if node_index.is_some() {
@@ -533,7 +540,7 @@ impl<K: DepKind> DepGraph<K> {
             return None;
         }
 
-        let dep_dep_node_color = data.graph.read().color(parent_dep_node_index);
+        let dep_dep_node_color = data.serialized.color(parent_dep_node_index);
 
         match dep_dep_node_color {
             Some(DepNodeColor::Green) => {
@@ -592,15 +599,15 @@ impl<K: DepKind> DepGraph<K> {
 
         // We never try to mark eval_always nodes as green
         debug_assert!(!dep_node.kind.is_eval_always());
-        debug_assert_eq!(data.graph.read().index_to_node(prev_dep_node_index), *dep_node);
+        debug_assert_eq!(data.serialized.index_to_node(prev_dep_node_index), *dep_node);
 
         for &dep_dep_node_index in prev_deps {
             self.try_mark_parent_green(tcx, data, dep_dep_node_index, dep_node)?
         }
 
         #[cfg(not(parallel_compiler))]
         debug_assert_eq!(
-            data.graph.read().color(prev_dep_node_index),
+            data.serialized.color(prev_dep_node_index),
             None,
             "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
                       insertion for {:?}",
@@ -616,7 +623,7 @@ impl<K: DepKind> DepGraph<K> {
         let dep_node_index = {
             // We allocating an entry for the node in the current dependency graph and
             // adding all the appropriate edges imported from the previous graph
-            data.graph.write().intern_dark_green_node(prev_dep_node_index)
+            data.serialized.intern_dark_green_node(prev_dep_node_index)
         };
 
         // ... and emitting any stored diagnostic.
@@ -672,11 +679,10 @@ impl<K: DepKind> DepGraph<K> {
         let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion");
 
         let data = self.data.as_ref().unwrap();
-        let graph = data.graph.read();
-        for prev_index in graph.serialized_indices() {
-            match graph.color(prev_index) {
+        for prev_index in data.serialized.serialized_indices() {
+            match data.serialized.color(prev_index) {
                 Some(DepNodeColor::Green) => {
-                    let dep_node = data.graph.read().index_to_node(prev_index);
+                    let dep_node = data.serialized.index_to_node(prev_index);
                     debug!("PROMOTE {:?} {:?}", prev_index, dep_node);
                     qcx.try_load_from_on_disk_cache(&dep_node);
                 }
@@ -692,11 +698,10 @@ impl<K: DepKind> DepGraph<K> {
     // Register reused dep nodes (i.e. nodes we've marked red or green) with the context.
     pub fn register_reused_dep_nodes<Ctxt: DepContext<DepKind = K>>(&self, tcx: Ctxt) {
         let data = self.data.as_ref().unwrap();
-        let graph = data.graph.read();
-        for prev_index in graph.serialized_indices() {
-            match graph.color(prev_index) {
+        for prev_index in data.serialized.serialized_indices() {
+            match data.serialized.color(prev_index) {
                 Some(_) => {
-                    let dep_node = data.graph.read().index_to_node(prev_index);
+                    let dep_node = data.serialized.index_to_node(prev_index);
                     tcx.register_reused_dep_node(&dep_node);
                 }
                 None => {}
@@ -717,17 +722,17 @@ impl<K: DepKind> DepGraph<K> {
 
         let mut stats: FxHashMap<_, Stat<K>> = FxHashMap::with_hasher(Default::default());
 
-        for index in prev.live_indices() {
-            let kind = prev.dep_node_of(index).kind;
-            let edge_count = prev.edge_targets_from(index).len();
+        for index in prev.live_indices(&data.serialized) {
+            let kind = prev.index_to_node(&data.serialized, index).kind;
+            let edge_count = prev.edge_targets_from(&data.serialized, index).len();
 
             let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
             stat.node_counter += 1;
             stat.edge_counter += edge_count as u64;
         }
 
-        let total_node_count = prev.node_count();
-        let total_edge_count = prev.edge_count();
+        let total_node_count = prev.node_count(&data.serialized);
+        let total_edge_count = prev.edge_count(&data.serialized);
 
         // Drop the lock guard.
         std::mem::drop(prev);
@@ -792,14 +797,19 @@ impl<K: DepKind> DepGraph<K> {
     }
 
     pub fn compression_map(&self) -> IndexVec<DepNodeIndex, Option<SerializedDepNodeIndex>> {
-        self.data.as_ref().unwrap().graph.read().compression_map()
+        let data = self.data.as_ref().unwrap();
+        data.graph.read().compression_map(&data.serialized)
     }
 
     pub fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error>
     where
         K: Encodable<E>,
     {
-        if let Some(data) = &self.data { data.graph.read().encode(encoder) } else { Ok(()) }
+        if let Some(data) = &self.data {
+            data.graph.read().encode(&data.serialized, encoder)
+        } else {
+            Ok(())
+        }
     }
 }