Auto merge of rust-lang#115693 - Zoxc:early-dep-graph-load, r=<try>

Load the dep graph before parsing

This changes the dep graph to start loading before parsing. We don't have access to the crate name defined in the source code at this point, so if it's not specified on the command line, incremental compilation will fallback to the file path as a crate identifier.

r? `@oli-obk`

Author: bors

compiler/rustc_incremental/src/persist/fs.rs

@@ -105,15 +105,18 @@
 
 use crate::errors;
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
+use rustc_data_structures::stable_hasher::{Hash64, StableHasher};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::unord::{UnordMap, UnordSet};
 use rustc_data_structures::{base_n, flock};
 use rustc_errors::ErrorGuaranteed;
 use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy};
+use rustc_session::config::{CrateType, Input};
 use rustc_session::{Session, StableCrateId};
-use rustc_span::Symbol;
+use rustc_span::symbol;
 
 use std::fs as std_fs;
+use std::hash::Hash;
 use std::io::{self, ErrorKind};
 use std::path::{Path, PathBuf};
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
@@ -202,11 +205,7 @@ pub fn in_incr_comp_dir(incr_comp_session_dir: &Path, file_name: &str) -> PathBu
 /// The garbage collection will take care of it.
 ///
 /// [`rustc_interface::queries::dep_graph`]: ../../rustc_interface/struct.Queries.html#structfield.dep_graph
-pub fn prepare_session_directory(
-    sess: &Session,
-    crate_name: Symbol,
-    stable_crate_id: StableCrateId,
-) -> Result<(), ErrorGuaranteed> {
+pub fn prepare_session_directory(sess: &Session) -> Result<(), ErrorGuaranteed> {
     if sess.opts.incremental.is_none() {
         return Ok(());
     }
@@ -216,7 +215,7 @@ pub fn prepare_session_directory(
     debug!("prepare_session_directory");
 
     // {incr-comp-dir}/{crate-name-and-disambiguator}
-    let crate_dir = crate_path(sess, crate_name, stable_crate_id);
+    let crate_dir = crate_path(sess);
     debug!("crate-dir: {}", crate_dir.display());
     create_dir(sess, &crate_dir, "crate")?;
 
@@ -601,12 +600,39 @@ fn string_to_timestamp(s: &str) -> Result<SystemTime, &'static str> {
     Ok(UNIX_EPOCH + duration)
 }
 
-fn crate_path(sess: &Session, crate_name: Symbol, stable_crate_id: StableCrateId) -> PathBuf {
+fn crate_path(sess: &Session) -> PathBuf {
+    // Use the crate name from the command line if available and valid
+    let crate_name = sess.opts.crate_name.as_ref().and_then(|name| {
+        name.as_str().chars().all(|c| c.is_alphanumeric() || c == '_').then_some(&name[..])
+    });
+
+    // Create a hash for metadata and the rust version
+    let metadata_and_version = StableCrateId::new(
+        symbol::kw::Empty,
+        // JUSTIFICATION: before wrapper fn is available
+        #[allow(rustc::bad_opt_access)]
+        sess.opts.crate_types.contains(&CrateType::Executable),
+        sess.opts.cg.metadata.clone(),
+        sess.cfg_version,
+    );
+
+    // Find a file path to differentiate crates if crate name is missing
+    let canonical_file_path = crate_name.is_none().then(|| match &sess.io.input {
+        Input::File(file) => try_canonicalize(file).ok(),
+        Input::Str { .. } => None,
+    });
+
+    let mut hasher = StableHasher::new();
+    crate_name.hash(&mut hasher);
+    metadata_and_version.hash(&mut hasher);
+    canonical_file_path.hash(&mut hasher);
+    let stable_crate_id: Hash64 = hasher.finish();
+
     let incr_dir = sess.opts.incremental.as_ref().unwrap().clone();
 
     let stable_crate_id = base_n::encode(stable_crate_id.as_u64() as u128, INT_ENCODE_BASE);
 
-    let crate_name = format!("{crate_name}-{stable_crate_id}");
+    let crate_name = format!("{}-{stable_crate_id}", crate_name.unwrap_or("unknown.crate"));
     incr_dir.join(crate_name)
 }
 

compiler/rustc_interface/src/queries.rs

@@ -21,7 +21,6 @@ use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
 use rustc_session::cstore::Untracked;
 use rustc_session::{output::find_crate_name, Session};
 use rustc_span::symbol::sym;
-use rustc_span::Symbol;
 use std::any::Any;
 use std::cell::{RefCell, RefMut};
 use std::sync::Arc;
@@ -85,6 +84,7 @@ pub struct Queries<'tcx> {
     arena: WorkerLocal<Arena<'tcx>>,
     hir_arena: WorkerLocal<rustc_hir::Arena<'tcx>>,
 
+    dep_graph_future: Query<Option<DepGraphFuture>>,
     parse: Query<ast::Crate>,
     pre_configure: Query<(ast::Crate, ast::AttrVec)>,
     // This just points to what's in `gcx_cell`.
@@ -98,6 +98,7 @@ impl<'tcx> Queries<'tcx> {
             gcx_cell: OnceLock::new(),
             arena: WorkerLocal::new(|_| Arena::default()),
             hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
+            dep_graph_future: Default::default(),
             parse: Default::default(),
             pre_configure: Default::default(),
             gcx: Default::default(),
@@ -112,8 +113,13 @@ impl<'tcx> Queries<'tcx> {
     }
 
     pub fn parse(&self) -> Result<QueryResult<'_, ast::Crate>> {
-        self.parse
-            .compute(|| passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit()))
+        self.parse.compute(|| {
+            // Compute the dependency graph (in the background). We want to do this as early as
+            // possible, to give the DepGraph maximum time to load before `dep_graph` is called.
+            self.dep_graph_future()?;
+
+            passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit())
+        })
     }
 
     pub fn pre_configure(&self) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec)>> {
@@ -133,41 +139,41 @@ impl<'tcx> Queries<'tcx> {
         })
     }
 
-    fn dep_graph_future(
-        &self,
-        crate_name: Symbol,
-        stable_crate_id: StableCrateId,
-    ) -> Result<Option<DepGraphFuture>> {
-        let sess = self.session();
-
-        // `load_dep_graph` can only be called after `prepare_session_directory`.
-        rustc_incremental::prepare_session_directory(sess, crate_name, stable_crate_id)?;
-        let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));
-
-        if sess.opts.incremental.is_some() {
-            sess.time("incr_comp_garbage_collect_session_directories", || {
-                if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
-                    warn!(
-                        "Error while trying to garbage collect incremental \
-                         compilation cache directory: {}",
-                        e
-                    );
-                }
-            });
-        }
+    fn dep_graph_future(&self) -> Result<QueryResult<'_, Option<DepGraphFuture>>> {
+        self.dep_graph_future.compute(|| {
+            let sess = self.session();
+
+            // `load_dep_graph` can only be called after `prepare_session_directory`.
+            rustc_incremental::prepare_session_directory(sess)?;
+            let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));
+
+            if sess.opts.incremental.is_some() {
+                sess.time("incr_comp_garbage_collect_session_directories", || {
+                    if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
+                        warn!(
+                            "Error while trying to garbage collect incremental \
+                            compilation cache directory: {}",
+                            e
+                        );
+                    }
+                });
+            }
 
-        Ok(res)
+            Ok(res)
+        })
     }
 
-    fn dep_graph(&self, dep_graph_future: Option<DepGraphFuture>) -> DepGraph {
-        dep_graph_future
+    fn dep_graph(&self) -> Result<DepGraph> {
+        Ok(self
+            .dep_graph_future()?
+            .steal()
             .and_then(|future| {
                 let sess = self.session();
                 let (prev_graph, prev_work_products) =
                     sess.time("blocked_on_dep_graph_loading", || future.open().open(sess));
                 rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products)
             })
-            .unwrap_or_else(DepGraph::new_disabled)
+            .unwrap_or_else(DepGraph::new_disabled))
     }
 
     pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
@@ -185,10 +191,6 @@ impl<'tcx> Queries<'tcx> {
                 sess.cfg_version,
             );
 
-            // Compute the dependency graph (in the background). We want to do this as early as
-            // possible, to give the DepGraph maximum time to load before `dep_graph` is called.
-            let dep_graph_future = self.dep_graph_future(crate_name, stable_crate_id)?;
-
             let lint_store = Lrc::new(passes::create_lint_store(
                 sess,
                 &*self.codegen_backend().metadata_loader(),
@@ -210,7 +212,7 @@ impl<'tcx> Queries<'tcx> {
                 crate_types,
                 stable_crate_id,
                 lint_store,
-                self.dep_graph(dep_graph_future),
+                self.dep_graph()?,
                 untracked,
                 &self.gcx_cell,
                 &self.arena,