strip prefix of temporary file names when it exceeds filesystem name length limit

When doing lto, rustc generates filenames that are concatenating many information.

In the case of this testcase, it is concatenating crate name and rust file name, plus some hash, and the extension.
In some other cases it will concatenate even more information reducing the maximum effective crate name to about 110 chars on linux filesystems where
filename max length is 255

This commit is ensuring that the temporary file names are limited in size, while still reasonabily ensuring the unicity (with hashing of the stripped part)

Author: tardyp

compiler/rustc_session/src/config.rs

@@ -16,10 +16,11 @@ use std::{cmp, fmt, fs, iter};
 
 use externs::{ExternOpt, split_extern_opt};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
-use rustc_data_structures::stable_hasher::{StableOrd, ToStableHashKey};
+use rustc_data_structures::stable_hasher::{StableHasher, StableOrd, ToStableHashKey};
 use rustc_errors::emitter::HumanReadableErrorType;
 use rustc_errors::{ColorConfig, DiagArgValue, DiagCtxtFlags, IntoDiagArg};
 use rustc_feature::UnstableFeatures;
+use rustc_hashes::Hash64;
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_span::edition::{DEFAULT_EDITION, EDITION_NAME_LIST, Edition, LATEST_STABLE_EDITION};
 use rustc_span::source_map::FilePathMapping;
@@ -1198,7 +1199,25 @@ pub struct OutputFilenames {
 pub const RLINK_EXT: &str = "rlink";
 pub const RUST_CGU_EXT: &str = "rcgu";
 pub const DWARF_OBJECT_EXT: &str = "dwo";
+pub const MAX_FILENAME_LENGTH: usize = 143; // ecryptfs limits filenames to 143 bytes see #49914
 
+/// Ensure the filename is not too long, as some filesystems have a limit.
+/// If the filename is too long, hash part of it and append the hash to the filename.
+/// This is a workaround for long crate names generating overly long filenames.
+fn maybe_strip_file_name(mut path: PathBuf) -> PathBuf {
+    if path.file_name().map_or(0, |name| name.len()) > MAX_FILENAME_LENGTH {
+        let filename = path.file_name().unwrap().to_string_lossy();
+        let hash_len = 64 / 4; // Hash64 is 64 bits encoded in hex
+        let stripped_len = filename.len() - MAX_FILENAME_LENGTH + hash_len;
+
+        let mut hasher = StableHasher::new();
+        filename[..stripped_len].hash(&mut hasher);
+        let hash = hasher.finish::<Hash64>();
+
+        path.set_file_name(format!("{:x}-{}", hash, &filename[stripped_len..]));
+    }
+    path
+}
 impl OutputFilenames {
     pub fn new(
         out_directory: PathBuf,
@@ -1291,7 +1310,7 @@ impl OutputFilenames {
         }
 
         let temps_directory = self.temps_directory.as_ref().unwrap_or(&self.out_directory);
-        self.with_directory_and_extension(temps_directory, &extension)
+        maybe_strip_file_name(self.with_directory_and_extension(temps_directory, &extension))
     }
 
     pub fn temp_path_for_diagnostic(&self, ext: &str) -> PathBuf {

tests/run-make/lto-long-filenames/.gitignore

@@ -0,0 +1 @@
+generated*.rs

tests/run-make/lto-long-filenames/main.rs

@@ -0,0 +1,7 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+extern crate generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name;
+
+fn main() {}

tests/run-make/lto-long-filenames/rmake.rs

@@ -0,0 +1,33 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+// A variant of the smoke test to check that link time optimization
+// (LTO) is accepted by the compiler, and that
+// passing its various flags still results in successful compilation, even for very long crate names.
+// See https://github.com/rust-lang/rust/issues/49914
+
+//@ ignore-cross-compile
+
+use std::fs;
+
+use run_make_support::rustc;
+
+// This test make sure we don't get such following error:
+// error: could not write output to generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.9384edb61bfd127c-cgu.0.rcgu.o: File name too long
+// as reported in issue #49914
+fn main() {
+    let lto_flags = ["-Clto", "-Clto=yes", "-Clto=off", "-Clto=thin", "-Clto=fat"];
+    let aux_file = "generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.rs";
+    // The auxiliary file is used to test long crate names.
+    // The file name is intentionally long to test the handling of long filenames.
+    // but we don't commit it to avoid issues with windows paths, which have a limitation for the full path length.
+    // Linux only has a limit for the length of the file name.
+    // and we want to make sure we can compile files with long names.
+    fs::write(aux_file, "#![crate_type = \"rlib\"]\n").expect("Failed to write lib.rs");
+
+    for flag in lto_flags {
+        rustc().input(aux_file).arg(flag).run();
+        rustc().input("main.rs").arg(flag).run();
+    }
+}