diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index e7f692144ff05..c4f062405bb5c 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -907,8 +907,6 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
     module: CachedModuleCodegen,
     module_config: &ModuleConfig,
 ) -> WorkItemResult<B> {
-    assert!(module_config.emit_obj != EmitObj::None);
-
     let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
 
     let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
@@ -928,12 +926,6 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
         }
     };
 
-    let object = load_from_incr_comp_dir(
-        cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)),
-        module.source.saved_files.get("o").unwrap_or_else(|| {
-            cgcx.create_dcx().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
-        }),
-    );
     let dwarf_object =
         module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
             let dwarf_obj_out = cgcx
@@ -955,9 +947,14 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
         }
     };
 
+    let should_emit_obj = module_config.emit_obj != EmitObj::None;
     let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
     let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
     let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
+    let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
+    if should_emit_obj && object.is_none() {
+        cgcx.create_dcx().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
+    }
 
     WorkItemResult::Finished(CompiledModule {
         name: module.name,
diff --git a/tests/run-make/artifact-incr-cache-no-obj/lib.rs b/tests/run-make/artifact-incr-cache-no-obj/lib.rs
new file mode 100644
index 0000000000000..fa4048594e369
--- /dev/null
+++ b/tests/run-make/artifact-incr-cache-no-obj/lib.rs
@@ -0,0 +1,6 @@
+#![crate_name = "foo"]
+
+#[inline(never)]
+pub fn add(a: u32, b: u32) -> u32 {
+    a + b
+}
diff --git a/tests/run-make/artifact-incr-cache-no-obj/rmake.rs b/tests/run-make/artifact-incr-cache-no-obj/rmake.rs
new file mode 100644
index 0000000000000..de55de2a1ee20
--- /dev/null
+++ b/tests/run-make/artifact-incr-cache-no-obj/rmake.rs
@@ -0,0 +1,23 @@
+// emitting an object file is not necessary if user didn't ask for one
+//
+// This test is similar to run-make/artifact-incr-cache but it doesn't
+// require to emit an object file
+//
+// Fixes: rust-lang/rust#123234
+
+extern crate run_make_support;
+
+use run_make_support::{rustc, tmp_dir};
+
+fn main() {
+    let inc_dir = tmp_dir();
+
+    for _ in 0..=1 {
+        rustc()
+            .input("lib.rs")
+            .crate_type("lib")
+            .emit("asm,dep-info,link,mir,llvm-ir,llvm-bc")
+            .incremental(&inc_dir)
+            .run();
+    }
+}