clang linker on MacOS doesn't support -warn_commons for target linux. (IDFGH-12129)

[morgana2313]

The clang linker on MacOs (target=linux) doesn't support the -warn_commons option:

hello_world $ idf.py --preview set-target linux
hello_world $ idf.py build
ld: unknown options: -warn_commons

If I remove this option it builds without a hitch:

esp-idf/CMakeLists.txt

Line 219 in 4d90eed

list(APPEND link_options "-Wl,-warn_commons")

-    list(APPEND link_options "-Wl,-warn_commons")
+    list(APPEND link_options "-Wl")

Details

Executing action: all (aliases: build)
Running ninja in directory /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build
Executing "ninja all"...
[3/3] Linking C executable hello_world.elfFAILED: hello_world.elf
: && /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc -arch arm64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.2.sdk -mmacosx-version-min=14.1 -Wl,-search_paths_first -Wl,-headerpad_max_install_names -fno-rtti -fno-lto -Wl,-dead_strip -Wl,-warn_commons CMakeFiles/hello_world.elf.dir/project_elf_src_linux.c.o -o hello_world.elf  esp-idf/linux/liblinux.a  esp-idf/esp_system/libesp_system.a  esp-idf/esp_common/libesp_common.a  esp-idf/esp_rom/libesp_rom.a  esp-idf/log/liblog.a  esp-idf/heap/libheap.a  esp-idf/esp_hw_support/libesp_hw_support.a  esp-idf/freertos/libfreertos.a  esp-idf/main/libmain.a  esp-idf/linux/liblinux.a  esp-idf/esp_system/libesp_system.a  esp-idf/esp_common/libesp_common.a  esp-idf/esp_rom/libesp_rom.a  esp-idf/log/liblog.a  esp-idf/heap/libheap.a  esp-idf/esp_hw_support/libesp_hw_support.a  esp-idf/freertos/libfreertos.a  esp-idf/linux/liblinux.a  esp-idf/esp_system/libesp_system.a  esp-idf/esp_common/libesp_common.a  esp-idf/esp_rom/libesp_rom.a  esp-idf/log/liblog.a  esp-idf/heap/libheap.a  esp-idf/esp_hw_support/libesp_hw_support.a  esp-idf/freertos/libfreertos.a  esp-idf/linux/liblinux.a  esp-idf/esp_system/libesp_system.a  esp-idf/esp_common/libesp_common.a  esp-idf/esp_rom/libesp_rom.a  esp-idf/log/liblog.a  esp-idf/heap/libheap.a  esp-idf/esp_hw_support/libesp_hw_support.a  esp-idf/freertos/libfreertos.a  esp-idf/linux/liblinux.a  esp-idf/esp_system/libesp_system.a  esp-idf/esp_common/libesp_common.a  esp-idf/esp_rom/libesp_rom.a  esp-idf/log/liblog.a  esp-idf/heap/libheap.a  esp-idf/esp_hw_support/libesp_hw_support.a  esp-idf/freertos/libfreertos.a  -lpthread && :
ld: unknown options: -warn_commons
clang: error: linker command failed with exit code 1 (use -v to see invocation)
ninja: build stopped: subcommand failed.
ninja failed with exit code 1, output of the command is in the /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build/log/idf_py_stderr_output_94370 and /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build/log/idf_py_stdout_output_94370

I've minimalised main/hello_world_main.c to:

#include <stdio.h>
void app_main(void)
{
    printf("Hello world!\n");
}

With the above patch it builds and executes ok:

hello_world $ idf.py build
Executing action: all (aliases: build)
Running ninja in directory /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build
Executing "ninja all"...
[0/1] Re-running CMake...-- Component directory /Users/jasper/mp-v5-flexy/components/sn_faust does not contain a CMakeLists.txt file. No component will be added
-- Building ESP-IDF components for target linux
-- Project sdkconfig file /Users/jasper/esp/esp-idf/examples/get-started/hello_world/sdkconfig
Loading defaults file /Users/jasper/esp/esp-idf/examples/get-started/hello_world/sdkconfig.defaults...
Loading defaults file /Users/jasper/mp-v5-flexy/sn-idf/sn-sdkconfig.defaults...
Loading defaults file /Users/jasper/.sdkconfig.defaults...
-- Components: esp_common esp_hw_support esp_rom esp_system freertos hal heap linux log main soc
-- Component paths: /Users/jasper/esp/esp-idf/components/esp_common /Users/jasper/esp/esp-idf/components/esp_hw_support /Users/jasper/esp/esp-idf/components/esp_rom /Users/jasper/esp/esp-idf/components/esp_system /Users/jasper/esp/esp-idf/components/freertos /Users/jasper/esp/esp-idf/components/hal /Users/jasper/esp/esp-idf/components/heap /Users/jasper/esp/esp-idf/components/linux /Users/jasper/esp/esp-idf/components/log /Users/jasper/esp/esp-idf/examples/get-started/hello_world/main /Users/jasper/esp/esp-idf/components/soc
-- Configuring done (1.4s)
-- Generating done (0.0s)
-- Build files have been written to: /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build
[1/1] Linking C executable hello_world.elfld: warning: ignoring duplicate libraries: 'esp-idf/esp_common/libesp_common.a', 'esp-idf/esp_hw_support/libesp_hw_support.a', 'esp-idf/esp_rom/libesp_rom.a', 'esp-idf/esp_system/libesp_system.a', 'esp-idf/freertos/libfreertos.a', 'esp-idf/heap/libheap.a', 'esp-idf/linux/liblinux.a', 'esp-idf/log/liblog.a'
Done

hello_world $ idf.py monitor
Executing action: monitor
Running idf_monitor in directory /Users/jasper/esp/esp-idf/examples/get-started/hello_world
Executing "/Users/jasper/.espressif/python_env/idf5.1_py3.11_env/bin/python /Users/jasper/esp/esp-idf/tools/idf_monitor.py --toolchain-prefix  --target linux /Users/jasper/esp/esp-idf/examples/get-started/hello_world/build/hello_world.elf -m '/Users/jasper/.espressif/python_env/idf5.1_py3.11_env/bin/python' '/Users/jasper/esp/esp-idf/tools/idf.py'"...
--- esp-idf-monitor 1.3.4 on linux ---
--- Quit: Ctrl+] | Menu: Ctrl+T | Help: Ctrl+T followed by Ctrl+H ---
I (2780252600) port: Starting scheduler.
Hello world!

Done

[igrr]

@morgana2313 Could you please mention the version of macOS and the version of ld you have? Seems like this is version-dependent.

[morgana2313]

@morgana2313 Could you please mention the version of macOS and the version of ld you have? Seems like this is version-dependent.

macOS Sonoma 14.1.1 (23B81) on a Apple M1 Pro

$ clang --version
Apple clang version 15.0.0 (clang-1500.1.0.2.5)
Target: arm64-apple-darwin23.1.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin

$ ld -v
@(#)PROGRAM:ld  PROJECT:dyld-1022.1
BUILD 05:27:37 Dec  7 2023
configured to support archs: armv6 armv7 armv7s arm64 arm64e arm64_32 i386 x86_64 x86_64h
will use ld-classic for: armv6 armv7 armv7s arm64_32 i386 armv6m armv7k armv7m armv7em
LTO support using: LLVM version 15.0.0 (static support for 29, runtime is 29)
TAPI support using: Apple TAPI version 15.0.0 (tapi-1500.0.12.8)
Library search paths:
Framework search paths:

[morgana2313]

I have a similar problem when compiling esp-idf/examples/system/unit_test/test:

ld: unknown options: --whole-archive --no-whole-archive

Fix:

@@ -598,13 +601,23 @@ macro(project project_name)
     endif()

     if(test_components)
-        target_link_libraries(${project_elf} PRIVATE "-Wl,--whole-archive")
+        if(CONFIG_IDF_TARGET_LINUX AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+            set(whole_archive "-Wl,-force_load")
+            set(no_whole_archive "")
+        else()
+            set(whole_archive "-Wl,--whole-archive")
+            set(no_whole_archive "-Wl,--no-whole-archive")
+        endif()
+
+        target_link_libraries(${project_elf} PRIVATE ${whole_archive})
         foreach(test_component ${test_components})
             if(TARGET ${test_component})
                 target_link_libraries(${project_elf} PRIVATE ${test_component})
             endif()
         endforeach()
-        target_link_libraries(${project_elf} PRIVATE "-Wl,--no-whole-archive")
+        target_link_libraries(${project_elf} PRIVATE ${no_whole_archive})
     endif()

     idf_build_get_property(build_components BUILD_COMPONENT_ALIASES)

[igrr]

I can confirm the issue occurs with recent enough /usr/bin/ld. Interestingly, man ld still mentions the -warn_commons option, and ld64 source code still handles it: https://github.com/apple-oss-distributions/ld64/blob/main/src/ld/Options.cpp#L3495. I haven't yet found an explanation why the option isn't working in practice.

[wilmer-suarez-simplisafe]

Is there a workaround for this that doesn't require changing the ESP-IDF source code?

[sudeep-mohanty]

The newer version of ld now works with the -warn_commons flag. We will add a condition check when enabling it 

▶ ld -v
@(#)PROGRAM:ld PROJECT:ld-1053.12
BUILD 15:45:29 Feb 3 2024