Fix osx-dynamic install names for updated dependent shared library ids (#39889)

scripts/cmake/z_vcpkg_fixup_rpath_macho.cmake

@@ -30,6 +30,15 @@ function(z_vcpkg_calculate_corrected_macho_rpath)
     set("${arg_OUT_NEW_RPATH_VAR}" "${new_rpath}" PARENT_SCOPE)
 endfunction()
 
+function(z_vcpkg_regex_escape)
+    cmake_parse_arguments(PARSE_ARGV 0 "arg"
+      ""
+      "STRING;OUT_REGEX_ESCAPED_STRING_VAR"
+      "")
+  string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" regex_escaped "${arg_STRING}")
+  set("${arg_OUT_REGEX_ESCAPED_STRING_VAR}" "${regex_escaped}" PARENT_SCOPE)
+endfunction()
+
 function(z_vcpkg_fixup_macho_rpath_in_dir)
     # We need to iterate through everything because we
     # can't predict where a Mach-O file will be located
@@ -95,6 +104,8 @@ function(z_vcpkg_fixup_macho_rpath_in_dir)
                 continue()
             endif()
 
+            list(APPEND macho_executables_and_shared_libs "${macho_file}")
+
             get_filename_component(macho_file_dir "${macho_file}" DIRECTORY)
             get_filename_component(macho_file_name "${macho_file}" NAME)
 
@@ -106,15 +117,40 @@ function(z_vcpkg_fixup_macho_rpath_in_dir)
             if("${file_type}" STREQUAL "shared")
                 # Set the install name for shared libraries
                 execute_process(
-                    COMMAND "${install_name_tool_cmd}" -id "@rpath/${macho_file_name}" "${macho_file}"
+                    COMMAND "${otool_cmd}" -D "${macho_file}"
+                    OUTPUT_VARIABLE get_id_ov
+                    RESULT_VARIABLE get_id_rv
+                )
+                if(NOT get_id_rv EQUAL 0)
+                    message(FATAL_ERROR "Could not obtain install name id from '${macho_file}'")
+                endif()
+                set(macho_new_id "@rpath/${macho_file_name}")
+                message(STATUS "Setting install name id of '${macho_file}' to '@rpath/${macho_file_name}'")
+                execute_process(
+                    COMMAND "${install_name_tool_cmd}" -id "${macho_new_id}" "${macho_file}"
                     OUTPUT_QUIET
                     ERROR_VARIABLE set_id_error
+                    RESULT_VARIABLE set_id_exit_code
                 )
-                message(STATUS "Set install name id of '${macho_file}' to '@rpath/${macho_file_name}'")
-                if(NOT "${set_id_error}" STREQUAL "")
+                if(NOT "${set_id_error}" STREQUAL "" AND NOT set_id_exit_code EQUAL 0)
                     message(WARNING "Couldn't adjust install name of '${macho_file}': ${set_id_error}")
                     continue()
                 endif()
+
+                # otool -D <macho_file> typically returns lines like:
+
+                # <macho_file>:
+                # <id>
+
+                # But also with ARM64 binaries, it can return:
+                # <macho_file> (architecture arm64):
+                # <id>
+
+                # Either way we need to remove the first line and trim the trailing newline char.
+                string(REGEX REPLACE "[^\n]+:\n" "" get_id_ov "${get_id_ov}")
+                string(REGEX REPLACE "\n.*" "" get_id_ov "${get_id_ov}")
+                list(APPEND adjusted_shared_lib_old_ids "${get_id_ov}")
+                list(APPEND adjusted_shared_lib_new_ids "${macho_new_id}")
             endif()
 
             # List all existing rpaths
@@ -151,9 +187,10 @@ function(z_vcpkg_fixup_macho_rpath_in_dir)
                 COMMAND "${install_name_tool_cmd}" ${rpath_args} "${macho_file}"
                 OUTPUT_QUIET
                 ERROR_VARIABLE set_rpath_error
+                RESULT_VARIABLE set_rpath_exit_code
             )
 
-            if(NOT "${set_rpath_error}" STREQUAL "")
+            if(NOT "${set_rpath_error}" STREQUAL "" AND NOT set_rpath_exit_code EQUAL 0)
                 message(WARNING "Couldn't adjust RPATH of '${macho_file}': ${set_rpath_error}")
                 continue()
             endif()
@@ -161,4 +198,46 @@ function(z_vcpkg_fixup_macho_rpath_in_dir)
             message(STATUS "Adjusted RPATH of '${macho_file}' to '${new_rpath}'")
         endforeach()
     endforeach()
+
+    # Check for dependent libraries in executables and shared libraries that
+    # need adjusting after id change
+    list(LENGTH adjusted_shared_lib_old_ids last_adjusted_index)
+    if(NOT last_adjusted_index EQUAL 0)
+        math(EXPR last_adjusted_index "${last_adjusted_index} - 1")
+        foreach(macho_file IN LISTS macho_executables_and_shared_libs)
+            execute_process(
+                COMMAND "${otool_cmd}" -L "${macho_file}"
+                OUTPUT_VARIABLE get_deps_ov
+                RESULT_VARIABLE get_deps_rv
+            )
+            if(NOT get_deps_rv EQUAL 0)
+                message(FATAL_ERROR "Could not obtain dependencies list from '${macho_file}'")
+            endif()
+            # change adjusted_shared_lib_old_ids[i] -> adjusted_shared_lib_new_ids[i]
+            foreach(i RANGE ${last_adjusted_index})
+                list(GET adjusted_shared_lib_old_ids ${i} adjusted_old_id)
+                z_vcpkg_regex_escape(
+                    STRING "${adjusted_old_id}"
+                    OUT_REGEX_ESCAPED_STRING_VAR regex
+                )
+                if(NOT get_deps_ov MATCHES "[ \t]${regex} ")
+                    continue()
+                endif()
+                list(GET adjusted_shared_lib_new_ids ${i} adjusted_new_id)
+
+                # Replace the old id with the new id
+                execute_process(
+                    COMMAND "${install_name_tool_cmd}" -change "${adjusted_old_id}" "${adjusted_new_id}" "${macho_file}"
+                    OUTPUT_QUIET
+                    ERROR_VARIABLE change_id_error
+                    RESULT_VARIABLE change_id_exit_code
+                )
+                if(NOT "${change_id_error}" STREQUAL "" AND NOT change_id_exit_code EQUAL 0)
+                    message(WARNING "Couldn't adjust dependent shared library install name in '${macho_file}': ${change_id_error}")
+                    continue()
+                endif()
+                message(STATUS "Adjusted dependent shared library install name in '${macho_file}' (From '${adjusted_old_id}' -> To '${adjusted_new_id}')")
+            endforeach()
+        endforeach()
+    endif()
 endfunction()

scripts/test_ports/rpath-macho-test-binaries/portfile.cmake

@@ -0,0 +1,46 @@
+set(VCPKG_POLICY_EMPTY_INCLUDE_FOLDER enabled)
+vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${CURRENT_PORT_DIR}/project"
+    OPTIONS_RELEASE
+        -DTEST_STRING=release
+    OPTIONS_DEBUG
+        -DTEST_STRING=debug
+)
+vcpkg_cmake_install()
+
+function(make_rpath_absolute lib_dir)
+string(REPLACE "/" "_" logname "make_rpath_absolute-${lib_dir}")
+    vcpkg_execute_required_process(
+        COMMAND "install_name_tool" -id ${CURRENT_INSTALLED_DIR}/${lib_dir}/librpath-macho-backend-lib++.dylib ${CURRENT_PACKAGES_DIR}/${lib_dir}/librpath-macho-backend-lib++.dylib
+        WORKING_DIRECTORY "${CURRENT_PACKAGES_DIR}"
+        LOGNAME "${logname}-id"
+    )
+    
+    vcpkg_execute_required_process(
+        COMMAND "install_name_tool" -change @rpath/librpath-macho-backend-lib++.dylib ${CURRENT_INSTALLED_DIR}/${lib_dir}/librpath-macho-backend-lib++.dylib ${CURRENT_PACKAGES_DIR}/${lib_dir}/librpath-macho-test-lib.dylib
+        WORKING_DIRECTORY "${CURRENT_PACKAGES_DIR}"
+        LOGNAME "${logname}-change"
+    )
+endfunction()
+
+if(NOT VCPKG_BUILD_TYPE)
+    vcpkg_copy_tools(TOOL_NAMES rpath-macho-test-tool
+        SEARCH_DIR "${CURRENT_PACKAGES_DIR}/debug/bin"
+        DESTINATION "${CURRENT_PACKAGES_DIR}/debug/tools/${PORT}"
+    )
+    vcpkg_copy_tools(TOOL_NAMES rpath-macho-test-tool
+        SEARCH_DIR "${CURRENT_PACKAGES_DIR}/debug/bin"
+        DESTINATION "${CURRENT_PACKAGES_DIR}/manual-tools/${PORT}/debug"
+    )
+    vcpkg_copy_tools(TOOL_NAMES rpath-macho-test-tool
+        SEARCH_DIR "${CURRENT_PACKAGES_DIR}/debug/bin"
+        DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug"
+    )
+    make_rpath_absolute("debug/lib")
+endif()
+make_rpath_absolute("lib")
+vcpkg_copy_tools(TOOL_NAMES rpath-macho-test-tool DESTINATION "${CURRENT_PACKAGES_DIR}/manual-tools/${PORT}")
+vcpkg_copy_tools(TOOL_NAMES rpath-macho-test-tool AUTO_CLEAN)
+file(WRITE "${CURRENT_PACKAGES_DIR}/share/${PORT}/copyright" "This test port is part of vcpkg.")

scripts/test_ports/rpath-macho-test-binaries/project/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.7)
+project(rpath-macho-test CXX)
+
+set(TEST_STRING "" CACHE STRING "")
+
+set(CMAKE_SKIP_INSTALL_RPATH TRUE)
+
+add_library(rpath-macho-backend-lib++ transitive.cpp)
+target_compile_definitions(rpath-macho-backend-lib++ PRIVATE "TEST_STRING=\"${TEST_STRING}\"")
+
+add_library(rpath-macho-test-lib lib.cpp)
+target_link_libraries(rpath-macho-test-lib PRIVATE rpath-macho-backend-lib++)
+
+add_executable(rpath-macho-test-tool main.cpp)
+target_link_libraries(rpath-macho-test-tool PRIVATE rpath-macho-test-lib)
+
+install(TARGETS rpath-macho-backend-lib++ rpath-macho-test-lib rpath-macho-test-tool)

scripts/test_ports/rpath-macho-test-binaries/project/lib.cpp

@@ -0,0 +1,6 @@
+extern const char* getTestStringBackend();
+
+const char* getTestString()
+{
+    return getTestStringBackend();
+}

scripts/test_ports/rpath-macho-test-binaries/project/main.cpp

@@ -0,0 +1,8 @@
+#include <stdio.h>
+
+extern const char* getTestString();
+
+int main()
+{
+    puts(getTestString());
+}

scripts/test_ports/rpath-macho-test-binaries/project/transitive.cpp

@@ -0,0 +1,4 @@
+const char* getTestStringBackend()
+{
+    return TEST_STRING;
+}

scripts/test_ports/rpath-macho-test-binaries/vcpkg.json

@@ -0,0 +1,12 @@
+{
+  "name": "rpath-macho-test-binaries",
+  "version-string": "ci",
+  "description": "Provides installed binaries for rpath macho fixup test",
+  "supports": "native & osx",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    }
+  ]
+}

scripts/test_ports/rpath-macho-test/portfile.cmake

@@ -0,0 +1,62 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+
+foreach(dir IN ITEMS tools/rpath-macho-test-binaries manual-tools/rpath-macho-test-binaries)
+    string(REPLACE "/" "_" logname "execute-rel-${dir}")
+    vcpkg_execute_required_process(
+        COMMAND "${CURRENT_INSTALLED_DIR}/${dir}/rpath-macho-test-tool"
+        WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}"
+        OUTPUT_VARIABLE output
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        LOGNAME "${logname}"
+    )
+    if(NOT output STREQUAL "release")
+        message(SEND_ERROR "${dir}: $Actual: '${output}', expected: 'release'")
+    endif()
+endforeach()
+
+if(NOT VCPKG_BUILD_TYPE)
+    foreach(dir IN ITEMS tools/rpath-macho-test-binaries/debug manual-tools/rpath-macho-test-binaries/debug debug/tools/rpath-macho-test-binaries)
+        string(REPLACE "/" "_" logname "execute-dbg-${dir}")
+        vcpkg_execute_required_process(
+            COMMAND "${CURRENT_INSTALLED_DIR}/${dir}/rpath-macho-test-tool"
+            WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}"
+            OUTPUT_VARIABLE output
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            LOGNAME "${logname}"
+        )
+        if(NOT output STREQUAL "debug")
+            message(SEND_ERROR "${dir}: Actual: '${output}', expected: 'debug'")
+        endif()
+    endforeach()
+endif()
+
+function(check_proper_rpath macho_lib)
+    vcpkg_execute_required_process(
+        COMMAND "otool" "-L" "${macho_lib}"
+        WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}"
+        OUTPUT_VARIABLE output
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        LOGNAME "${logname}"
+    )
+
+    set(found_rpath_backend_lib OFF)
+
+    string(REPLACE "\n" ";" output_lines "${output}")
+    # Ignore first line, it contains the path to the lib which we are checking
+    list(REMOVE_AT output_lines 0)
+    foreach(line IN LISTS output_lines)
+        if("${line}" MATCHES "\\s+/.*librpath-macho-backend-lib\\+\\+\\.dylib")
+            message(SEND_ERROR "${line} contains an absolute path")
+        endif()
+        if("${line}" MATCHES "@rpath/librpath-macho-backend-lib\\+\\+.dylib")
+            set(found_rpath_backend_lib ON)
+        endif()
+    endforeach()
+
+    if(NOT found_rpath_backend_lib)
+        message(SEND_ERROR "@rpath/librpath-macho-backend-lib++.dylib not found in ${output}")
+    endif()
+endfunction()
+
+check_proper_rpath("${CURRENT_INSTALLED_DIR}/lib/librpath-macho-test-lib.dylib")
+check_proper_rpath("${CURRENT_INSTALLED_DIR}/debug/lib/librpath-macho-test-lib.dylib")

scripts/test_ports/rpath-macho-test/vcpkg.json

@@ -0,0 +1,8 @@
+{
+  "name": "rpath-macho-test",
+  "version-string": "ci",
+  "description": "Test rpath macho fixup",
+  "dependencies": [
+    "rpath-macho-test-binaries"
+  ]
+}