[scripts-audit] minor vcpkg_from_* fixes (#19657)

* [scripts-audit] minor vcpkg_from_* fixes

* minor Billy CRs

* hopefully solve Billy's problems

docs/maintainers/vcpkg_extract_source_archive.md

@@ -19,6 +19,7 @@ The latter is suggested to use for all future `vcpkg_extract_source_archive`s.
 vcpkg_extract_source_archive(<out-var>
     ARCHIVE <path>
     [NO_REMOVE_ONE_LEVEL]
+    [SKIP_PATCH_CHECK]
     [PATCHES <patch>...]
     [SOURCE_BASE <base>]
     [BASE_DIRECTORY <relative-path> | WORKING_DIRECTORY <absolute-path>]
@@ -59,6 +60,10 @@ prevent `vcpkg_extract_source_archive` from performing this transformation.
 
 If the source needs to be patched in some way, the `PATCHES` argument
 allows one to do this, just like other `vcpkg_from_*` functions.
+Additionally, the `SKIP_PATCH_CHECK` is provided for `--head` mode -
+this allows patches to fail to apply silently.
+This argument should _only_ be used when installing a `--head` library,
+since otherwise we want a patch failing to appply to be a hard error.
 
 `vcpkg_extract_source_archive` extracts the files to
 `${CURRENT_BUILDTREES_DIR}/<base-directory>/<source-base>-<hash>.clean`.

scripts/cmake/vcpkg_extract_source_archive.cmake

@@ -18,6 +18,7 @@ The latter is suggested to use for all future `vcpkg_extract_source_archive`s.
 vcpkg_extract_source_archive(<out-var>
     ARCHIVE <path>
     [NO_REMOVE_ONE_LEVEL]
+    [SKIP_PATCH_CHECK]
     [PATCHES <patch>...]
     [SOURCE_BASE <base>]
     [BASE_DIRECTORY <relative-path> | WORKING_DIRECTORY <absolute-path>]
@@ -58,6 +59,10 @@ prevent `vcpkg_extract_source_archive` from performing this transformation.
 
 If the source needs to be patched in some way, the `PATCHES` argument
 allows one to do this, just like other `vcpkg_from_*` functions.
+Additionally, the `SKIP_PATCH_CHECK` is provided for `--head` mode -
+this allows patches to fail to apply silently.
+This argument should _only_ be used when installing a `--head` library,
+since otherwise we want a patch failing to appply to be a hard error.
 
 `vcpkg_extract_source_archive` extracts the files to
 `${CURRENT_BUILDTREES_DIR}/<base-directory>/<source-base>-<hash>.clean`.
@@ -221,7 +226,7 @@ function(vcpkg_extract_source_archive)
         endif()
     endif()
 
-    if (arg_Z_SKIP_PATCH_CHECK)
+    if (arg_SKIP_PATCH_CHECK)
         set(quiet_param QUIET)
     else()
         set(quiet_param "")
@@ -239,4 +244,3 @@ function(vcpkg_extract_source_archive)
     set("${out_source_path}" "${source_path}" PARENT_SCOPE)
     message(STATUS "Using source at ${source_path}")
 endfunction()
-

scripts/cmake/vcpkg_from_git.cmake

@@ -79,30 +79,38 @@ function(vcpkg_from_git)
 
     vcpkg_list(SET git_fetch_shallow_param --depth 1)
     vcpkg_list(SET extract_working_directory_param)
+    vcpkg_list(SET skip_patch_check_param)
     set(git_working_directory "${DOWNLOADS}/git-tmp")
-    if(VCPKG_USE_HEAD_VERSION)
-        if(DEFINED arg_HEAD_REF)
-            vcpkg_list(SET working_directory_param "WORKING_DIRECTORY" "${CURRENT_BUILDTREES_DIR}/src/head")
-            vcpkg_list(SET git_fetch_shallow_param --depth 1)
-            set(ref_to_use "${arg_HEAD_REF}")
-            set(git_working_directory "${CURRENT_BUILDTREES_DIR}/src/git-tmp")
-        else()
-            message(STATUS "Package does not specify HEAD_REF. Falling back to non-HEAD version.")
+    set(do_download OFF)
+
+    if(VCPKG_USE_HEAD_VERSION AND DEFINED arg_HEAD_REF)
+        vcpkg_list(SET working_directory_param "WORKING_DIRECTORY" "${CURRENT_BUILDTREES_DIR}/src/head")
+        vcpkg_list(SET git_fetch_shallow_param --depth 1)
+        vcpkg_list(SET skip_patch_check_param SKIP_PATCH_CHECK)
+        set(ref_to_fetch "${arg_HEAD_REF}")
+        set(git_working_directory "${CURRENT_BUILDTREES_DIR}/src/git-tmp")
+        string(REPLACE "/" "_-" sanitized_ref "${arg_HEAD_REF}")
+
+        if(NOT _VCPKG_NO_DOWNLOADS)
+            set(do_download ON)
         endif()
     else()
         if(NOT DEFINED arg_REF)
             message(FATAL_ERROR "Package does not specify REF. It must be built using --head.")
         endif()
+        if(VCPKG_USE_HEAD_VERSION)
+            message(STATUS "Package does not specify HEAD_REF. Falling back to non-HEAD version.")
+        endif()
 
         if(DEFINED arg_FETCH_REF)
-            set(ref_to_use "${arg_FETCH_REF}")
+            set(ref_to_fetch "${arg_FETCH_REF}")
             vcpkg_list(SET git_fetch_shallow_param)
         else()
-            set(ref_to_use "${arg_REF}")
+            set(ref_to_fetch "${arg_REF}")
         endif()
+        string(REPLACE "/" "_-" sanitized_ref "${arg_REF}")
     endif()
 
-    string(REPLACE "/" "_-" sanitized_ref "${ref_to_use}")
     set(temp_archive "${DOWNLOADS}/temp/${PORT}-${sanitized_ref}.tar.gz")
     set(archive "${DOWNLOADS}/${PORT}-${sanitized_ref}.tar.gz")
 
@@ -110,7 +118,11 @@ function(vcpkg_from_git)
         if(_VCPKG_NO_DOWNLOADS)
             message(FATAL_ERROR "Downloads are disabled, but '${archive}' does not exist.")
         endif()
-        message(STATUS "Fetching ${arg_URL} ${ref_to_use}...")
+        set(do_download ON)
+    endif()
+
+    if(do_download)
+        message(STATUS "Fetching ${arg_URL} ${ref_to_fetch}...")
         find_program(GIT NAMES git git.cmd)
         file(MAKE_DIRECTORY "${DOWNLOADS}")
         # Note: git init is safe to run multiple times
@@ -122,7 +134,7 @@ function(vcpkg_from_git)
         )
         vcpkg_execute_required_process(
             ALLOW_IN_DOWNLOAD_MODE
-            COMMAND "${GIT}" fetch "${arg_URL}" "${ref_to_use}" ${git_fetch_shallow_param} -n
+            COMMAND "${GIT}" fetch "${arg_URL}" "${ref_to_fetch}" ${git_fetch_shallow_param} -n
             WORKING_DIRECTORY "${git_working_directory}"
             LOGNAME "git-fetch-${TARGET_TRIPLET}"
         )
@@ -179,6 +191,7 @@ function(vcpkg_from_git)
         PATCHES ${arg_PATCHES}
         NO_REMOVE_ONE_LEVEL
         ${extract_working_directory_param}
+        ${skip_patch_check_param}
     )
 
     set("${arg_OUT_SOURCE_PATH}" "${SOURCE_PATH}" PARENT_SCOPE)

scripts/cmake/vcpkg_from_github.cmake

@@ -123,54 +123,61 @@ function(vcpkg_from_github)
     set(org_name "${CMAKE_MATCH_1}")
     set(repo_name "${CMAKE_MATCH_2}")
 
-    set(redownload_param "")
-    set(working_directory_param "")
-    set(sha512_param "SHA512" "${arg_SHA512}")
-    set(ref_to_use "${arg_REF}")
-    if(VCPKG_USE_HEAD_VERSION)
-        if(DEFINED arg_HEAD_REF)
-            set(redownload_param "ALWAYS_REDOWNLOAD")
-            set(sha512_param "SKIP_SHA512")
-            set(working_directory_param "WORKING_DIRECTORY" "${CURRENT_BUILDTREES_DIR}/src/head")
-            set(ref_to_use "${arg_HEAD_REF}")
-        else()
-            message(STATUS "Package does not specify HEAD_REF. Falling back to non-HEAD version.")
-        endif()
-    elseif(NOT DEFINED arg_REF)
+    if(VCPKG_USE_HEAD_VERSION AND NOT DEFINED arg_HEAD_REF)
+        message(STATUS "Package does not specify HEAD_REF. Falling back to non-HEAD version.")
+        set(VCPKG_USE_HEAD_VERSION OFF)
+    elseif(NOT VCPKG_USE_HEAD_VERSION AND NOT DEFINED arg_REF)
         message(FATAL_ERROR "Package does not specify REF. It must be built using --head.")
     endif()
 
-    # avoid using either - or _, to allow both `foo/bar` and `foo-bar` to coexist
-    # we assume that no one will name a ref "foo_-bar"
-    string(REPLACE "/" "_-" sanitized_ref "${ref_to_use}")
-    if(DEFINED arg_FILE_DISAMBIGUATOR AND NOT VCPKG_USE_HEAD_VERSION)
-        set(downloaded_file_name "${org_name}-${repo_name}-${sanitized_ref}-${arg_FILE_DISAMBIGUATOR}.tar.gz")
-    else()
-        set(downloaded_file_name "${org_name}-${repo_name}-${sanitized_ref}.tar.gz")
-    endif()
-
-
     # exports VCPKG_HEAD_VERSION to the caller. This will get picked up by ports.cmake after the build.
     if(VCPKG_USE_HEAD_VERSION)
+        string(REPLACE "/" "_-" sanitized_head_ref "${arg_HEAD_REF}")
         vcpkg_download_distfile(archive_version
             URLS "${github_api_url}/repos/${org_name}/${repo_name}/git/refs/heads/${arg_HEAD_REF}"
-            FILENAME "${downloaded_file_name}.version"
+            FILENAME "${org_name}-${repo_name}-${sanitized_head_ref}.version"
             ${headers_param}
             SKIP_SHA512
             ALWAYS_REDOWNLOAD
         )
         # Parse the github refs response with regex.
-        # TODO: add json-pointer support to vcpkg
         file(READ "${archive_version}" version_contents)
-        if(NOT version_contents MATCHES [["sha":(\ *)"([a-f0-9]+)"]])
+        string(JSON head_version
+            ERROR_VARIABLE head_version_err
+            GET "${version_contents}"
+            "object"
+            "sha"
+        )
+        if(NOT "${head_version_err}" STREQUAL "NOTFOUND")
             message(FATAL_ERROR "Failed to parse API response from '${version_url}':
-
 ${version_contents}
+
+Error was: ${head_version_err}
 ")
         endif()
-        set(VCPKG_HEAD_VERSION "${CMAKE_MATCH_1}" PARENT_SCOPE)
+
+        set(VCPKG_HEAD_VERSION "${head_version}" PARENT_SCOPE)
+        set(ref_to_use "${head_version}")
+
+        vcpkg_list(SET redownload_param ALWAYS_REDOWNLOAD)
+        vcpkg_list(SET sha512_param SKIP_SHA512)
+        vcpkg_list(SET working_directory_param WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/src/head")
+        vcpkg_list(SET skip_patch_check_param SKIP_PATCH_CHECK)
+    else()
+        set(ref_to_use "${arg_REF}")
+
+        vcpkg_list(SET redownload_param)
+        vcpkg_list(SET working_directory_param)
+        vcpkg_list(SET skip_patch_check_param)
+        vcpkg_list(SET sha512_param SHA512 "${arg_SHA512}")
     endif()
 
+    string(REPLACE "/" "_-" sanitized_ref "${ref_to_use}")
+    if(DEFINED arg_FILE_DISAMBIGUATOR AND NOT VCPKG_USE_HEAD_REF)
+        set(downloaded_file_name "${org_name}-${repo_name}-${sanitized_ref}-${arg_FILE_DISAMBIGUATOR}.tar.gz")
+    else()
+        set(downloaded_file_name "${org_name}-${repo_name}-${sanitized_ref}.tar.gz")
+    endif()
     # Try to download the file information from github
     vcpkg_download_distfile(archive
         URLS "${github_host}/${org_name}/${repo_name}/archive/${ref_to_use}.tar.gz"
@@ -185,6 +192,7 @@ ${version_contents}
         REF "${sanitized_ref}"
         PATCHES ${arg_PATCHES}
         ${working_directory_param}
+        ${skip_patch_check_param}
     )
     set("${arg_OUT_SOURCE_PATH}" "${SOURCE_PATH}" PARENT_SCOPE)
 endfunction()

scripts/cmake/z_vcpkg_apply_patches.cmake

@@ -57,8 +57,12 @@ function(z_vcpkg_apply_patches)
         )
         file(WRITE "${CURRENT_BUILDTREES_DIR}/${logname}-err.log" "${error}")
 
-        if(error_code AND NOT arg_QUIET)
-            message(FATAL_ERROR "Applying patch failed: ${error}")
+        if(error_code)
+            if(arg_QUIET)
+                message(STATUS "Applying patch ${patch} - failure silenced")
+            else()
+                message(FATAL_ERROR "Applying patch failed: ${error}")
+            endif()
         endif()
 
         math(EXPR patchnum "${patchnum} + 1")

scripts/test_ports/vcpkg-from-git-test/portfile.cmake

@@ -0,0 +1,193 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+
+set(git_remote "${CURRENT_BUILDTREES_DIR}/test-git-repo")
+file(REMOVE_RECURSE "${git_remote}")
+
+vcpkg_find_acquire_program(GIT)
+vcpkg_list(SET git_config
+    -c core.autocrlf=false
+    -c user.email=vcpkg@example.com
+    -c user.name=vcpkg
+)
+
+vcpkg_list(SET git ${GIT} ${git_config})
+
+vcpkg_execute_required_process(
+    COMMAND ${git} init "${git_remote}"
+    WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}"
+    LOGNAME "git-init"
+)
+vcpkg_execute_required_process(
+    COMMAND ${git} config uploadpack.allowReachableSHA1InWant true
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-config"
+)
+vcpkg_execute_required_process(
+    COMMAND ${git} checkout -b main
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-new-branch"
+)
+
+file(WRITE "${git_remote}/README.txt" "first commit")
+vcpkg_execute_required_process(
+    COMMAND ${git} add "${git_remote}/README.txt"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-add.1"
+)
+vcpkg_execute_required_process(
+    COMMAND ${git} commit -m "first commit"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-commit.1"
+)
+vcpkg_execute_in_download_mode(
+    COMMAND ${git} rev-parse HEAD
+    OUTPUT_VARIABLE ref
+    RESULT_VARIABLE error_code
+    WORKING_DIRECTORY "${git_remote}"
+)
+if(NOT "${error_code}" EQUAL "0")
+    message(FATAL_ERROR "Failed to rev-parse HEAD: ${error_code}")
+endif()
+string(STRIP "${ref}" ref)
+
+file(WRITE "${git_remote}/README.txt" "second commit")
+vcpkg_execute_required_process(
+    COMMAND ${git} add "${git_remote}/README.txt"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-add.2"
+)
+vcpkg_execute_required_process(
+    COMMAND ${git} commit -m "second commit"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-commit.2"
+)
+vcpkg_execute_in_download_mode(
+    COMMAND ${git} rev-parse HEAD
+    OUTPUT_VARIABLE head_ref
+    RESULT_VARIABLE error_code
+    WORKING_DIRECTORY "${git_remote}"
+)
+if(NOT "${error_code}" EQUAL "0")
+    message(FATAL_ERROR "Failed to rev-parse HEAD: ${error_code}")
+endif()
+string(STRIP "${head_ref}" head_ref)
+
+# test regular mode
+set(VCPKG_USE_HEAD_VERSION OFF)
+vcpkg_from_git(
+    OUT_SOURCE_PATH source_path
+    URL "${git_remote}"
+    REF "${ref}"
+    HEAD_REF main
+)
+file(READ "${source_path}/README.txt" contents)
+if(NOT "${contents}" STREQUAL "first commit")
+    message(FATAL_ERROR "Failed to checkout the first commit. Contents were:
+${contents}
+")
+endif()
+
+# test regular mode with FETCH_REF
+vcpkg_execute_required_process(
+    COMMAND ${git} config uploadpack.allowReachableSHA1InWant false
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-config"
+)
+set(VCPKG_USE_HEAD_VERSION OFF)
+vcpkg_from_git(
+    OUT_SOURCE_PATH source_path
+    URL "${git_remote}"
+    REF "${ref}"
+    FETCH_REF main
+    HEAD_REF main
+)
+file(READ "${source_path}/README.txt" contents)
+if(NOT "${contents}" STREQUAL "first commit")
+    message(FATAL_ERROR "Failed to checkout the first commit. Contents were:
+${contents}
+")
+endif()
+
+vcpkg_execute_required_process(
+    COMMAND ${git} config uploadpack.allowReachableSHA1InWant true
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git-config"
+)
+
+# test head mode
+set(VCPKG_USE_HEAD_VERSION ON)
+vcpkg_from_git(
+    OUT_SOURCE_PATH source_path
+    URL "${git_remote}"
+    REF "${ref}"
+    HEAD_REF main
+)
+file(READ "${source_path}/README.txt" contents)
+if(NOT "${contents}" STREQUAL "second commit")
+    message(FATAL_ERROR "Failed to checkout the HEAD commit. Contents were:
+${contents}
+")
+endif()
+if(NOT "${VCPKG_HEAD_VERSION}" STREQUAL "${head_ref}")
+    message(FATAL_ERROR "Failed to checkout the right HEAD commit.
+    Expected: ${head_ref}
+    Actual  : ${VCPKG_HEAD_VERSION}
+")
+endif()
+
+# test head mode + no HEAD_REF -> just uses REF
+set(VCPKG_USE_HEAD_VERSION ON)
+vcpkg_from_git(
+    OUT_SOURCE_PATH source_path
+    URL "${git_remote}"
+    REF "${ref}"
+)
+file(READ "${source_path}/README.txt" contents)
+if(NOT "${contents}" STREQUAL "first commit")
+    message(FATAL_ERROR "Failed to checkout the regular commit. Contents were:
+${contents}
+")
+endif()
+
+# test new head ref
+file(WRITE "${git_remote}/README.txt" "third commit")
+vcpkg_execute_required_process(
+    COMMAND ${git} add "${git_remote}/README.txt"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git.7"
+)
+vcpkg_execute_required_process(
+    COMMAND ${git} commit -m "second commit"
+    WORKING_DIRECTORY "${git_remote}"
+    LOGNAME "git.8"
+)
+vcpkg_execute_in_download_mode(
+    COMMAND ${git} rev-parse HEAD
+    OUTPUT_VARIABLE new_head_ref
+    RESULT_VARIABLE error_code
+    WORKING_DIRECTORY "${git_remote}"
+)
+if(NOT "${error_code}" EQUAL "0")
+    message(FATAL_ERROR "Failed to rev-parse HEAD: ${error_code}")
+endif()
+string(STRIP "${new_head_ref}" new_head_ref)
+
+set(VCPKG_USE_HEAD_VERSION ON)
+vcpkg_from_git(
+    OUT_SOURCE_PATH source_path
+    URL "${git_remote}"
+    REF "${ref}"
+    HEAD_REF main
+)
+file(READ "${source_path}/README.txt" contents)
+if(NOT "${contents}" STREQUAL "third commit")
+    message(FATAL_ERROR "Failed to checkout the right HEAD commit. Contents were:
+${contents}
+")
+endif()
+if(NOT "${VCPKG_HEAD_VERSION}" STREQUAL "${new_head_ref}")
+    message(FATAL_ERROR "Failed to checkout the right HEAD commit.
+    Expected: ${new_head_ref}
+    Actual  : ${VCPKG_HEAD_VERSION}
+")
+endif()

scripts/test_ports/vcpkg-from-git-test/vcpkg.json

@@ -0,0 +1,4 @@
+{
+  "name": "vcpkg-from-git-test",
+  "version": "0"
+}