diff --git a/docs/users/binarycaching.md b/docs/users/binarycaching.md index 3798fbde76..c3fb98a4ff 100644 --- a/docs/users/binarycaching.md +++ b/docs/users/binarycaching.md @@ -108,7 +108,8 @@ By default, zip-based archives will be cached at the first valid location of: | `default[,]` | Adds the default file-based location | `files,[,]` | Adds a custom file-based location | `nuget,[,]` | Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI -| `nugetconfig,[,]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter
of the NuGet CLI. This config should specify `defaultPushSource` for uploads. +| `nugetconfig,[,]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This config should specify `defaultPushSource` for uploads. +| `x-azblob,,[,]` | **Experimental: will change or be removed without warning**
Adds an Azure Blob Storage source. Uses Shared Access Signature validation. URL should include the container path. | `interactive` | Enables interactive credential management for NuGet (for debugging; requires `--debug` on the command line) The `` optional parameter for certain sources controls whether they will be consulted for diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 6f2e90fec0..c0e1c3a260 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -14,6 +14,9 @@ jobs: value: /Users/vagrant/Data - name: VCPKG_DOWNLOADS value: /Users/vagrant/Data/downloads + - group: azblob-test-sas-group + - name: BINARY_SOURCE_STUB + value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)" steps: - bash: df -h @@ -40,7 +43,7 @@ jobs: inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' - arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)' + arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -BinarySourceStub "$(BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)' - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 index bac16c595a..98ffbf53a9 100755 --- a/scripts/azure-pipelines/test-modified-ports.ps1 +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -16,8 +16,11 @@ The location used as scratch space for 'installed', 'packages', and 'buildtrees' The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory. .PARAMETER ArchivesRoot -The location where the binary caching archives are stored. Shared across runs of this script. If -this parameter is not set, binary caching will not be used. +Equivalent to '-BinarySourceStub "files,$ArchivesRoot"' + +.PARAMETER BinarySourceStub +The type and parameters of the binary source. Shared across runs of this script. If +this parameter is not set, binary caching will not be used. Example: "files,W:\" .PARAMETER BuildReason The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used). @@ -25,7 +28,7 @@ If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set binary caching will default to read-write mode. #> -[CmdletBinding()] +[CmdletBinding(DefaultParameterSetName="ArchivesRoot")] Param( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] @@ -35,7 +38,10 @@ Param( $WorkingRoot, [ValidateNotNullOrEmpty()] $ArtifactStagingDirectory = '.', + [Parameter(ParameterSetName='ArchivesRoot')] $ArchivesRoot = $null, + [Parameter(ParameterSetName='BinarySourceStub')] + $BinarySourceStub = $null, $BuildReason = $null ) @@ -49,7 +55,7 @@ $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees' $installRoot = Join-Path $WorkingRoot 'installed' $packagesRoot = Join-Path $WorkingRoot 'packages' -$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) +$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or -Not ([string]::IsNullOrWhiteSpace($BinarySourceStub)) $commonArgs = @() if ($usingBinaryCaching) { $commonArgs += @('--binarycaching') @@ -78,8 +84,12 @@ if ($usingBinaryCaching) { Write-Host "Build reason was $BuildReason, using binary caching in write only mode." $binaryCachingMode = 'write' } - - $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode") + if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) { + $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode") + } + else { + $commonArgs += @("--binarysource=clear;files,$ArchivesRoot,$binaryCachingMode") + } } if ($Triplet -eq 'x64-linux') { diff --git a/toolsrc/include/vcpkg/base/downloads.h b/toolsrc/include/vcpkg/base/downloads.h index 61c7924885..bf97c0650e 100644 --- a/toolsrc/include/vcpkg/base/downloads.h +++ b/toolsrc/include/vcpkg/base/downloads.h @@ -1,16 +1,41 @@ #pragma once #include +#include +#include namespace vcpkg::Downloads { + namespace details + { + struct SplitURIView + { + StringView scheme; + Optional authority; + StringView path_query_fragment; + }; + + // e.g. {"https","//example.org", "/index.html"} + Optional split_uri_view(StringView uri); + } + void verify_downloaded_file_hash(const Files::Filesystem& fs, const std::string& url, const fs::path& path, const std::string& sha512); + // Returns url that was successfully downloaded from + std::string download_file(Files::Filesystem& fs, + View urls, + const fs::path& download_path, + const std::string& sha512); + void download_file(Files::Filesystem& fs, const std::string& url, const fs::path& download_path, const std::string& sha512); + + std::vector download_files(Files::Filesystem& fs, View> url_pairs); + int put_file(const Files::Filesystem&, StringView url, const fs::path& file); + std::vector url_heads(View urls); } diff --git a/toolsrc/include/vcpkg/base/fwd/lockguarded.h b/toolsrc/include/vcpkg/base/fwd/lockguarded.h new file mode 100644 index 0000000000..e7e88b5a1d --- /dev/null +++ b/toolsrc/include/vcpkg/base/fwd/lockguarded.h @@ -0,0 +1,10 @@ +#pragma once + +namespace vcpkg::Util +{ + template + struct LockGuardPtr; + + template + struct LockGuarded; +} diff --git a/toolsrc/include/vcpkg/base/lockguarded.h b/toolsrc/include/vcpkg/base/lockguarded.h new file mode 100644 index 0000000000..e573ec7b0e --- /dev/null +++ b/toolsrc/include/vcpkg/base/lockguarded.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include + +namespace vcpkg::Util +{ + template + struct LockGuarded + { + friend struct LockGuardPtr; + + LockGuardPtr lock() { return *this; } + + private: + std::mutex m_mutex; + T m_t; + }; + + template + struct LockGuardPtr + { + T& operator*() { return m_ptr; } + T* operator->() { return &m_ptr; } + + T* get() { return &m_ptr; } + + LockGuardPtr(LockGuarded& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { } + + private: + std::lock_guard m_lock; + T& m_ptr; + }; +} diff --git a/toolsrc/include/vcpkg/base/util.h b/toolsrc/include/vcpkg/base/util.h index 0453ac20a6..ca5104c234 100644 --- a/toolsrc/include/vcpkg/base/util.h +++ b/toolsrc/include/vcpkg/base/util.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -222,36 +221,6 @@ namespace vcpkg::Util ~ResourceBase() = default; }; - template - struct LockGuardPtr; - - template - struct LockGuarded - { - friend struct LockGuardPtr; - - LockGuardPtr lock() { return *this; } - - private: - std::mutex m_mutex; - T m_t; - }; - - template - struct LockGuardPtr - { - T& operator*() { return m_ptr; } - T* operator->() { return &m_ptr; } - - T* get() { return &m_ptr; } - - LockGuardPtr(LockGuarded& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { } - - private: - std::unique_lock m_lock; - T& m_ptr; - }; - namespace Enum { template diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h index 9e77bc7638..5a127cb5c2 100644 --- a/toolsrc/include/vcpkg/binarycaching.h +++ b/toolsrc/include/vcpkg/binarycaching.h @@ -8,8 +8,12 @@ #include +#include + namespace vcpkg { + struct MergeBinaryProviders; + enum class RestoreResult { missing, @@ -20,18 +24,32 @@ namespace vcpkg struct IBinaryProvider { virtual ~IBinaryProvider() = default; - /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing - /// `plan`. - virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0; + /// Attempts to restore the package referenced by `action` into the packages directory. virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0; + /// Called upon a successful build of `action` virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0; - /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine - /// missing packages. - virtual RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0; + + /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for + /// executing `plan`. + /// Must only be called once for a given binary provider instance + /// InOut vector of actions to be prefetched + virtual void prefetch(const VcpkgPaths& paths, + std::vector& actions) = 0; + + /// Requests the result of try_restore() without actually downloading the package. Used by CI to + /// determine missing packages. + /// InOut map to track the restored packages. Should be initialized to + /// {&action, RestoreResult::missing} for all install actions + virtual void precheck( + const VcpkgPaths& paths, + std::unordered_map& results_map) = 0; }; + std::unordered_map binary_provider_precheck( + const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider); + IBinaryProvider& null_binary_provider(); ExpectedS> create_binary_provider_from_configs(View args); diff --git a/toolsrc/include/vcpkg/dependencies.h b/toolsrc/include/vcpkg/dependencies.h index 5e46a61948..1ea81c185e 100644 --- a/toolsrc/include/vcpkg/dependencies.h +++ b/toolsrc/include/vcpkg/dependencies.h @@ -61,6 +61,7 @@ namespace vcpkg::Dependencies std::string displayname() const; const std::string& public_abi() const; bool has_package_abi() const; + Optional package_abi() const; const Build::PreBuildInfo& pre_build_info(LineInfo linfo) const; PackageSpec spec; diff --git a/toolsrc/include/vcpkg/globalstate.h b/toolsrc/include/vcpkg/globalstate.h index 35ca71dbba..44c34f25c2 100644 --- a/toolsrc/include/vcpkg/globalstate.h +++ b/toolsrc/include/vcpkg/globalstate.h @@ -1,7 +1,8 @@ #pragma once +#include + #include -#include #include #include diff --git a/toolsrc/include/vcpkg/metrics.h b/toolsrc/include/vcpkg/metrics.h index 3ab1bc7586..b1a5e61043 100644 --- a/toolsrc/include/vcpkg/metrics.h +++ b/toolsrc/include/vcpkg/metrics.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include diff --git a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp index 259f1986e1..1c46790bf4 100644 --- a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp +++ b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp @@ -298,3 +298,39 @@ TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]") REQUIRE(parsed.has_value()); } } + +TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") +{ + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,?sas", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,,sas", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,invalid", {}); + REQUIRE(!parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,read", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,write", {}); + REQUIRE(parsed.has_value()); + } + { + auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,readwrite", {}); + REQUIRE(parsed.has_value()); + } +} diff --git a/toolsrc/src/vcpkg-test/downloads.cpp b/toolsrc/src/vcpkg-test/downloads.cpp new file mode 100644 index 0000000000..b182e46d6b --- /dev/null +++ b/toolsrc/src/vcpkg-test/downloads.cpp @@ -0,0 +1,59 @@ +#include + +#include + +using namespace vcpkg; + +TEST_CASE ("Downloads::details::split_uri_view", "[downloads]") +{ + { + auto x = Downloads::details::split_uri_view("https://github.com/Microsoft/vcpkg"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "https"); + REQUIRE(x.get()->authority.value_or("") == "//github.com"); + REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg"); + } + { + auto x = Downloads::details::split_uri_view(""); + REQUIRE(!x.has_value()); + } + { + auto x = Downloads::details::split_uri_view("hello"); + REQUIRE(!x.has_value()); + } + { + auto x = Downloads::details::split_uri_view("file:"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "file"); + REQUIRE(!x.get()->authority.has_value()); + REQUIRE(x.get()->path_query_fragment == ""); + } + { + auto x = Downloads::details::split_uri_view("file:path"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "file"); + REQUIRE(!x.get()->authority.has_value()); + REQUIRE(x.get()->path_query_fragment == "path"); + } + { + auto x = Downloads::details::split_uri_view("file:/path"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "file"); + REQUIRE(!x.get()->authority.has_value()); + REQUIRE(x.get()->path_query_fragment == "/path"); + } + { + auto x = Downloads::details::split_uri_view("file://user:pw@host"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "file"); + REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host"); + REQUIRE(x.get()->path_query_fragment == ""); + } + { + auto x = Downloads::details::split_uri_view("ftp://host:port/"); + REQUIRE(x.has_value()); + REQUIRE(x.get()->scheme == "ftp"); + REQUIRE(x.get()->authority.value_or({}) == "//host:port"); + REQUIRE(x.get()->path_query_fragment == "/"); + } +} diff --git a/toolsrc/src/vcpkg/base/downloads.cpp b/toolsrc/src/vcpkg/base/downloads.cpp index 40f0494f9c..e263f70375 100644 --- a/toolsrc/src/vcpkg/base/downloads.cpp +++ b/toolsrc/src/vcpkg/base/downloads.cpp @@ -1,6 +1,10 @@ +#include #include #include +#include +#include #include +#include #include #include @@ -11,133 +15,178 @@ namespace vcpkg::Downloads { #if defined(_WIN32) - static void winhttp_download_file(Files::Filesystem& fs, - ZStringView target_file_path, - StringView hostname, - StringView url_path) + struct WinHttpHandleDeleter { - // Make sure the directories are present, otherwise fopen_s fails - const auto dir = fs::path(target_file_path.c_str()).parent_path(); - std::error_code ec; - fs.create_directories(dir, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not create directories %s", fs::u8string(dir)); + void operator()(HINTERNET h) const { WinHttpCloseHandle(h); } + }; - FILE* f = nullptr; - const errno_t err = fopen_s(&f, target_file_path.c_str(), "wb"); - Checks::check_exit(VCPKG_LINE_INFO, - !err, - "Could not download https://%s%s. Failed to open file %s. Error code was %s", - hostname, - url_path, - target_file_path, - std::to_string(err)); - ASSUME(f != nullptr); - - auto hSession = WinHttpOpen(L"vcpkg/1.0", - IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY - : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, - WINHTTP_NO_PROXY_NAME, - WINHTTP_NO_PROXY_BYPASS, - 0); - Checks::check_exit(VCPKG_LINE_INFO, hSession, "WinHttpOpen() failed: %d", GetLastError()); - - // If the environment variable HTTPS_PROXY is set - // use that variable as proxy. This situation might exist when user is in a company network - // with restricted network/proxy settings - auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY"); - if (auto p_https_proxy = maybe_https_proxy_env.get()) + struct WinHttpRequest + { + static ExpectedS make(HINTERNET hConnect, StringView url_path, const wchar_t* method = L"GET") { - std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy); - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = env_proxy_settings.data(); - proxy.lpszProxyBypass = nullptr; + WinHttpRequest ret; + // Create an HTTP request handle. + auto h = WinHttpOpenRequest(hConnect, + method, + Strings::to_utf16(url_path).c_str(), + nullptr, + WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, + WINHTTP_FLAG_SECURE); + if (!h) return Strings::concat("WinHttpOpenRequest() failed: ", GetLastError()); + ret.m_hRequest.reset(h); - WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + // Send a request. + auto bResults = WinHttpSendRequest( + ret.m_hRequest.get(), WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); + + if (!bResults) return Strings::concat("WinHttpSendRequest() failed: ", GetLastError()); + + // End the request. + bResults = WinHttpReceiveResponse(ret.m_hRequest.get(), NULL); + if (!bResults) return Strings::concat("WinHttpReceiveResponse() failed: ", GetLastError()); + + DWORD dwStatusCode = 0; + DWORD dwSize = sizeof(dwStatusCode); + + bResults = WinHttpQueryHeaders(ret.m_hRequest.get(), + WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + WINHTTP_HEADER_NAME_BY_INDEX, + &dwStatusCode, + &dwSize, + WINHTTP_NO_HEADER_INDEX); + if (!bResults) return Strings::concat("WinHttpQueryHeaders() failed: ", GetLastError()); + if (dwStatusCode < 200 || dwStatusCode >= 300) return Strings::concat("failed: status code ", dwStatusCode); + + return std::move(ret); } - // Win7 IE Proxy fallback - else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater()) - { - // First check if any proxy has been found automatically - WINHTTP_PROXY_INFO proxyInfo; - DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO); - auto noProxyFound = !WinHttpQueryOption(hSession, WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) || - proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY; - // If no proxy was found automatically, use IE's proxy settings, if any - if (noProxyFound) + template + ExpectedS forall_data(F f) + { + std::vector buf; + + size_t total_downloaded_size = 0; + DWORD dwSize = 0; + do { - WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy; - if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr) + DWORD downloaded_size = 0; + auto bResults = WinHttpQueryDataAvailable(m_hRequest.get(), &dwSize); + if (!bResults) return Strings::concat("WinHttpQueryDataAvailable() failed: ", GetLastError()); + + if (buf.size() < dwSize) buf.resize(static_cast(dwSize) * 2); + + bResults = WinHttpReadData(m_hRequest.get(), (LPVOID)buf.data(), dwSize, &downloaded_size); + if (!bResults) return Strings::concat("WinHttpReadData() failed: ", GetLastError()); + f(Span(buf.data(), downloaded_size)); + + total_downloaded_size += downloaded_size; + } while (dwSize > 0); + return 1; + } + + std::unique_ptr m_hRequest; + }; + + struct WinHttpSession + { + static ExpectedS make() + { + auto h = WinHttpOpen(L"vcpkg/1.0", + IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY + : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, + WINHTTP_NO_PROXY_NAME, + WINHTTP_NO_PROXY_BYPASS, + 0); + if (!h) return Strings::concat("WinHttpOpen() failed: ", GetLastError()); + WinHttpSession ret; + ret.m_hSession.reset(h); + + // If the environment variable HTTPS_PROXY is set + // use that variable as proxy. This situation might exist when user is in a company network + // with restricted network/proxy settings + auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY"); + if (auto p_https_proxy = maybe_https_proxy_env.get()) + { + std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy); + WINHTTP_PROXY_INFO proxy; + proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; + proxy.lpszProxy = env_proxy_settings.data(); + proxy.lpszProxyBypass = nullptr; + + WinHttpSetOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + } + // Win7 IE Proxy fallback + else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater()) + { + // First check if any proxy has been found automatically + WINHTTP_PROXY_INFO proxyInfo; + DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO); + auto noProxyFound = + !WinHttpQueryOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) || + proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY; + + // If no proxy was found automatically, use IE's proxy settings, if any + if (noProxyFound) { - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = ieProxy.lpszProxy; - proxy.lpszProxyBypass = ieProxy.lpszProxyBypass; - WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); - GlobalFree(ieProxy.lpszProxy); - GlobalFree(ieProxy.lpszProxyBypass); - GlobalFree(ieProxy.lpszAutoConfigUrl); + WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy; + if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr) + { + WINHTTP_PROXY_INFO proxy; + proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; + proxy.lpszProxy = ieProxy.lpszProxy; + proxy.lpszProxyBypass = ieProxy.lpszProxyBypass; + WinHttpSetOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + GlobalFree(ieProxy.lpszProxy); + GlobalFree(ieProxy.lpszProxyBypass); + GlobalFree(ieProxy.lpszAutoConfigUrl); + } } } + + // Use Windows 10 defaults on Windows 7 + DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | + WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); + WinHttpSetOption( + ret.m_hSession.get(), WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); + + return ret; } - // Use Windows 10 defaults on Windows 7 - DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_SSL3 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | - WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - WinHttpSetOption(hSession, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); + std::unique_ptr m_hSession; + }; - // Specify an HTTP server. - auto hConnect = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), INTERNET_DEFAULT_HTTPS_PORT, 0); - Checks::check_exit(VCPKG_LINE_INFO, hConnect, "WinHttpConnect() failed: %d", GetLastError()); - - // Create an HTTP request handle. - auto hRequest = WinHttpOpenRequest(hConnect, - L"GET", - Strings::to_utf16(url_path).c_str(), - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - WINHTTP_FLAG_SECURE); - Checks::check_exit(VCPKG_LINE_INFO, hRequest, "WinHttpOpenRequest() failed: %d", GetLastError()); - - // Send a request. - auto bResults = - WinHttpSendRequest(hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); - - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpSendRequest() failed: %d", GetLastError()); - - // End the request. - bResults = WinHttpReceiveResponse(hRequest, NULL); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReceiveResponse() failed: %d", GetLastError()); - - std::vector buf; - - size_t total_downloaded_size = 0; - DWORD dwSize = 0; - do + struct WinHttpConnection + { + static ExpectedS make(HINTERNET hSession, StringView hostname, INTERNET_PORT port) { - DWORD downloaded_size = 0; - bResults = WinHttpQueryDataAvailable(hRequest, &dwSize); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpQueryDataAvailable() failed: %d", GetLastError()); + // Specify an HTTP server. + auto h = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), port, 0); + if (!h) return Strings::concat("WinHttpConnect() failed: ", GetLastError()); + WinHttpConnection ret; + ret.m_hConnect.reset(h); + return ret; + } - if (buf.size() < dwSize) buf.resize(static_cast(dwSize) * 2); - - bResults = WinHttpReadData(hRequest, (LPVOID)buf.data(), dwSize, &downloaded_size); - Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReadData() failed: %d", GetLastError()); - fwrite(buf.data(), 1, downloaded_size, f); - - total_downloaded_size += downloaded_size; - } while (dwSize > 0); - - WinHttpCloseHandle(hSession); - WinHttpCloseHandle(hConnect); - WinHttpCloseHandle(hRequest); - fflush(f); - fclose(f); - } + std::unique_ptr m_hConnect; + }; #endif + Optional details::split_uri_view(StringView uri) + { + auto sep = std::find(uri.begin(), uri.end(), ':'); + if (sep == uri.end()) return nullopt; + + StringView scheme(uri.begin(), sep); + if (Strings::starts_with({sep + 1, uri.end()}, "//")) + { + auto path_start = std::find(sep + 3, uri.end(), '/'); + return details::SplitURIView{scheme, StringView{sep + 1, path_start}, {path_start, uri.end()}}; + } + // no authority + return details::SplitURIView{scheme, {}, {sep + 1, uri.end()}}; + } + void verify_downloaded_file_hash(const Files::Filesystem& fs, const std::string& url, const fs::path& path, @@ -167,30 +216,251 @@ namespace vcpkg::Downloads actual_hash); } - void download_file(vcpkg::Files::Filesystem& fs, + static void url_heads_inner(View urls, std::vector* out) + { + static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e"; + + System::CmdLineBuilder cmd; + cmd.string_arg("curl") + .string_arg("--head") + .string_arg("--location") + .string_arg("-w") + .string_arg(Strings::concat(guid_marker, " %{http_code}\\n")); + for (auto&& url : urls) + { + cmd.string_arg(url); + } + auto res = System::cmd_execute_and_stream_lines(cmd, [out](const std::string& line) { + if (Strings::starts_with(line, guid_marker)) + { + out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10)); + } + }); + Checks::check_exit(VCPKG_LINE_INFO, res == 0, "curl failed to execute with exit code: %d", res); + } + std::vector url_heads(View urls) + { + static constexpr size_t batch_size = 100; + + std::vector ret; + + size_t i = 0; + for (; i + batch_size <= urls.size(); i += batch_size) + { + url_heads_inner({urls.data() + i, batch_size}, &ret); + } + if (i != urls.size()) url_heads_inner({urls.begin() + i, urls.end()}, &ret); + + return ret; + } + + static void download_files_inner(Files::Filesystem&, + View> url_pairs, + std::vector* out) + { + static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e"; + + System::CmdLineBuilder cmd; + cmd.string_arg("curl") + .string_arg("--location") + .string_arg("-w") + .string_arg(Strings::concat(guid_marker, " %{http_code}\\n")); + for (auto&& url : url_pairs) + { + cmd.string_arg(url.first).string_arg("-o").path_arg(url.second); + } + auto res = System::cmd_execute_and_stream_lines(cmd, [out](const std::string& line) { + if (Strings::starts_with(line, guid_marker)) + { + out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10)); + } + }); + Checks::check_exit(VCPKG_LINE_INFO, res == 0, "curl failed to execute with exit code: %d", res); + } + std::vector download_files(Files::Filesystem& fs, View> url_pairs) + { + static constexpr size_t batch_size = 50; + + std::vector ret; + + size_t i = 0; + for (; i + batch_size <= url_pairs.size(); i += batch_size) + { + download_files_inner(fs, {url_pairs.data() + i, batch_size}, &ret); + } + if (i != url_pairs.size()) download_files_inner(fs, {url_pairs.begin() + i, url_pairs.end()}, &ret); + + Checks::check_exit(VCPKG_LINE_INFO, ret.size() == url_pairs.size()); + return ret; + } + + int put_file(const Files::Filesystem&, StringView url, const fs::path& file) + { + static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; + + System::CmdLineBuilder cmd; + cmd.string_arg("curl").string_arg("-X").string_arg("PUT"); + cmd.string_arg("-w").string_arg(Strings::concat("\\n", guid_marker, "%{http_code}")); + cmd.string_arg(url); + cmd.string_arg("-T").path_arg(file); + cmd.string_arg("-H").string_arg("x-ms-blob-type: BlockBlob"); + int code = 0; + auto res = System::cmd_execute_and_stream_lines(cmd, [&code](const std::string& line) { + if (Strings::starts_with(line, guid_marker)) + { + code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); + } + }); + if (res != 0) + { + System::print2(System::Color::warning, "curl failed to execute with exit code: ", res, '\n'); + } + return code; + } + + void download_file(Files::Filesystem& fs, const std::string& url, const fs::path& download_path, const std::string& sha512) { - const std::string download_path_part = fs::u8string(download_path) + ".part"; - auto download_path_part_path = fs::u8path(download_path_part); - std::error_code ec; - fs.remove(download_path, ec); - fs.remove(download_path_part_path, ec); -#if defined(_WIN32) - auto url_no_proto = url.substr(8); // drop https:// - auto path_begin = Util::find(url_no_proto, '/'); - std::string hostname(url_no_proto.begin(), path_begin); - std::string path(path_begin, url_no_proto.end()); + download_file(fs, {&url, 1}, download_path, sha512); + } - winhttp_download_file(fs, download_path_part, hostname, path); -#else - const auto code = System::cmd_execute( - Strings::format(R"(curl -L '%s' --create-dirs --output '%s')", url, download_path_part)); - Checks::check_exit(VCPKG_LINE_INFO, code == 0, "Could not download %s", url); +#if defined(_WIN32) + namespace + { + struct WriteFlushFile + { + WriteFlushFile(const fs::path& p) + { + auto err = _wfopen_s(&f, p.c_str(), L"wb"); + Checks::check_exit(VCPKG_LINE_INFO, + !err, + "Failed to open file %s. Error code was %s", + fs::u8string(p), + std::to_string(err)); + ASSUME(f != nullptr); + } + ~WriteFlushFile() + { + if (f) + { + fflush(f); + fclose(f); + } + } + FILE* f = nullptr; + }; + + /// + /// Download a file using WinHTTP -- only supports HTTP and HTTPS + /// + static bool download_winhttp(Files::Filesystem& fs, + const fs::path& download_path_part_path, + details::SplitURIView split_uri, + const std::string& url, + std::string& errors) + { + // `download_winhttp` does not support user or port syntax in authorities + auto hostname = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); + INTERNET_PORT port; + if (split_uri.scheme == "https") + { + port = INTERNET_DEFAULT_HTTPS_PORT; + } + else if (split_uri.scheme == "http") + { + port = INTERNET_DEFAULT_HTTP_PORT; + } + else + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + // Make sure the directories are present, otherwise fopen_s fails + const auto dir = download_path_part_path.parent_path(); + fs.create_directories(dir, VCPKG_LINE_INFO); + + WriteFlushFile f(download_path_part_path); + + Debug::print("Downloading ", url, "\n"); + static auto s = WinHttpSession::make().value_or_exit(VCPKG_LINE_INFO); + auto conn = WinHttpConnection::make(s.m_hSession.get(), hostname, port); + if (!conn) + { + Strings::append(errors, url, ": ", conn.error(), '\n'); + return false; + } + auto req = WinHttpRequest::make(conn.get()->m_hConnect.get(), split_uri.path_query_fragment); + if (!req) + { + Strings::append(errors, url, ": ", req.error(), '\n'); + return false; + } + auto forall_data = + req.get()->forall_data([&](Span span) { fwrite(span.data(), 1, span.size(), f.f); }); + if (!forall_data) + { + Strings::append(errors, url, ": ", forall_data.error(), '\n'); + return false; + } + return true; + } + } #endif - verify_downloaded_file_hash(fs, url, download_path_part_path, sha512); - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + std::string download_file(vcpkg::Files::Filesystem& fs, + View urls, + const fs::path& download_path, + const std::string& sha512) + { + Checks::check_exit(VCPKG_LINE_INFO, urls.size() > 0); + + auto download_path_part_path = download_path; + download_path_part_path += fs::u8path(".part"); + fs.remove(download_path, ignore_errors); + fs.remove(download_path_part_path, ignore_errors); + + std::string errors; + for (const std::string& url : urls) + { +#if defined(_WIN32) + auto split_uri = details::split_uri_view(url).value_or_exit(VCPKG_LINE_INFO); + auto authority = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); + if (split_uri.scheme == "https" || split_uri.scheme == "http") + { + // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to curl.exe + if (Strings::find_first_of(authority, ":@") == authority.end()) + { + if (download_winhttp(fs, download_path_part_path, split_uri, url, errors)) + { + verify_downloaded_file_hash(fs, url, download_path_part_path, sha512); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return url; + } + continue; + } + } +#endif + System::CmdLineBuilder cmd; + cmd.string_arg("curl") + .string_arg("--fail") + .string_arg("-L") + .string_arg(url) + .string_arg("--create-dirs") + .string_arg("--output") + .path_arg(download_path_part_path); + const auto out = System::cmd_execute_and_capture_output(cmd); + if (out.exit_code != 0) + { + Strings::append(errors, url, ": ", out.output, '\n'); + continue; + } + + verify_downloaded_file_hash(fs, url, download_path_part_path, sha512); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return url; + } + Checks::exit_with_message(VCPKG_LINE_INFO, "Failed to download from mirror set:\n%s", errors); } } diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index 5512182c53..8624cb3d31 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -16,31 +17,68 @@ using namespace vcpkg; namespace { + struct NullBinaryProvider : IBinaryProvider + { + void prefetch(const VcpkgPaths&, std::vector&) { } + + void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) { } + + RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&) + { + return RestoreResult::missing; + } + + void precheck(const VcpkgPaths&, std::unordered_map&) { } + }; +} + +std::unordered_map vcpkg::binary_provider_precheck( + const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider) +{ + std::unordered_map checked; + for (auto&& action : plan.install_actions) + checked.emplace(&action, RestoreResult::missing); + provider.precheck(paths, checked); + return checked; +} + +namespace +{ + static void clean_prepare_dir(Files::Filesystem& fs, const fs::path& dir) + { + fs.remove_all(dir, VCPKG_LINE_INFO); + bool created_last = fs.create_directories(dir, VCPKG_LINE_INFO); + Checks::check_exit(VCPKG_LINE_INFO, created_last, "unable to clear path: %s", fs::u8string(dir)); + } + static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths, - const PackageSpec& spec, + const fs::path& dst, const fs::path& archive_path) { - auto& fs = paths.get_filesystem(); - - auto pkg_path = paths.package_dir(spec); - fs.remove_all(pkg_path, VCPKG_LINE_INFO); - std::error_code ec; - fs.create_directories(pkg_path, ec); - auto files = fs.get_files_non_recursive(pkg_path); - Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", fs::u8string(pkg_path)); - + System::CmdLineBuilder cmd; #if defined(_WIN32) auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); - auto cmd = Strings::format(R"("%s" x "%s" -o"%s" -y)", - fs::u8string(seven_zip_exe), - fs::u8string(archive_path), - fs::u8string(pkg_path)); + cmd.path_arg(seven_zip_exe) + .string_arg("x") + .path_arg(archive_path) + .string_arg("-o" + fs::u8string(dst)) + .string_arg("-y"); #else - auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", fs::u8string(archive_path), fs::u8string(pkg_path)); + (void)paths; + cmd.string_arg("unzip").string_arg("-qq").path_arg(archive_path).string_arg("-d" + fs::u8string(dst)); #endif return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment()); } + static System::ExitCodeAndOutput clean_decompress_archive(const VcpkgPaths& paths, + const PackageSpec& spec, + const fs::path& archive_path) + { + auto pkg_path = paths.package_dir(spec); + clean_prepare_dir(paths.get_filesystem(), pkg_path); + return decompress_archive(paths, pkg_path, archive_path); + } + // Compress the source directory into the destination file. static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination) { @@ -66,69 +104,95 @@ namespace struct ArchivesBinaryProvider : IBinaryProvider { - ArchivesBinaryProvider(std::vector&& read_dirs, std::vector&& write_dirs) - : m_read_dirs(std::move(read_dirs)), m_write_dirs(std::move(write_dirs)) + ArchivesBinaryProvider(std::vector&& read_dirs, + std::vector&& write_dirs, + std::vector&& put_url_templates) + : m_read_dirs(std::move(read_dirs)) + , m_write_dirs(std::move(write_dirs)) + , m_put_url_templates(std::move(put_url_templates)) { } - ~ArchivesBinaryProvider() = default; - void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override { } - RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override + + void prefetch(const VcpkgPaths& paths, std::vector& actions) override { - const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; - auto& spec = action.spec; auto& fs = paths.get_filesystem(); - std::error_code ec; - for (auto&& archives_root_dir : m_read_dirs) - { - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - if (fs.exists(archive_path)) + Util::erase_remove_if(actions, [this, &fs, &paths](const Dependencies::InstallPlanAction* action) { + auto& spec = action->spec; + const auto& abi_tag = action->abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; + const auto archive_name = fs::u8path(abi_tag + ".zip"); + for (const auto& archives_root_dir : m_read_dirs) { - System::print2("Using cached binary package: ", fs::u8string(archive_path), "\n"); - - int archive_result = decompress_archive(paths, spec, archive_path).exit_code; - - if (archive_result == 0) + auto archive_path = archives_root_dir; + archive_path /= fs::u8path(abi_tag.substr(0, 2)); + archive_path /= archive_name; + if (fs.exists(archive_path)) { - return RestoreResult::success; - } - else - { - System::print2("Failed to decompress archive package\n"); - if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO) + System::print2("Using cached binary package: ", fs::u8string(archive_path), "\n"); + + int archive_result = clean_decompress_archive(paths, spec, archive_path).exit_code; + + if (archive_result == 0) { - return RestoreResult::build_failed; + m_restored.insert(spec); + return true; } else { - System::print2("Purging bad archive\n"); - fs.remove(archive_path, ec); + System::print2("Failed to decompress archive package\n"); + if (action->build_options.purge_decompress_failure == Build::PurgeDecompressFailure::YES) + { + System::print2("Purging bad archive\n"); + fs.remove(archive_path, ignore_errors); + } } } + + System::printf("Could not locate cached archive: %s\n", fs::u8string(archive_path)); } - - System::printf("Could not locate cached archive: %s\n", fs::u8string(archive_path)); - } - - return RestoreResult::missing; + return false; + }); + } + RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override + { + if (Util::Sets::contains(m_restored, action.spec)) + return RestoreResult::success; + else + return RestoreResult::missing; } void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { - if (m_write_dirs.empty()) return; + if (m_write_dirs.empty() && m_put_url_templates.empty()) return; const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; auto& spec = action.spec; auto& fs = paths.get_filesystem(); const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); compress_directory(paths, paths.package_dir(spec), tmp_archive_path); - for (auto&& m_directory : m_write_dirs) + size_t http_remotes_pushed = 0; + for (auto&& put_url_template : m_put_url_templates) { - const fs::path& archives_root_dir = m_directory; - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; + auto url = Strings::replace_all(std::string(put_url_template), "", abi_tag); + auto code = Downloads::put_file(fs, url, tmp_archive_path); + if (code >= 200 && code < 300) + { + http_remotes_pushed++; + } + else + { + Debug::print("Failed to upload to ", url, ": ", code, '\n'); + } + } + if (!m_put_url_templates.empty()) + { + System::print2("Uploaded binaries to ", http_remotes_pushed, " HTTP remotes.\n"); + } + const auto archive_name = fs::u8path(abi_tag + ".zip"); + for (const auto& archives_root_dir : m_write_dirs) + { + auto archive_path = archives_root_dir; + archive_path /= fs::u8path(abi_tag.substr(0, 2)); + archive_path /= archive_name; fs.create_directories(archive_path.parent_path(), ignore_errors); std::error_code ec; if (m_write_dirs.size() > 1) @@ -145,29 +209,143 @@ namespace else System::printf("Stored binary cache: %s\n", fs::u8string(archive_path)); } - if (m_write_dirs.size() > 1) fs.remove(tmp_archive_path, ignore_errors); - } - RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override - { - const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; - auto& fs = paths.get_filesystem(); - std::error_code ec; - for (auto&& archives_root_dir : m_read_dirs) + // In the case of 1 write dir, the file will be moved instead of copied + if (m_write_dirs.size() != 1) { - const std::string archive_name = abi_tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; + fs.remove(tmp_archive_path, ignore_errors); + } + } + void precheck(const VcpkgPaths& paths, + std::unordered_map& results_map) override + { + auto& fs = paths.get_filesystem(); - if (fs.exists(archive_path)) + for (auto&& result_pair : results_map) + { + if (result_pair.second != RestoreResult::missing) continue; + + const auto& abi_tag = result_pair.first->abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; + std::error_code ec; + for (auto&& archives_root_dir : m_read_dirs) { - return RestoreResult::success; + const std::string archive_name = abi_tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + + if (fs.exists(archive_path)) + { + result_pair.second = RestoreResult::success; + break; + } } } - return RestoreResult::missing; } private: - std::vector m_read_dirs, m_write_dirs; + std::vector m_read_dirs; + std::vector m_write_dirs; + std::vector m_put_url_templates; + + std::set m_restored; + }; + struct HttpGetBinaryProvider : NullBinaryProvider + { + HttpGetBinaryProvider(std::vector&& url_templates) : m_url_templates(std::move(url_templates)) { } + void prefetch(const VcpkgPaths& paths, std::vector& actions) override + { + auto& fs = paths.get_filesystem(); + + const size_t current_restored = m_restored.size(); + + for (auto&& url_template : m_url_templates) + { + std::vector> url_paths; + std::vector specs; + + for (auto&& action : actions) + { + auto abi = action->package_abi(); + if (!abi) continue; + + specs.push_back(action->spec); + auto pkgdir = paths.package_dir(action->spec); + clean_prepare_dir(fs, pkgdir); + pkgdir /= fs::u8path(Strings::concat(*abi.get(), ".zip")); + url_paths.emplace_back(Strings::replace_all(std::string(url_template), "", *abi.get()), + pkgdir); + } + + if (url_paths.empty()) break; + + System::print2("Attempting to fetch ", url_paths.size(), " packages from HTTP servers.\n"); + + auto codes = Downloads::download_files(fs, url_paths); + for (size_t i = 0; i < codes.size(); ++i) + { + if (codes[i] == 200) + { + int archive_result = + decompress_archive(paths, paths.package_dir(specs[i]), url_paths[i].second).exit_code; + if (archive_result == 0) + { + // decompression success + fs.remove(url_paths[i].second, VCPKG_LINE_INFO); + m_restored.insert(specs[i]); + } + else + { + Debug::print("Failed to decompress ", fs::u8string(url_paths[i].second), '\n'); + } + } + } + + Util::erase_remove_if(actions, [this](const Dependencies::InstallPlanAction* action) { + return Util::Sets::contains(m_restored, action->spec); + }); + } + System::print2("Restored ", + m_restored.size() - current_restored, + " packages from HTTP servers. Use --debug for more information.\n"); + } + RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override + { + if (Util::Sets::contains(m_restored, action.spec)) + return RestoreResult::success; + else + return RestoreResult::missing; + } + void precheck(const VcpkgPaths&, + std::unordered_map& results_map) override + { + std::vector urls; + std::vector url_actions; + for (auto&& url_template : m_url_templates) + { + urls.clear(); + url_actions.clear(); + for (auto&& result_pair : results_map) + { + if (result_pair.second != RestoreResult::missing) continue; + auto abi = result_pair.first->package_abi(); + if (!abi) continue; + urls.push_back(Strings::replace_all(std::string(url_template), "", *abi.get())); + url_actions.push_back(result_pair.first); + } + if (urls.empty()) break; + auto codes = Downloads::url_heads(urls); + Checks::check_exit(VCPKG_LINE_INFO, codes.size() == url_actions.size()); + for (size_t i = 0; i < codes.size(); ++i) + { + if (codes[i] == 200) + { + results_map[url_actions[i]] = RestoreResult::success; + } + } + } + } + + std::vector m_url_templates; + std::set m_restored; }; static std::string trim_leading_zeroes(std::string v) @@ -184,7 +362,7 @@ namespace return v; } - struct NugetBinaryProvider : IBinaryProvider + struct NugetBinaryProvider : NullBinaryProvider { NugetBinaryProvider(std::vector&& read_sources, std::vector&& write_sources, @@ -198,7 +376,8 @@ namespace , m_interactive(interactive) { } - void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override + + void prefetch(const VcpkgPaths& paths, std::vector& actions) override { if (m_read_sources.empty() && m_read_configs.empty()) return; @@ -206,14 +385,14 @@ namespace std::vector> nuget_refs; - for (auto&& action : plan.install_actions) + for (auto&& action : actions) { - if (!action.has_package_abi()) continue; + if (!action->has_package_abi()) continue; - auto& spec = action.spec; + auto& spec = action->spec; fs.remove_all(paths.package_dir(spec), VCPKG_LINE_INFO); - nuget_refs.emplace_back(spec, NugetReference(action)); + nuget_refs.emplace_back(spec, NugetReference(*action)); } if (nuget_refs.empty()) return; @@ -294,7 +473,7 @@ namespace cmdlines.push_back(cmdline.extract()); } - size_t num_restored = 0; + const size_t current_restored = m_restored.size(); for (const auto& cmdline : cmdlines) { @@ -327,7 +506,6 @@ namespace "Unable to remove nupkg after restoring: %s", fs::u8string(nupkg_path)); m_restored.emplace(nuget_ref.first); - ++num_restored; return true; } else @@ -337,7 +515,13 @@ namespace }); } - System::print2("Restored ", num_restored, " packages. Use --debug for more information.\n"); + Util::erase_remove_if(actions, [this](const Dependencies::InstallPlanAction* action) { + return Util::Sets::contains(m_restored, action->spec); + }); + + System::print2("Restored ", + m_restored.size() - current_restored, + " packages from NuGet. Use --debug for more information.\n"); } RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override { @@ -450,10 +634,6 @@ namespace paths.get_filesystem().remove(nupkg_path, ignore_errors); } } - RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override - { - return RestoreResult::missing; - } private: std::vector m_read_sources; @@ -465,19 +645,22 @@ namespace std::set m_restored; bool m_interactive; }; +} - struct MergeBinaryProviders : IBinaryProvider +namespace vcpkg +{ + struct MergeBinaryProviders : NullBinaryProvider { explicit MergeBinaryProviders(std::vector>&& providers) : m_providers(std::move(providers)) { } - void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override + void prefetch(const VcpkgPaths& paths, std::vector& actions) override { for (auto&& provider : m_providers) { - provider->prefetch(paths, plan); + provider->prefetch(paths, actions); } } RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override @@ -502,39 +685,18 @@ namespace provider->push_success(paths, action); } } - RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override + void precheck(const VcpkgPaths& paths, + std::unordered_map& results_map) override { for (auto&& provider : m_providers) { - auto result = provider->precheck(paths, action); - switch (result) - { - case RestoreResult::build_failed: - case RestoreResult::success: return result; - case RestoreResult::missing: continue; - default: Checks::unreachable(VCPKG_LINE_INFO); - } + provider->precheck(paths, results_map); } - return RestoreResult::missing; } private: std::vector> m_providers; }; - - struct NullBinaryProvider : IBinaryProvider - { - void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override { } - RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override - { - return RestoreResult::missing; - } - void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override { } - RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override - { - return RestoreResult::missing; - } - }; } XmlSerializer& XmlSerializer::emit_declaration() @@ -650,12 +812,6 @@ IBinaryProvider& vcpkg::null_binary_provider() return p; } -ExpectedS> vcpkg::create_binary_provider_from_configs(View args) -{ - std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or(""); - - return create_binary_provider_from_configs_pure(env_string, args); -} namespace { const ExpectedS& default_cache_path() @@ -692,10 +848,6 @@ namespace return {"default path was not absolute: " + fs::u8string(p), expected_right_tag}; } }); - if (cachepath.has_value()) - Debug::print("Default binary cache path is: ", fs::u8string(*cachepath.get()), '\n'); - else - Debug::print("No binary cache path. Reason: ", cachepath.error(), '\n'); return cachepath; } @@ -707,6 +859,9 @@ namespace std::vector archives_to_read; std::vector archives_to_write; + std::vector url_templates_to_get; + std::vector azblob_templates_to_put; + std::vector sources_to_read; std::vector sources_to_write; @@ -719,6 +874,8 @@ namespace interactive = false; archives_to_read.clear(); archives_to_write.clear(); + url_templates_to_get.clear(); + azblob_templates_to_put.clear(); sources_to_read.clear(); sources_to_write.clear(); configs_to_read.clear(); @@ -901,6 +1058,40 @@ namespace handle_readwrite( state->archives_to_read, state->archives_to_write, fs::path(*maybe_home.get()), segments, 1); } + else if (segments[0].second == "x-azblob") + { + // Scheme: x-azblob,,[,] + if (segments.size() < 3) + { + return add_error( + "expected arguments: binary config 'azblob' requires at least a base-url and a SAS token", + segments[0].first); + } + if (!Strings::starts_with(segments[1].second, "https://")) + { + return add_error( + "invalid argument: binary config 'azblob' requires an https base url as the first argument", + segments[1].first); + } + if (Strings::starts_with(segments[2].second, "?")) + { + return add_error("invalid argument: binary config 'azblob' requires a SAS token without a " + "preceeding '?' as the second argument", + segments[2].first); + } + auto p = segments[1].second; + if (p.back() != '/') p.push_back('/'); + p.append(".zip"); + if (!Strings::starts_with(segments[2].second, "?")) p.push_back('?'); + p.append(segments[2].second); + handle_readwrite( + state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3); + if (segments.size() > 4) + { + return add_error("unexpected arguments: binary config 'azblob' requires 2 or 3 arguments", + segments[4].first); + } + } else { return add_error( @@ -912,6 +1103,21 @@ namespace }; } +ExpectedS> vcpkg::create_binary_provider_from_configs(View args) +{ + std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or(""); + if (Debug::g_debugging) + { + const auto& cachepath = default_cache_path(); + if (cachepath.has_value()) + Debug::print("Default binary cache path is: ", fs::u8string(*cachepath.get()), '\n'); + else + Debug::print("No binary cache path. Reason: ", cachepath.error(), '\n'); + } + + return create_binary_provider_from_configs_pure(env_string, args); +} + ExpectedS> vcpkg::create_binary_provider_from_configs_pure( const std::string& env_string, View args) { @@ -939,9 +1145,16 @@ ExpectedS> vcpkg::create_binary_provider_from_c if (s.m_cleared) Metrics::g_metrics.lock()->track_property("binarycaching-clear", "defined"); std::vector> providers; - if (!s.archives_to_read.empty() || !s.archives_to_write.empty()) - providers.push_back( - std::make_unique(std::move(s.archives_to_read), std::move(s.archives_to_write))); + if (!s.archives_to_read.empty() || !s.archives_to_write.empty() || !s.azblob_templates_to_put.empty()) + { + providers.push_back(std::make_unique( + std::move(s.archives_to_read), std::move(s.archives_to_write), std::move(s.azblob_templates_to_put))); + } + if (!s.url_templates_to_get.empty()) + { + Metrics::g_metrics.lock()->track_property("binarycaching-url-get", "defined"); + providers.push_back(std::make_unique(std::move(s.url_templates_to_get))); + } if (!s.sources_to_read.empty() || !s.sources_to_write.empty() || !s.configs_to_read.empty() || !s.configs_to_write.empty()) { @@ -1088,6 +1301,9 @@ void vcpkg::help_topic_binary_caching(const VcpkgPaths&) tbl.format("nugetconfig,[,]", "Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This " "config should specify `defaultPushSource` for uploads."); + tbl.format("x-azblob,,[,]", + "**Experimental: will change or be removed without warning** Adds an Azure Blob Storage source. Uses " + "Shared Access Signature validation. URL should include the container path."); tbl.format("interactive", "Enables interactive credential management for some source types"); tbl.blank(); tbl.text("The `` optional parameter for certain strings controls whether they will be consulted for " diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 1d83f471b9..ee6222d6ed 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -332,6 +332,8 @@ namespace vcpkg::Commands::CI { vcpkg::System::BufferedPrint stdout_print; + auto precheck_results = binary_provider_precheck(paths, action_plan, binaryprovider); + for (auto&& action : action_plan.install_actions) { auto p = &action; @@ -345,7 +347,7 @@ namespace vcpkg::Commands::CI p->build_options = vcpkg::Build::backcompat_prohibiting_package_options; } - auto precheck_result = binaryprovider.precheck(paths, action); + auto precheck_result = precheck_results.at(&action); bool b_will_build = false; std::string state; diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index 69db32e27c..75509343a3 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -433,6 +433,12 @@ namespace vcpkg::Dependencies if (!abi_info) return false; return !abi_info.get()->package_abi.empty(); } + Optional InstallPlanAction::package_abi() const + { + if (!abi_info) return nullopt; + if (abi_info.get()->package_abi.empty()) return nullopt; + return abi_info.get()->package_abi; + } const Build::PreBuildInfo& InstallPlanAction::pre_build_info(LineInfo linfo) const { return *abi_info.value_or_exit(linfo).pre_build_info.get(); diff --git a/toolsrc/src/vcpkg/globalstate.cpp b/toolsrc/src/vcpkg/globalstate.cpp index b07690bbd3..896e96c81f 100644 --- a/toolsrc/src/vcpkg/globalstate.cpp +++ b/toolsrc/src/vcpkg/globalstate.cpp @@ -1,3 +1,5 @@ +#include + #include namespace vcpkg diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 2abe20774f..f3da9c5d67 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -481,7 +481,8 @@ namespace vcpkg::Install Build::compute_all_abis(paths, action_plan, var_provider, status_db); - binaryprovider.prefetch(paths, action_plan); + auto to_prefetch = Util::fmap(action_plan.install_actions, [](const auto& x) { return &x; }); + binaryprovider.prefetch(paths, to_prefetch); for (auto&& action : action_plan.install_actions) {