[vcpkg] Add experimental x-azblob binary provider (#13626)

* [vcpkg] Add experimental x-azblob binary provider

* [vcpkg] Test azblob storage provider in CI

* [vcpkg] Address some CR comments from #13639

* [vcpkg] Fixup azure-pipelines

* [vcpkg] Fix regression where the downloaded package is purged before decompressing

* [vcpkg] Further refactor vcpkg::Downloads

* [vcpkg] Enable OSX for x-azblob testing

* [vcpkg] Reduce diff against master

* [vcpkg] Extract Downloads::details::split_uri_view

* [vcpkg] Address PR comments

* [vcpkg] Add testing and metrics for x-azblob

* [vcpkg] Add docs for x-azblob

This includes a note that it is currently experimental

* [vcpkg] Address CR comments

* [vcpkg] Revert pipeline changes except OSX to minimize disruption

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

docs/users/binarycaching.md

@@ -108,7 +108,8 @@ By default, zip-based archives will be cached at the first valid location of:
 | `default[,<rw>]`            | Adds the default file-based location
 | `files,<path>[,<rw>]`       | Adds a custom file-based location
 | `nuget,<uri>[,<rw>]`        | Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI
-| `nugetconfig,<path>[,<rw>]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter <br>of the NuGet CLI. This config should specify `defaultPushSource` for uploads.
+| `nugetconfig,<path>[,<rw>]` | Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This config should specify `defaultPushSource` for uploads.
+| `x-azblob,<baseuri>,<sas>[,<rw>]`    | **Experimental: will change or be removed without warning**<br> Adds an Azure Blob Storage source. Uses Shared Access Signature validation. URL should include the container path.
 | `interactive`               | Enables interactive credential management for NuGet (for debugging; requires `--debug` on the command line)
 
 The `<rw>` optional parameter for certain sources controls whether they will be consulted for

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -14,6 +14,9 @@ jobs:
     value: /Users/vagrant/Data
   - name: VCPKG_DOWNLOADS
     value: /Users/vagrant/Data/downloads
+  - group: azblob-test-sas-group
+  - name: BINARY_SOURCE_STUB
+    value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"
 
   steps:
   - bash: df -h
@@ -40,7 +43,7 @@ jobs:
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -BinarySourceStub "$(BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
   - bash: |
       df -h
     displayName: 'Report on Disk Space After Build'

scripts/azure-pipelines/test-modified-ports.ps1

@@ -16,8 +16,11 @@ The location used as scratch space for 'installed', 'packages', and 'buildtrees'
 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
 
 .PARAMETER ArchivesRoot
-The location where the binary caching archives are stored. Shared across runs of this script. If
-this parameter is not set, binary caching will not be used.
+Equivalent to '-BinarySourceStub "files,$ArchivesRoot"'
+
+.PARAMETER BinarySourceStub
+The type and parameters of the binary source. Shared across runs of this script. If
+this parameter is not set, binary caching will not be used. Example: "files,W:\"
 
 .PARAMETER BuildReason
 The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).
@@ -25,7 +28,7 @@ If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set
 binary caching will default to read-write mode.
 #>
 
-[CmdletBinding()]
+[CmdletBinding(DefaultParameterSetName="ArchivesRoot")]
 Param(
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
@@ -35,7 +38,10 @@ Param(
     $WorkingRoot,
     [ValidateNotNullOrEmpty()]
     $ArtifactStagingDirectory = '.',
+    [Parameter(ParameterSetName='ArchivesRoot')]
     $ArchivesRoot = $null,
+    [Parameter(ParameterSetName='BinarySourceStub')]
+    $BinarySourceStub = $null,
     $BuildReason = $null
 )
 
@@ -49,7 +55,7 @@ $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'packages'
 
-$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot))
+$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) -Or -Not ([string]::IsNullOrWhiteSpace($BinarySourceStub))
 $commonArgs = @()
 if ($usingBinaryCaching) {
     $commonArgs += @('--binarycaching')
@@ -78,8 +84,12 @@ if ($usingBinaryCaching) {
         Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
         $binaryCachingMode = 'write'
     }
-
-    $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+    if ([string]::IsNullOrWhiteSpace($ArchivesRoot)) {
+        $commonArgs += @("--binarysource=clear;$BinarySourceStub,$binaryCachingMode")
+    }
+    else {
+        $commonArgs += @("--binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+    }
 }
 
 if ($Triplet -eq 'x64-linux') {

toolsrc/include/vcpkg/base/downloads.h

@@ -1,16 +1,41 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/optional.h>
+#include <vcpkg/base/view.h>
 
 namespace vcpkg::Downloads
 {
+    namespace details
+    {
+        struct SplitURIView
+        {
+            StringView scheme;
+            Optional<StringView> authority;
+            StringView path_query_fragment;
+        };
+
+        // e.g. {"https","//example.org", "/index.html"}
+        Optional<SplitURIView> split_uri_view(StringView uri);
+    }
+
     void verify_downloaded_file_hash(const Files::Filesystem& fs,
                                      const std::string& url,
                                      const fs::path& path,
                                      const std::string& sha512);
 
+    // Returns url that was successfully downloaded from
+    std::string download_file(Files::Filesystem& fs,
+                              View<std::string> urls,
+                              const fs::path& download_path,
+                              const std::string& sha512);
+
     void download_file(Files::Filesystem& fs,
                        const std::string& url,
                        const fs::path& download_path,
                        const std::string& sha512);
+
+    std::vector<int> download_files(Files::Filesystem& fs, View<std::pair<std::string, fs::path>> url_pairs);
+    int put_file(const Files::Filesystem&, StringView url, const fs::path& file);
+    std::vector<int> url_heads(View<std::string> urls);
 }

toolsrc/include/vcpkg/base/fwd/lockguarded.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuardPtr;
+
+    template<class T>
+    struct LockGuarded;
+}

toolsrc/include/vcpkg/base/lockguarded.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vcpkg/base/fwd/lockguarded.h>
+
+#include <mutex>
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuarded
+    {
+        friend struct LockGuardPtr<T>;
+
+        LockGuardPtr<T> lock() { return *this; }
+
+    private:
+        std::mutex m_mutex;
+        T m_t;
+    };
+
+    template<class T>
+    struct LockGuardPtr
+    {
+        T& operator*() { return m_ptr; }
+        T* operator->() { return &m_ptr; }
+
+        T* get() { return &m_ptr; }
+
+        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
+
+    private:
+        std::lock_guard<std::mutex> m_lock;
+        T& m_ptr;
+    };
+}

toolsrc/include/vcpkg/base/util.h

@@ -5,7 +5,6 @@
 #include <algorithm>
 #include <functional>
 #include <map>
-#include <mutex>
 #include <type_traits>
 #include <unordered_map>
 #include <utility>
@@ -222,36 +221,6 @@ namespace vcpkg::Util
         ~ResourceBase() = default;
     };
 
-    template<class T>
-    struct LockGuardPtr;
-
-    template<class T>
-    struct LockGuarded
-    {
-        friend struct LockGuardPtr<T>;
-
-        LockGuardPtr<T> lock() { return *this; }
-
-    private:
-        std::mutex m_mutex;
-        T m_t;
-    };
-
-    template<class T>
-    struct LockGuardPtr
-    {
-        T& operator*() { return m_ptr; }
-        T* operator->() { return &m_ptr; }
-
-        T* get() { return &m_ptr; }
-
-        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
-
-    private:
-        std::unique_lock<std::mutex> m_lock;
-        T& m_ptr;
-    };
-
     namespace Enum
     {
         template<class E>

toolsrc/include/vcpkg/binarycaching.h

@@ -8,8 +8,12 @@
 
 #include <vcpkg/packagespec.h>
 
+#include <unordered_map>
+
 namespace vcpkg
 {
+    struct MergeBinaryProviders;
+
     enum class RestoreResult
     {
         missing,
@@ -20,18 +24,32 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
-        /// `plan`.
-        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+
         /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
         /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
-        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
-        /// missing packages.
-        virtual RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
+        /// <summary>Gives the BinaryProvider an opportunity to batch any downloading or server communication for
+        /// executing `plan`.</summary>
+        /// <remarks>Must only be called once for a given binary provider instance</remarks>
+        /// <param name="actions">InOut vector of actions to be prefetched</param>
+        virtual void prefetch(const VcpkgPaths& paths,
+                              std::vector<const Dependencies::InstallPlanAction*>& actions) = 0;
+
+        /// <summary>Requests the result of <c>try_restore()</c> without actually downloading the package. Used by CI to
+        /// determine missing packages.</summary>
+        /// <param name="results_map">InOut map to track the restored packages. Should be initialized to
+        /// <c>{&amp;action, RestoreResult::missing}</c> for all install actions</param>
+        virtual void precheck(
+            const VcpkgPaths& paths,
+            std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) = 0;
     };
 
+    std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> binary_provider_precheck(
+        const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider);
+
     IBinaryProvider& null_binary_provider();
 
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(View<std::string> args);

toolsrc/include/vcpkg/dependencies.h

@@ -61,6 +61,7 @@ namespace vcpkg::Dependencies
         std::string displayname() const;
         const std::string& public_abi() const;
         bool has_package_abi() const;
+        Optional<const std::string&> package_abi() const;
         const Build::PreBuildInfo& pre_build_info(LineInfo linfo) const;
 
         PackageSpec spec;

toolsrc/include/vcpkg/globalstate.h

@@ -1,7 +1,8 @@
 #pragma once
 
+#include <vcpkg/base/fwd/lockguarded.h>
+
 #include <vcpkg/base/chrono.h>
-#include <vcpkg/base/util.h>
 
 #include <atomic>
 #include <string>

toolsrc/include/vcpkg/metrics.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/lockguarded.h>
 #include <vcpkg/base/util.h>
 
 #include <string>

toolsrc/src/vcpkg-test/binaryconfigparser.cpp

@@ -298,3 +298,39 @@ TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]")
         REQUIRE(parsed.has_value());
     }
 }
+
+TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,?sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,,sas", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,invalid", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,read", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,write", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("x-azblob,https://azure/container,sas,readwrite", {});
+        REQUIRE(parsed.has_value());
+    }
+}

toolsrc/src/vcpkg-test/downloads.cpp

@@ -0,0 +1,59 @@
+#include <catch2/catch.hpp>
+
+#include <vcpkg/base/downloads.h>
+
+using namespace vcpkg;
+
+TEST_CASE ("Downloads::details::split_uri_view", "[downloads]")
+{
+    {
+        auto x = Downloads::details::split_uri_view("https://github.com/Microsoft/vcpkg");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "https");
+        REQUIRE(x.get()->authority.value_or("") == "//github.com");
+        REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("hello");
+        REQUIRE(!x.has_value());
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file:/path");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(!x.get()->authority.has_value());
+        REQUIRE(x.get()->path_query_fragment == "/path");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("file://user:pw@host");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "file");
+        REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host");
+        REQUIRE(x.get()->path_query_fragment == "");
+    }
+    {
+        auto x = Downloads::details::split_uri_view("ftp://host:port/");
+        REQUIRE(x.has_value());
+        REQUIRE(x.get()->scheme == "ftp");
+        REQUIRE(x.get()->authority.value_or({}) == "//host:port");
+        REQUIRE(x.get()->path_query_fragment == "/");
+    }
+}

toolsrc/src/vcpkg/base/downloads.cpp

@@ -1,6 +1,10 @@
+#include <vcpkg/base/cache.h>
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/hash.h>
+#include <vcpkg/base/lockguarded.h>
+#include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.h>
+#include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
 
@@ -11,133 +15,178 @@
 namespace vcpkg::Downloads
 {
 #if defined(_WIN32)
-    static void winhttp_download_file(Files::Filesystem& fs,
-                                      ZStringView target_file_path,
-                                      StringView hostname,
-                                      StringView url_path)
+    struct WinHttpHandleDeleter
     {
-        // Make sure the directories are present, otherwise fopen_s fails
-        const auto dir = fs::path(target_file_path.c_str()).parent_path();
-        std::error_code ec;
-        fs.create_directories(dir, ec);
-        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not create directories %s", fs::u8string(dir));
+        void operator()(HINTERNET h) const { WinHttpCloseHandle(h); }
+    };
 
-        FILE* f = nullptr;
-        const errno_t err = fopen_s(&f, target_file_path.c_str(), "wb");
-        Checks::check_exit(VCPKG_LINE_INFO,
-                           !err,
-                           "Could not download https://%s%s. Failed to open file %s. Error code was %s",
-                           hostname,
-                           url_path,
-                           target_file_path,
-                           std::to_string(err));
-        ASSUME(f != nullptr);
-
-        auto hSession = WinHttpOpen(L"vcpkg/1.0",
-                                    IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY
-                                                                : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
-                                    WINHTTP_NO_PROXY_NAME,
-                                    WINHTTP_NO_PROXY_BYPASS,
-                                    0);
-        Checks::check_exit(VCPKG_LINE_INFO, hSession, "WinHttpOpen() failed: %d", GetLastError());
-
-        // If the environment variable HTTPS_PROXY is set
-        // use that variable as proxy. This situation might exist when user is in a company network
-        // with restricted network/proxy settings
-        auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY");
-        if (auto p_https_proxy = maybe_https_proxy_env.get())
+    struct WinHttpRequest
+    {
+        static ExpectedS<WinHttpRequest> make(HINTERNET hConnect, StringView url_path, const wchar_t* method = L"GET")
         {
-            std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy);
-            WINHTTP_PROXY_INFO proxy;
-            proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY;
-            proxy.lpszProxy = env_proxy_settings.data();
-            proxy.lpszProxyBypass = nullptr;
+            WinHttpRequest ret;
+            // Create an HTTP request handle.
+            auto h = WinHttpOpenRequest(hConnect,
+                                        method,
+                                        Strings::to_utf16(url_path).c_str(),
+                                        nullptr,
+                                        WINHTTP_NO_REFERER,
+                                        WINHTTP_DEFAULT_ACCEPT_TYPES,
+                                        WINHTTP_FLAG_SECURE);
+            if (!h) return Strings::concat("WinHttpOpenRequest() failed: ", GetLastError());
+            ret.m_hRequest.reset(h);
 
-            WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy));
+            // Send a request.
+            auto bResults = WinHttpSendRequest(
+                ret.m_hRequest.get(), WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0);
+
+            if (!bResults) return Strings::concat("WinHttpSendRequest() failed: ", GetLastError());
+
+            // End the request.
+            bResults = WinHttpReceiveResponse(ret.m_hRequest.get(), NULL);
+            if (!bResults) return Strings::concat("WinHttpReceiveResponse() failed: ", GetLastError());
+
+            DWORD dwStatusCode = 0;
+            DWORD dwSize = sizeof(dwStatusCode);
+
+            bResults = WinHttpQueryHeaders(ret.m_hRequest.get(),
+                                           WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER,
+                                           WINHTTP_HEADER_NAME_BY_INDEX,
+                                           &dwStatusCode,
+                                           &dwSize,
+                                           WINHTTP_NO_HEADER_INDEX);
+            if (!bResults) return Strings::concat("WinHttpQueryHeaders() failed: ", GetLastError());
+            if (dwStatusCode < 200 || dwStatusCode >= 300) return Strings::concat("failed: status code ", dwStatusCode);
+
+            return std::move(ret);
         }
-        // Win7 IE Proxy fallback
-        else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater())
-        {
-            // First check if any proxy has been found automatically
-            WINHTTP_PROXY_INFO proxyInfo;
-            DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO);
-            auto noProxyFound = !WinHttpQueryOption(hSession, WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) ||
-                                proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY;
 
-            // If no proxy was found automatically, use IE's proxy settings, if any
-            if (noProxyFound)
+        template<class F>
+        ExpectedS<int> forall_data(F f)
+        {
+            std::vector<char> buf;
+
+            size_t total_downloaded_size = 0;
+            DWORD dwSize = 0;
+            do
             {
-                WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy;
-                if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr)
+                DWORD downloaded_size = 0;
+                auto bResults = WinHttpQueryDataAvailable(m_hRequest.get(), &dwSize);
+                if (!bResults) return Strings::concat("WinHttpQueryDataAvailable() failed: ", GetLastError());
+
+                if (buf.size() < dwSize) buf.resize(static_cast<size_t>(dwSize) * 2);
+
+                bResults = WinHttpReadData(m_hRequest.get(), (LPVOID)buf.data(), dwSize, &downloaded_size);
+                if (!bResults) return Strings::concat("WinHttpReadData() failed: ", GetLastError());
+                f(Span<char>(buf.data(), downloaded_size));
+
+                total_downloaded_size += downloaded_size;
+            } while (dwSize > 0);
+            return 1;
+        }
+
+        std::unique_ptr<void, WinHttpHandleDeleter> m_hRequest;
+    };
+
+    struct WinHttpSession
+    {
+        static ExpectedS<WinHttpSession> make()
+        {
+            auto h = WinHttpOpen(L"vcpkg/1.0",
+                                 IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY
+                                                             : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
+                                 WINHTTP_NO_PROXY_NAME,
+                                 WINHTTP_NO_PROXY_BYPASS,
+                                 0);
+            if (!h) return Strings::concat("WinHttpOpen() failed: ", GetLastError());
+            WinHttpSession ret;
+            ret.m_hSession.reset(h);
+
+            // If the environment variable HTTPS_PROXY is set
+            // use that variable as proxy. This situation might exist when user is in a company network
+            // with restricted network/proxy settings
+            auto maybe_https_proxy_env = System::get_environment_variable("HTTPS_PROXY");
+            if (auto p_https_proxy = maybe_https_proxy_env.get())
+            {
+                std::wstring env_proxy_settings = Strings::to_utf16(*p_https_proxy);
+                WINHTTP_PROXY_INFO proxy;
+                proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY;
+                proxy.lpszProxy = env_proxy_settings.data();
+                proxy.lpszProxyBypass = nullptr;
+
+                WinHttpSetOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy));
+            }
+            // Win7 IE Proxy fallback
+            else if (IsWindows7OrGreater() && !IsWindows8Point1OrGreater())
+            {
+                // First check if any proxy has been found automatically
+                WINHTTP_PROXY_INFO proxyInfo;
+                DWORD proxyInfoSize = sizeof(WINHTTP_PROXY_INFO);
+                auto noProxyFound =
+                    !WinHttpQueryOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxyInfo, &proxyInfoSize) ||
+                    proxyInfo.dwAccessType == WINHTTP_ACCESS_TYPE_NO_PROXY;
+
+                // If no proxy was found automatically, use IE's proxy settings, if any
+                if (noProxyFound)
                 {
-                    WINHTTP_PROXY_INFO proxy;
-                    proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY;
-                    proxy.lpszProxy = ieProxy.lpszProxy;
-                    proxy.lpszProxyBypass = ieProxy.lpszProxyBypass;
-                    WinHttpSetOption(hSession, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy));
-                    GlobalFree(ieProxy.lpszProxy);
-                    GlobalFree(ieProxy.lpszProxyBypass);
-                    GlobalFree(ieProxy.lpszAutoConfigUrl);
+                    WINHTTP_CURRENT_USER_IE_PROXY_CONFIG ieProxy;
+                    if (WinHttpGetIEProxyConfigForCurrentUser(&ieProxy) && ieProxy.lpszProxy != nullptr)
+                    {
+                        WINHTTP_PROXY_INFO proxy;
+                        proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY;
+                        proxy.lpszProxy = ieProxy.lpszProxy;
+                        proxy.lpszProxyBypass = ieProxy.lpszProxyBypass;
+                        WinHttpSetOption(ret.m_hSession.get(), WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy));
+                        GlobalFree(ieProxy.lpszProxy);
+                        GlobalFree(ieProxy.lpszProxyBypass);
+                        GlobalFree(ieProxy.lpszAutoConfigUrl);
+                    }
                 }
             }
+
+            // Use Windows 10 defaults on Windows 7
+            DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 |
+                                   WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2);
+            WinHttpSetOption(
+                ret.m_hSession.get(), WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols));
+
+            return ret;
         }
 
-        // Use Windows 10 defaults on Windows 7
-        DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_SSL3 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 |
-                               WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2);
-        WinHttpSetOption(hSession, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols));
+        std::unique_ptr<void, WinHttpHandleDeleter> m_hSession;
+    };
 
-        // Specify an HTTP server.
-        auto hConnect = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), INTERNET_DEFAULT_HTTPS_PORT, 0);
-        Checks::check_exit(VCPKG_LINE_INFO, hConnect, "WinHttpConnect() failed: %d", GetLastError());
-
-        // Create an HTTP request handle.
-        auto hRequest = WinHttpOpenRequest(hConnect,
-                                           L"GET",
-                                           Strings::to_utf16(url_path).c_str(),
-                                           nullptr,
-                                           WINHTTP_NO_REFERER,
-                                           WINHTTP_DEFAULT_ACCEPT_TYPES,
-                                           WINHTTP_FLAG_SECURE);
-        Checks::check_exit(VCPKG_LINE_INFO, hRequest, "WinHttpOpenRequest() failed: %d", GetLastError());
-
-        // Send a request.
-        auto bResults =
-            WinHttpSendRequest(hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0);
-
-        Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpSendRequest() failed: %d", GetLastError());
-
-        // End the request.
-        bResults = WinHttpReceiveResponse(hRequest, NULL);
-        Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReceiveResponse() failed: %d", GetLastError());
-
-        std::vector<char> buf;
-
-        size_t total_downloaded_size = 0;
-        DWORD dwSize = 0;
-        do
+    struct WinHttpConnection
+    {
+        static ExpectedS<WinHttpConnection> make(HINTERNET hSession, StringView hostname, INTERNET_PORT port)
         {
-            DWORD downloaded_size = 0;
-            bResults = WinHttpQueryDataAvailable(hRequest, &dwSize);
-            Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpQueryDataAvailable() failed: %d", GetLastError());
+            // Specify an HTTP server.
+            auto h = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), port, 0);
+            if (!h) return Strings::concat("WinHttpConnect() failed: ", GetLastError());
+            WinHttpConnection ret;
+            ret.m_hConnect.reset(h);
+            return ret;
+        }
 
-            if (buf.size() < dwSize) buf.resize(static_cast<size_t>(dwSize) * 2);
-
-            bResults = WinHttpReadData(hRequest, (LPVOID)buf.data(), dwSize, &downloaded_size);
-            Checks::check_exit(VCPKG_LINE_INFO, bResults, "WinHttpReadData() failed: %d", GetLastError());
-            fwrite(buf.data(), 1, downloaded_size, f);
-
-            total_downloaded_size += downloaded_size;
-        } while (dwSize > 0);
-
-        WinHttpCloseHandle(hSession);
-        WinHttpCloseHandle(hConnect);
-        WinHttpCloseHandle(hRequest);
-        fflush(f);
-        fclose(f);
-    }
+        std::unique_ptr<void, WinHttpHandleDeleter> m_hConnect;
+    };
 #endif
 
+    Optional<details::SplitURIView> details::split_uri_view(StringView uri)
+    {
+        auto sep = std::find(uri.begin(), uri.end(), ':');
+        if (sep == uri.end()) return nullopt;
+
+        StringView scheme(uri.begin(), sep);
+        if (Strings::starts_with({sep + 1, uri.end()}, "//"))
+        {
+            auto path_start = std::find(sep + 3, uri.end(), '/');
+            return details::SplitURIView{scheme, StringView{sep + 1, path_start}, {path_start, uri.end()}};
+        }
+        // no authority
+        return details::SplitURIView{scheme, {}, {sep + 1, uri.end()}};
+    }
+
     void verify_downloaded_file_hash(const Files::Filesystem& fs,
                                      const std::string& url,
                                      const fs::path& path,
@@ -167,30 +216,251 @@ namespace vcpkg::Downloads
                            actual_hash);
     }
 
-    void download_file(vcpkg::Files::Filesystem& fs,
+    static void url_heads_inner(View<std::string> urls, std::vector<int>* out)
+    {
+        static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e";
+
+        System::CmdLineBuilder cmd;
+        cmd.string_arg("curl")
+            .string_arg("--head")
+            .string_arg("--location")
+            .string_arg("-w")
+            .string_arg(Strings::concat(guid_marker, " %{http_code}\\n"));
+        for (auto&& url : urls)
+        {
+            cmd.string_arg(url);
+        }
+        auto res = System::cmd_execute_and_stream_lines(cmd, [out](const std::string& line) {
+            if (Strings::starts_with(line, guid_marker))
+            {
+                out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10));
+            }
+        });
+        Checks::check_exit(VCPKG_LINE_INFO, res == 0, "curl failed to execute with exit code: %d", res);
+    }
+    std::vector<int> url_heads(View<std::string> urls)
+    {
+        static constexpr size_t batch_size = 100;
+
+        std::vector<int> ret;
+
+        size_t i = 0;
+        for (; i + batch_size <= urls.size(); i += batch_size)
+        {
+            url_heads_inner({urls.data() + i, batch_size}, &ret);
+        }
+        if (i != urls.size()) url_heads_inner({urls.begin() + i, urls.end()}, &ret);
+
+        return ret;
+    }
+
+    static void download_files_inner(Files::Filesystem&,
+                                     View<std::pair<std::string, fs::path>> url_pairs,
+                                     std::vector<int>* out)
+    {
+        static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e";
+
+        System::CmdLineBuilder cmd;
+        cmd.string_arg("curl")
+            .string_arg("--location")
+            .string_arg("-w")
+            .string_arg(Strings::concat(guid_marker, " %{http_code}\\n"));
+        for (auto&& url : url_pairs)
+        {
+            cmd.string_arg(url.first).string_arg("-o").path_arg(url.second);
+        }
+        auto res = System::cmd_execute_and_stream_lines(cmd, [out](const std::string& line) {
+            if (Strings::starts_with(line, guid_marker))
+            {
+                out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10));
+            }
+        });
+        Checks::check_exit(VCPKG_LINE_INFO, res == 0, "curl failed to execute with exit code: %d", res);
+    }
+    std::vector<int> download_files(Files::Filesystem& fs, View<std::pair<std::string, fs::path>> url_pairs)
+    {
+        static constexpr size_t batch_size = 50;
+
+        std::vector<int> ret;
+
+        size_t i = 0;
+        for (; i + batch_size <= url_pairs.size(); i += batch_size)
+        {
+            download_files_inner(fs, {url_pairs.data() + i, batch_size}, &ret);
+        }
+        if (i != url_pairs.size()) download_files_inner(fs, {url_pairs.begin() + i, url_pairs.end()}, &ret);
+
+        Checks::check_exit(VCPKG_LINE_INFO, ret.size() == url_pairs.size());
+        return ret;
+    }
+
+    int put_file(const Files::Filesystem&, StringView url, const fs::path& file)
+    {
+        static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e";
+
+        System::CmdLineBuilder cmd;
+        cmd.string_arg("curl").string_arg("-X").string_arg("PUT");
+        cmd.string_arg("-w").string_arg(Strings::concat("\\n", guid_marker, "%{http_code}"));
+        cmd.string_arg(url);
+        cmd.string_arg("-T").path_arg(file);
+        cmd.string_arg("-H").string_arg("x-ms-blob-type: BlockBlob");
+        int code = 0;
+        auto res = System::cmd_execute_and_stream_lines(cmd, [&code](const std::string& line) {
+            if (Strings::starts_with(line, guid_marker))
+            {
+                code = std::strtol(line.data() + guid_marker.size(), nullptr, 10);
+            }
+        });
+        if (res != 0)
+        {
+            System::print2(System::Color::warning, "curl failed to execute with exit code: ", res, '\n');
+        }
+        return code;
+    }
+
+    void download_file(Files::Filesystem& fs,
                        const std::string& url,
                        const fs::path& download_path,
                        const std::string& sha512)
     {
-        const std::string download_path_part = fs::u8string(download_path) + ".part";
-        auto download_path_part_path = fs::u8path(download_path_part);
-        std::error_code ec;
-        fs.remove(download_path, ec);
-        fs.remove(download_path_part_path, ec);
-#if defined(_WIN32)
-        auto url_no_proto = url.substr(8); // drop https://
-        auto path_begin = Util::find(url_no_proto, '/');
-        std::string hostname(url_no_proto.begin(), path_begin);
-        std::string path(path_begin, url_no_proto.end());
+        download_file(fs, {&url, 1}, download_path, sha512);
+    }
 
-        winhttp_download_file(fs, download_path_part, hostname, path);
-#else
-        const auto code = System::cmd_execute(
-            Strings::format(R"(curl -L '%s' --create-dirs --output '%s')", url, download_path_part));
-        Checks::check_exit(VCPKG_LINE_INFO, code == 0, "Could not download %s", url);
+#if defined(_WIN32)
+    namespace
+    {
+        struct WriteFlushFile
+        {
+            WriteFlushFile(const fs::path& p)
+            {
+                auto err = _wfopen_s(&f, p.c_str(), L"wb");
+                Checks::check_exit(VCPKG_LINE_INFO,
+                                   !err,
+                                   "Failed to open file %s. Error code was %s",
+                                   fs::u8string(p),
+                                   std::to_string(err));
+                ASSUME(f != nullptr);
+            }
+            ~WriteFlushFile()
+            {
+                if (f)
+                {
+                    fflush(f);
+                    fclose(f);
+                }
+            }
+            FILE* f = nullptr;
+        };
+
+        /// <summary>
+        /// Download a file using WinHTTP -- only supports HTTP and HTTPS
+        /// </summary>
+        static bool download_winhttp(Files::Filesystem& fs,
+                                     const fs::path& download_path_part_path,
+                                     details::SplitURIView split_uri,
+                                     const std::string& url,
+                                     std::string& errors)
+        {
+            // `download_winhttp` does not support user or port syntax in authorities
+            auto hostname = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2);
+            INTERNET_PORT port;
+            if (split_uri.scheme == "https")
+            {
+                port = INTERNET_DEFAULT_HTTPS_PORT;
+            }
+            else if (split_uri.scheme == "http")
+            {
+                port = INTERNET_DEFAULT_HTTP_PORT;
+            }
+            else
+            {
+                Checks::unreachable(VCPKG_LINE_INFO);
+            }
+
+            // Make sure the directories are present, otherwise fopen_s fails
+            const auto dir = download_path_part_path.parent_path();
+            fs.create_directories(dir, VCPKG_LINE_INFO);
+
+            WriteFlushFile f(download_path_part_path);
+
+            Debug::print("Downloading ", url, "\n");
+            static auto s = WinHttpSession::make().value_or_exit(VCPKG_LINE_INFO);
+            auto conn = WinHttpConnection::make(s.m_hSession.get(), hostname, port);
+            if (!conn)
+            {
+                Strings::append(errors, url, ": ", conn.error(), '\n');
+                return false;
+            }
+            auto req = WinHttpRequest::make(conn.get()->m_hConnect.get(), split_uri.path_query_fragment);
+            if (!req)
+            {
+                Strings::append(errors, url, ": ", req.error(), '\n');
+                return false;
+            }
+            auto forall_data =
+                req.get()->forall_data([&](Span<char> span) { fwrite(span.data(), 1, span.size(), f.f); });
+            if (!forall_data)
+            {
+                Strings::append(errors, url, ": ", forall_data.error(), '\n');
+                return false;
+            }
+            return true;
+        }
+    }
 #endif
 
-        verify_downloaded_file_hash(fs, url, download_path_part_path, sha512);
-        fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
+    std::string download_file(vcpkg::Files::Filesystem& fs,
+                              View<std::string> urls,
+                              const fs::path& download_path,
+                              const std::string& sha512)
+    {
+        Checks::check_exit(VCPKG_LINE_INFO, urls.size() > 0);
+
+        auto download_path_part_path = download_path;
+        download_path_part_path += fs::u8path(".part");
+        fs.remove(download_path, ignore_errors);
+        fs.remove(download_path_part_path, ignore_errors);
+
+        std::string errors;
+        for (const std::string& url : urls)
+        {
+#if defined(_WIN32)
+            auto split_uri = details::split_uri_view(url).value_or_exit(VCPKG_LINE_INFO);
+            auto authority = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2);
+            if (split_uri.scheme == "https" || split_uri.scheme == "http")
+            {
+                // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to curl.exe
+                if (Strings::find_first_of(authority, ":@") == authority.end())
+                {
+                    if (download_winhttp(fs, download_path_part_path, split_uri, url, errors))
+                    {
+                        verify_downloaded_file_hash(fs, url, download_path_part_path, sha512);
+                        fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
+                        return url;
+                    }
+                    continue;
+                }
+            }
+#endif
+            System::CmdLineBuilder cmd;
+            cmd.string_arg("curl")
+                .string_arg("--fail")
+                .string_arg("-L")
+                .string_arg(url)
+                .string_arg("--create-dirs")
+                .string_arg("--output")
+                .path_arg(download_path_part_path);
+            const auto out = System::cmd_execute_and_capture_output(cmd);
+            if (out.exit_code != 0)
+            {
+                Strings::append(errors, url, ": ", out.output, '\n');
+                continue;
+            }
+
+            verify_downloaded_file_hash(fs, url, download_path_part_path, sha512);
+            fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
+            return url;
+        }
+        Checks::exit_with_message(VCPKG_LINE_INFO, "Failed to download from mirror set:\n%s", errors);
     }
 }

toolsrc/src/vcpkg/binarycaching.cpp

@@ -1,4 +1,5 @@
 #include <vcpkg/base/checks.h>
+#include <vcpkg/base/downloads.h>
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/parse.h>
 #include <vcpkg/base/system.debug.h>
@@ -16,31 +17,68 @@ using namespace vcpkg;
 
 namespace
 {
+    struct NullBinaryProvider : IBinaryProvider
+    {
+        void prefetch(const VcpkgPaths&, std::vector<const Dependencies::InstallPlanAction*>&) { }
+
+        void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) { }
+
+        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&)
+        {
+            return RestoreResult::missing;
+        }
+
+        void precheck(const VcpkgPaths&, std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>&) { }
+    };
+}
+
+std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> vcpkg::binary_provider_precheck(
+    const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider)
+{
+    std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> checked;
+    for (auto&& action : plan.install_actions)
+        checked.emplace(&action, RestoreResult::missing);
+    provider.precheck(paths, checked);
+    return checked;
+}
+
+namespace
+{
+    static void clean_prepare_dir(Files::Filesystem& fs, const fs::path& dir)
+    {
+        fs.remove_all(dir, VCPKG_LINE_INFO);
+        bool created_last = fs.create_directories(dir, VCPKG_LINE_INFO);
+        Checks::check_exit(VCPKG_LINE_INFO, created_last, "unable to clear path: %s", fs::u8string(dir));
+    }
+
     static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,
-                                                        const PackageSpec& spec,
+                                                        const fs::path& dst,
                                                         const fs::path& archive_path)
     {
-        auto& fs = paths.get_filesystem();
-
-        auto pkg_path = paths.package_dir(spec);
-        fs.remove_all(pkg_path, VCPKG_LINE_INFO);
-        std::error_code ec;
-        fs.create_directories(pkg_path, ec);
-        auto files = fs.get_files_non_recursive(pkg_path);
-        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", fs::u8string(pkg_path));
-
+        System::CmdLineBuilder cmd;
 #if defined(_WIN32)
         auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
-        auto cmd = Strings::format(R"("%s" x "%s" -o"%s" -y)",
-                                   fs::u8string(seven_zip_exe),
-                                   fs::u8string(archive_path),
-                                   fs::u8string(pkg_path));
+        cmd.path_arg(seven_zip_exe)
+            .string_arg("x")
+            .path_arg(archive_path)
+            .string_arg("-o" + fs::u8string(dst))
+            .string_arg("-y");
 #else
-        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", fs::u8string(archive_path), fs::u8string(pkg_path));
+        (void)paths;
+        cmd.string_arg("unzip").string_arg("-qq").path_arg(archive_path).string_arg("-d" + fs::u8string(dst));
 #endif
         return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
     }
 
+    static System::ExitCodeAndOutput clean_decompress_archive(const VcpkgPaths& paths,
+                                                              const PackageSpec& spec,
+                                                              const fs::path& archive_path)
+    {
+        auto pkg_path = paths.package_dir(spec);
+        clean_prepare_dir(paths.get_filesystem(), pkg_path);
+        return decompress_archive(paths, pkg_path, archive_path);
+    }
+
     // Compress the source directory into the destination file.
     static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)
     {
@@ -66,69 +104,95 @@ namespace
 
     struct ArchivesBinaryProvider : IBinaryProvider
     {
-        ArchivesBinaryProvider(std::vector<fs::path>&& read_dirs, std::vector<fs::path>&& write_dirs)
-            : m_read_dirs(std::move(read_dirs)), m_write_dirs(std::move(write_dirs))
+        ArchivesBinaryProvider(std::vector<fs::path>&& read_dirs,
+                               std::vector<fs::path>&& write_dirs,
+                               std::vector<std::string>&& put_url_templates)
+            : m_read_dirs(std::move(read_dirs))
+            , m_write_dirs(std::move(write_dirs))
+            , m_put_url_templates(std::move(put_url_templates))
         {
         }
-        ~ArchivesBinaryProvider() = default;
-        void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override { }
-        RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+
+        void prefetch(const VcpkgPaths& paths, std::vector<const Dependencies::InstallPlanAction*>& actions) override
         {
-            const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
-            auto& spec = action.spec;
             auto& fs = paths.get_filesystem();
-            std::error_code ec;
-            for (auto&& archives_root_dir : m_read_dirs)
-            {
-                const std::string archive_name = abi_tag + ".zip";
-                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
-                const fs::path archive_path = archives_root_dir / archive_subpath;
-                if (fs.exists(archive_path))
+            Util::erase_remove_if(actions, [this, &fs, &paths](const Dependencies::InstallPlanAction* action) {
+                auto& spec = action->spec;
+                const auto& abi_tag = action->abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
+                const auto archive_name = fs::u8path(abi_tag + ".zip");
+                for (const auto& archives_root_dir : m_read_dirs)
                 {
-                    System::print2("Using cached binary package: ", fs::u8string(archive_path), "\n");
-
-                    int archive_result = decompress_archive(paths, spec, archive_path).exit_code;
-
-                    if (archive_result == 0)
+                    auto archive_path = archives_root_dir;
+                    archive_path /= fs::u8path(abi_tag.substr(0, 2));
+                    archive_path /= archive_name;
+                    if (fs.exists(archive_path))
                     {
-                        return RestoreResult::success;
-                    }
-                    else
-                    {
-                        System::print2("Failed to decompress archive package\n");
-                        if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)
+                        System::print2("Using cached binary package: ", fs::u8string(archive_path), "\n");
+
+                        int archive_result = clean_decompress_archive(paths, spec, archive_path).exit_code;
+
+                        if (archive_result == 0)
                         {
-                            return RestoreResult::build_failed;
+                            m_restored.insert(spec);
+                            return true;
                         }
                         else
                         {
-                            System::print2("Purging bad archive\n");
-                            fs.remove(archive_path, ec);
+                            System::print2("Failed to decompress archive package\n");
+                            if (action->build_options.purge_decompress_failure == Build::PurgeDecompressFailure::YES)
+                            {
+                                System::print2("Purging bad archive\n");
+                                fs.remove(archive_path, ignore_errors);
+                            }
                         }
                     }
+
+                    System::printf("Could not locate cached archive: %s\n", fs::u8string(archive_path));
                 }
-
-                System::printf("Could not locate cached archive: %s\n", fs::u8string(archive_path));
-            }
-
-            return RestoreResult::missing;
+                return false;
+            });
+        }
+        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override
+        {
+            if (Util::Sets::contains(m_restored, action.spec))
+                return RestoreResult::success;
+            else
+                return RestoreResult::missing;
         }
         void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
         {
-            if (m_write_dirs.empty()) return;
+            if (m_write_dirs.empty() && m_put_url_templates.empty()) return;
             const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
             auto& spec = action.spec;
             auto& fs = paths.get_filesystem();
             const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
             compress_directory(paths, paths.package_dir(spec), tmp_archive_path);
 
-            for (auto&& m_directory : m_write_dirs)
+            size_t http_remotes_pushed = 0;
+            for (auto&& put_url_template : m_put_url_templates)
             {
-                const fs::path& archives_root_dir = m_directory;
-                const std::string archive_name = abi_tag + ".zip";
-                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
-                const fs::path archive_path = archives_root_dir / archive_subpath;
+                auto url = Strings::replace_all(std::string(put_url_template), "<SHA>", abi_tag);
+                auto code = Downloads::put_file(fs, url, tmp_archive_path);
+                if (code >= 200 && code < 300)
+                {
+                    http_remotes_pushed++;
+                }
+                else
+                {
+                    Debug::print("Failed to upload to ", url, ": ", code, '\n');
+                }
+            }
+            if (!m_put_url_templates.empty())
+            {
+                System::print2("Uploaded binaries to ", http_remotes_pushed, " HTTP remotes.\n");
+            }
 
+            const auto archive_name = fs::u8path(abi_tag + ".zip");
+            for (const auto& archives_root_dir : m_write_dirs)
+            {
+                auto archive_path = archives_root_dir;
+                archive_path /= fs::u8path(abi_tag.substr(0, 2));
+                archive_path /= archive_name;
                 fs.create_directories(archive_path.parent_path(), ignore_errors);
                 std::error_code ec;
                 if (m_write_dirs.size() > 1)
@@ -145,29 +209,143 @@ namespace
                 else
                     System::printf("Stored binary cache: %s\n", fs::u8string(archive_path));
             }
-            if (m_write_dirs.size() > 1) fs.remove(tmp_archive_path, ignore_errors);
-        }
-        RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
-        {
-            const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
-            auto& fs = paths.get_filesystem();
-            std::error_code ec;
-            for (auto&& archives_root_dir : m_read_dirs)
+            // In the case of 1 write dir, the file will be moved instead of copied
+            if (m_write_dirs.size() != 1)
             {
-                const std::string archive_name = abi_tag + ".zip";
-                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
-                const fs::path archive_path = archives_root_dir / archive_subpath;
+                fs.remove(tmp_archive_path, ignore_errors);
+            }
+        }
+        void precheck(const VcpkgPaths& paths,
+                      std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) override
+        {
+            auto& fs = paths.get_filesystem();
 
-                if (fs.exists(archive_path))
+            for (auto&& result_pair : results_map)
+            {
+                if (result_pair.second != RestoreResult::missing) continue;
+
+                const auto& abi_tag = result_pair.first->abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
+                std::error_code ec;
+                for (auto&& archives_root_dir : m_read_dirs)
                 {
-                    return RestoreResult::success;
+                    const std::string archive_name = abi_tag + ".zip";
+                    const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;
+                    const fs::path archive_path = archives_root_dir / archive_subpath;
+
+                    if (fs.exists(archive_path))
+                    {
+                        result_pair.second = RestoreResult::success;
+                        break;
+                    }
                 }
             }
-            return RestoreResult::missing;
         }
 
     private:
-        std::vector<fs::path> m_read_dirs, m_write_dirs;
+        std::vector<fs::path> m_read_dirs;
+        std::vector<fs::path> m_write_dirs;
+        std::vector<std::string> m_put_url_templates;
+
+        std::set<PackageSpec> m_restored;
+    };
+    struct HttpGetBinaryProvider : NullBinaryProvider
+    {
+        HttpGetBinaryProvider(std::vector<std::string>&& url_templates) : m_url_templates(std::move(url_templates)) { }
+        void prefetch(const VcpkgPaths& paths, std::vector<const Dependencies::InstallPlanAction*>& actions) override
+        {
+            auto& fs = paths.get_filesystem();
+
+            const size_t current_restored = m_restored.size();
+
+            for (auto&& url_template : m_url_templates)
+            {
+                std::vector<std::pair<std::string, fs::path>> url_paths;
+                std::vector<PackageSpec> specs;
+
+                for (auto&& action : actions)
+                {
+                    auto abi = action->package_abi();
+                    if (!abi) continue;
+
+                    specs.push_back(action->spec);
+                    auto pkgdir = paths.package_dir(action->spec);
+                    clean_prepare_dir(fs, pkgdir);
+                    pkgdir /= fs::u8path(Strings::concat(*abi.get(), ".zip"));
+                    url_paths.emplace_back(Strings::replace_all(std::string(url_template), "<SHA>", *abi.get()),
+                                           pkgdir);
+                }
+
+                if (url_paths.empty()) break;
+
+                System::print2("Attempting to fetch ", url_paths.size(), " packages from HTTP servers.\n");
+
+                auto codes = Downloads::download_files(fs, url_paths);
+                for (size_t i = 0; i < codes.size(); ++i)
+                {
+                    if (codes[i] == 200)
+                    {
+                        int archive_result =
+                            decompress_archive(paths, paths.package_dir(specs[i]), url_paths[i].second).exit_code;
+                        if (archive_result == 0)
+                        {
+                            // decompression success
+                            fs.remove(url_paths[i].second, VCPKG_LINE_INFO);
+                            m_restored.insert(specs[i]);
+                        }
+                        else
+                        {
+                            Debug::print("Failed to decompress ", fs::u8string(url_paths[i].second), '\n');
+                        }
+                    }
+                }
+
+                Util::erase_remove_if(actions, [this](const Dependencies::InstallPlanAction* action) {
+                    return Util::Sets::contains(m_restored, action->spec);
+                });
+            }
+            System::print2("Restored ",
+                           m_restored.size() - current_restored,
+                           " packages from HTTP servers. Use --debug for more information.\n");
+        }
+        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override
+        {
+            if (Util::Sets::contains(m_restored, action.spec))
+                return RestoreResult::success;
+            else
+                return RestoreResult::missing;
+        }
+        void precheck(const VcpkgPaths&,
+                      std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) override
+        {
+            std::vector<std::string> urls;
+            std::vector<const Dependencies::InstallPlanAction*> url_actions;
+            for (auto&& url_template : m_url_templates)
+            {
+                urls.clear();
+                url_actions.clear();
+                for (auto&& result_pair : results_map)
+                {
+                    if (result_pair.second != RestoreResult::missing) continue;
+                    auto abi = result_pair.first->package_abi();
+                    if (!abi) continue;
+                    urls.push_back(Strings::replace_all(std::string(url_template), "<SHA>", *abi.get()));
+                    url_actions.push_back(result_pair.first);
+                }
+                if (urls.empty()) break;
+                auto codes = Downloads::url_heads(urls);
+                Checks::check_exit(VCPKG_LINE_INFO, codes.size() == url_actions.size());
+                for (size_t i = 0; i < codes.size(); ++i)
+                {
+                    if (codes[i] == 200)
+                    {
+                        results_map[url_actions[i]] = RestoreResult::success;
+                    }
+                }
+            }
+        }
+
+        std::vector<std::string> m_url_templates;
+        std::set<PackageSpec> m_restored;
     };
 
     static std::string trim_leading_zeroes(std::string v)
@@ -184,7 +362,7 @@ namespace
         return v;
     }
 
-    struct NugetBinaryProvider : IBinaryProvider
+    struct NugetBinaryProvider : NullBinaryProvider
     {
         NugetBinaryProvider(std::vector<std::string>&& read_sources,
                             std::vector<std::string>&& write_sources,
@@ -198,7 +376,8 @@ namespace
             , m_interactive(interactive)
         {
         }
-        void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override
+
+        void prefetch(const VcpkgPaths& paths, std::vector<const Dependencies::InstallPlanAction*>& actions) override
         {
             if (m_read_sources.empty() && m_read_configs.empty()) return;
 
@@ -206,14 +385,14 @@ namespace
 
             std::vector<std::pair<PackageSpec, NugetReference>> nuget_refs;
 
-            for (auto&& action : plan.install_actions)
+            for (auto&& action : actions)
             {
-                if (!action.has_package_abi()) continue;
+                if (!action->has_package_abi()) continue;
 
-                auto& spec = action.spec;
+                auto& spec = action->spec;
                 fs.remove_all(paths.package_dir(spec), VCPKG_LINE_INFO);
 
-                nuget_refs.emplace_back(spec, NugetReference(action));
+                nuget_refs.emplace_back(spec, NugetReference(*action));
             }
 
             if (nuget_refs.empty()) return;
@@ -294,7 +473,7 @@ namespace
                 cmdlines.push_back(cmdline.extract());
             }
 
-            size_t num_restored = 0;
+            const size_t current_restored = m_restored.size();
 
             for (const auto& cmdline : cmdlines)
             {
@@ -327,7 +506,6 @@ namespace
                                            "Unable to remove nupkg after restoring: %s",
                                            fs::u8string(nupkg_path));
                         m_restored.emplace(nuget_ref.first);
-                        ++num_restored;
                         return true;
                     }
                     else
@@ -337,7 +515,13 @@ namespace
                 });
             }
 
-            System::print2("Restored ", num_restored, " packages. Use --debug for more information.\n");
+            Util::erase_remove_if(actions, [this](const Dependencies::InstallPlanAction* action) {
+                return Util::Sets::contains(m_restored, action->spec);
+            });
+
+            System::print2("Restored ",
+                           m_restored.size() - current_restored,
+                           " packages from NuGet. Use --debug for more information.\n");
         }
         RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override
         {
@@ -450,10 +634,6 @@ namespace
                 paths.get_filesystem().remove(nupkg_path, ignore_errors);
             }
         }
-        RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
-        {
-            return RestoreResult::missing;
-        }
 
     private:
         std::vector<std::string> m_read_sources;
@@ -465,19 +645,22 @@ namespace
         std::set<PackageSpec> m_restored;
         bool m_interactive;
     };
+}
 
-    struct MergeBinaryProviders : IBinaryProvider
+namespace vcpkg
+{
+    struct MergeBinaryProviders : NullBinaryProvider
     {
         explicit MergeBinaryProviders(std::vector<std::unique_ptr<IBinaryProvider>>&& providers)
             : m_providers(std::move(providers))
         {
         }
 
-        void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override
+        void prefetch(const VcpkgPaths& paths, std::vector<const Dependencies::InstallPlanAction*>& actions) override
         {
             for (auto&& provider : m_providers)
             {
-                provider->prefetch(paths, plan);
+                provider->prefetch(paths, actions);
             }
         }
         RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
@@ -502,39 +685,18 @@ namespace
                 provider->push_success(paths, action);
             }
         }
-        RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        void precheck(const VcpkgPaths& paths,
+                      std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) override
         {
             for (auto&& provider : m_providers)
             {
-                auto result = provider->precheck(paths, action);
-                switch (result)
-                {
-                    case RestoreResult::build_failed:
-                    case RestoreResult::success: return result;
-                    case RestoreResult::missing: continue;
-                    default: Checks::unreachable(VCPKG_LINE_INFO);
-                }
+                provider->precheck(paths, results_map);
             }
-            return RestoreResult::missing;
         }
 
     private:
         std::vector<std::unique_ptr<IBinaryProvider>> m_providers;
     };
-
-    struct NullBinaryProvider : IBinaryProvider
-    {
-        void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override { }
-        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
-        {
-            return RestoreResult::missing;
-        }
-        void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override { }
-        RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
-        {
-            return RestoreResult::missing;
-        }
-    };
 }
 
 XmlSerializer& XmlSerializer::emit_declaration()
@@ -650,12 +812,6 @@ IBinaryProvider& vcpkg::null_binary_provider()
     return p;
 }
 
-ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs(View<std::string> args)
-{
-    std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or("");
-
-    return create_binary_provider_from_configs_pure(env_string, args);
-}
 namespace
 {
     const ExpectedS<fs::path>& default_cache_path()
@@ -692,10 +848,6 @@ namespace
                 return {"default path was not absolute: " + fs::u8string(p), expected_right_tag};
             }
         });
-        if (cachepath.has_value())
-            Debug::print("Default binary cache path is: ", fs::u8string(*cachepath.get()), '\n');
-        else
-            Debug::print("No binary cache path. Reason: ", cachepath.error(), '\n');
         return cachepath;
     }
 
@@ -707,6 +859,9 @@ namespace
         std::vector<fs::path> archives_to_read;
         std::vector<fs::path> archives_to_write;
 
+        std::vector<std::string> url_templates_to_get;
+        std::vector<std::string> azblob_templates_to_put;
+
         std::vector<std::string> sources_to_read;
         std::vector<std::string> sources_to_write;
 
@@ -719,6 +874,8 @@ namespace
             interactive = false;
             archives_to_read.clear();
             archives_to_write.clear();
+            url_templates_to_get.clear();
+            azblob_templates_to_put.clear();
             sources_to_read.clear();
             sources_to_write.clear();
             configs_to_read.clear();
@@ -901,6 +1058,40 @@ namespace
                 handle_readwrite(
                     state->archives_to_read, state->archives_to_write, fs::path(*maybe_home.get()), segments, 1);
             }
+            else if (segments[0].second == "x-azblob")
+            {
+                // Scheme: x-azblob,<baseurl>,<sas>[,<readwrite>]
+                if (segments.size() < 3)
+                {
+                    return add_error(
+                        "expected arguments: binary config 'azblob' requires at least a base-url and a SAS token",
+                        segments[0].first);
+                }
+                if (!Strings::starts_with(segments[1].second, "https://"))
+                {
+                    return add_error(
+                        "invalid argument: binary config 'azblob' requires an https base url as the first argument",
+                        segments[1].first);
+                }
+                if (Strings::starts_with(segments[2].second, "?"))
+                {
+                    return add_error("invalid argument: binary config 'azblob' requires a SAS token without a "
+                                     "preceeding '?' as the second argument",
+                                     segments[2].first);
+                }
+                auto p = segments[1].second;
+                if (p.back() != '/') p.push_back('/');
+                p.append("<SHA>.zip");
+                if (!Strings::starts_with(segments[2].second, "?")) p.push_back('?');
+                p.append(segments[2].second);
+                handle_readwrite(
+                    state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3);
+                if (segments.size() > 4)
+                {
+                    return add_error("unexpected arguments: binary config 'azblob' requires 2 or 3 arguments",
+                                     segments[4].first);
+                }
+            }
             else
             {
                 return add_error(
@@ -912,6 +1103,21 @@ namespace
     };
 }
 
+ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs(View<std::string> args)
+{
+    std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or("");
+    if (Debug::g_debugging)
+    {
+        const auto& cachepath = default_cache_path();
+        if (cachepath.has_value())
+            Debug::print("Default binary cache path is: ", fs::u8string(*cachepath.get()), '\n');
+        else
+            Debug::print("No binary cache path. Reason: ", cachepath.error(), '\n');
+    }
+
+    return create_binary_provider_from_configs_pure(env_string, args);
+}
+
 ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs_pure(
     const std::string& env_string, View<std::string> args)
 {
@@ -939,9 +1145,16 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
     if (s.m_cleared) Metrics::g_metrics.lock()->track_property("binarycaching-clear", "defined");
 
     std::vector<std::unique_ptr<IBinaryProvider>> providers;
-    if (!s.archives_to_read.empty() || !s.archives_to_write.empty())
-        providers.push_back(
-            std::make_unique<ArchivesBinaryProvider>(std::move(s.archives_to_read), std::move(s.archives_to_write)));
+    if (!s.archives_to_read.empty() || !s.archives_to_write.empty() || !s.azblob_templates_to_put.empty())
+    {
+        providers.push_back(std::make_unique<ArchivesBinaryProvider>(
+            std::move(s.archives_to_read), std::move(s.archives_to_write), std::move(s.azblob_templates_to_put)));
+    }
+    if (!s.url_templates_to_get.empty())
+    {
+        Metrics::g_metrics.lock()->track_property("binarycaching-url-get", "defined");
+        providers.push_back(std::make_unique<HttpGetBinaryProvider>(std::move(s.url_templates_to_get)));
+    }
     if (!s.sources_to_read.empty() || !s.sources_to_write.empty() || !s.configs_to_read.empty() ||
         !s.configs_to_write.empty())
     {
@@ -1088,6 +1301,9 @@ void vcpkg::help_topic_binary_caching(const VcpkgPaths&)
     tbl.format("nugetconfig,<path>[,<rw>]",
                "Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This "
                "config should specify `defaultPushSource` for uploads.");
+    tbl.format("x-azblob,<url>,<sas>[,<rw>]",
+               "**Experimental: will change or be removed without warning** Adds an Azure Blob Storage source. Uses "
+               "Shared Access Signature validation. URL should include the container path.");
     tbl.format("interactive", "Enables interactive credential management for some source types");
     tbl.blank();
     tbl.text("The `<rw>` optional parameter for certain strings controls whether they will be consulted for "

toolsrc/src/vcpkg/commands.ci.cpp

@@ -332,6 +332,8 @@ namespace vcpkg::Commands::CI
 
         {
             vcpkg::System::BufferedPrint stdout_print;
+            auto precheck_results = binary_provider_precheck(paths, action_plan, binaryprovider);
+
             for (auto&& action : action_plan.install_actions)
             {
                 auto p = &action;
@@ -345,7 +347,7 @@ namespace vcpkg::Commands::CI
                     p->build_options = vcpkg::Build::backcompat_prohibiting_package_options;
                 }
 
-                auto precheck_result = binaryprovider.precheck(paths, action);
+                auto precheck_result = precheck_results.at(&action);
                 bool b_will_build = false;
 
                 std::string state;

toolsrc/src/vcpkg/dependencies.cpp

@@ -433,6 +433,12 @@ namespace vcpkg::Dependencies
         if (!abi_info) return false;
         return !abi_info.get()->package_abi.empty();
     }
+    Optional<const std::string&> InstallPlanAction::package_abi() const
+    {
+        if (!abi_info) return nullopt;
+        if (abi_info.get()->package_abi.empty()) return nullopt;
+        return abi_info.get()->package_abi;
+    }
     const Build::PreBuildInfo& InstallPlanAction::pre_build_info(LineInfo linfo) const
     {
         return *abi_info.value_or_exit(linfo).pre_build_info.get();

toolsrc/src/vcpkg/globalstate.cpp

@@ -1,3 +1,5 @@
+#include <vcpkg/base/lockguarded.h>
+
 #include <vcpkg/globalstate.h>
 
 namespace vcpkg

toolsrc/src/vcpkg/install.cpp

@@ -481,7 +481,8 @@ namespace vcpkg::Install
 
         Build::compute_all_abis(paths, action_plan, var_provider, status_db);
 
-        binaryprovider.prefetch(paths, action_plan);
+        auto to_prefetch = Util::fmap(action_plan.install_actions, [](const auto& x) { return &x; });
+        binaryprovider.prefetch(paths, to_prefetch);
 
         for (auto&& action : action_plan.install_actions)
         {