From acb6b10e7fdf5e8519c18398d0b069e1d58ca025 Mon Sep 17 00:00:00 2001 From: ras0219 <533828+ras0219@users.noreply.github.com> Date: Tue, 8 Dec 2020 10:46:46 -0800 Subject: [PATCH] [vcpkg] Fix uploading to Azure DevOps Artifacts (#14952) --- docs/users/binarycaching.md | 45 ++++++++++++++-- toolsrc/src/vcpkg/binarycaching.cpp | 81 ++++++++++++++++------------- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/docs/users/binarycaching.md b/docs/users/binarycaching.md index c3fb98a4ff..9df885f4c3 100644 --- a/docs/users/binarycaching.md +++ b/docs/users/binarycaching.md @@ -86,7 +86,7 @@ More information about Azure DevOps Artifacts' NuGet support is available in the ## Configuration -Binary caching is configured via a combination of defaults, the environment variable `VCPKG_BINARY_SOURCES` (set to `;;...`), and the command line option `--binarysource=`. Source options are evaluated in order of defaults, then environment, then command line. +Binary caching is configured via a combination of defaults, the environment variable `VCPKG_BINARY_SOURCES` (set to `;;...`), and the command line option `--binarysource=`. Source options are evaluated in order of defaults, then environment, then command line. Binary caching can be completely disabled by passing `--binarysource=clear` as the last command line option. By default, zip-based archives will be cached at the first valid location of: @@ -115,6 +115,47 @@ By default, zip-based archives will be cached at the first valid location of: The `` optional parameter for certain sources controls whether they will be consulted for downloading binaries (`read`), whether on-demand builds will be uploaded to that remote (`write`), or both (`readwrite`). +### Nuget Provider Configuration + +#### Credentials + +Many NuGet servers require additional credentials to access. The most flexible way to supply credentials is via the `nugetconfig` provider with a custom `nuget.config` file. See https://docs.microsoft.com/en-us/nuget/consume-packages/consuming-packages-authenticated-feeds for more information on authenticating via `nuget.config`. + +However, it is still possible to authenticate against many servers using NuGet's built-in credential providers or via customizing your environment's default `nuget.config`. The default config can be extended via nuget client calls such as +``` +nuget sources add -Name MyRemote -Source https://... -Username $user -Password $pass +``` +and then passed to vcpkg via `--binarysource=nuget,MyRemote,readwrite`. You can get a path to the precise copy of NuGet used by vcpkg by running `vcpkg fetch nuget`, which will report something like: +``` +$ vcpkg fetch nuget +/vcpkg/downloads/tools/nuget-5.5.1-linux/nuget.exe +``` +Non-Windows users will need to call this through mono via `mono /path/to/nuget.exe sources add ...`. + +##### Credential Example for Azure Dev Ops +```bash +# On Linux or OSX +$ mono `vcpkg fetch nuget | tail -n1` sources add \ + -name ADO \ + -Source https://pkgs.dev.azure.com/$ORG/_packaging/$FEEDNAME/nuget/v3/index.json \ + -Username $USERNAME \ + -Password $PAT +$ export VCPKG_BINARY_SOURCES="nuget,ADO,readwrite" +``` +```powershell +# On Windows Powershell +PS> & $(vcpkg fetch nuget | select -last 1) sources add ` + -name ADO ` + -Source https://pkgs.dev.azure.com/$ORG/_packaging/$FEEDNAME/nuget/v3/index.json ` + -Username $USERNAME ` + -Password $PAT +PS> $env:VCPKG_BINARY_SOURCES="nuget,ADO,readwrite" +``` + +We recommend using a Personal Access Token (PAT) as the password for maximum security. You can generate a PAT in User Settings -> Personal Access Tokens or `https://dev.azure.com/$ORG/_usersSettings/tokens`. + +#### `metadata.repository` + The `nuget` and `nugetconfig` source providers additionally respect certain environment variables while generating nuget packages. The `metadata.repository` field of any packages will be generated as: ``` @@ -128,8 +169,6 @@ or ``` if the appropriate environment variables are defined and non-empty. This is specifically used to associate packages in GitHub Packages with the _building_ project and not intended to associate with the original package sources. -Finally, binary caching can be completely disabled by passing `--no-binarycaching` on the command line. - ## Implementation Notes (internal details subject to change without notice) Binary caching relies on hashing everything that contributes to a particular package build. This includes: diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index 966c2e77b7..c73e789d0a 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -378,6 +378,45 @@ namespace { } + int run_nuget_commandline(const std::string& cmdline) + { + if (m_interactive) + { + return System::cmd_execute(cmdline); + } + + auto res = System::cmd_execute_and_capture_output(cmdline); + if (Debug::g_debugging) + { + System::print2(res.output); + } + if (res.output.find("Authentication may require manual action.") != std::string::npos) + { + System::print2(System::Color::warning, + "One or more NuGet credential providers requested manual action. Add the binary " + "source 'interactive' to allow interactivity.\n"); + } + else if (res.output.find("Response status code does not indicate success: 401 (Unauthorized)") != + std::string::npos && + res.exit_code != 0) + { + System::print2(System::Color::warning, + "One or more NuGet credential providers failed to authenticate. See " + "https://github.com/Microsoft/vcpkg/tree/master/docs/users/binarycaching.md for " + "more details on how to provide credentials.\n"); + } + else if (res.output.find("for example \"-ApiKey AzureDevOps\"") != std::string::npos) + { + auto res2 = System::cmd_execute_and_capture_output(cmdline + " -ApiKey AzureDevOps"); + if (Debug::g_debugging) + { + System::print2(res2.output); + } + return res2.exit_code; + } + return res.exit_code; + } + void prefetch(const VcpkgPaths& paths, std::vector& actions) override { if (m_read_sources.empty() && m_read_configs.empty()) return; @@ -482,19 +521,7 @@ namespace [&] { generate_packages_config(); - if (Debug::g_debugging) - System::cmd_execute(cmdline); - else - { - auto res = System::cmd_execute_and_capture_output(cmdline); - if (res.output.find("Authentication may require manual action.") != std::string::npos) - { - System::print2( - System::Color::warning, - "One or more NuGet credential providers requested manual action. Add the binary " - "source 'interactive' to allow interactivity.\n"); - } - } + run_nuget_commandline(cmdline); }(); Util::erase_remove_if(nuget_refs, [&](const std::pair& nuget_ref) -> bool { @@ -555,12 +582,7 @@ namespace .string_arg("-ForceEnglishOutput"); if (!m_interactive) cmdline.string_arg("-NonInteractive"); - auto pack_rc = [&] { - if (Debug::g_debugging) - return System::cmd_execute(cmdline); - else - return System::cmd_execute_and_capture_output(cmdline).exit_code; - }(); + auto pack_rc = run_nuget_commandline(cmdline.extract()); if (pack_rc != 0) { @@ -585,12 +607,7 @@ namespace System::print2("Uploading binaries for ", spec, " to NuGet source ", write_src, ".\n"); - auto rc = [&] { - if (Debug::g_debugging) - return System::cmd_execute(cmd); - else - return System::cmd_execute_and_capture_output(cmd).exit_code; - }(); + auto rc = run_nuget_commandline(cmd.extract()); if (rc != 0) { @@ -617,12 +634,7 @@ namespace System::print2( "Uploading binaries for ", spec, " using NuGet config ", fs::u8string(write_cfg), ".\n"); - auto rc = [&] { - if (Debug::g_debugging) - return System::cmd_execute(cmd); - else - return System::cmd_execute_and_capture_output(cmd).exit_code; - }(); + auto rc = run_nuget_commandline(cmd.extract()); if (rc != 0) { @@ -1176,13 +1188,12 @@ void vcpkg::help_topic_binary_caching(const VcpkgPaths&) { HelpTableFormatter tbl; tbl.text("Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network." - " This functionality is currently enabled by default and can be disabled by either passing " - "`--no-binarycaching` to every vcpkg command line or setting the environment variable " - "`VCPKG_FEATURE_FLAGS` to `-binarycaching`."); + " By default, vcpkg will save builds to a local machine cache. This can be disabled by passing " + "`--binarysource=clear` as the last option on the command line."); tbl.blank(); tbl.blank(); tbl.text( - "Once caching is enabled, it can be further configured by either passing `--binarysource=` options " + "Binary caching can be further configured by either passing `--binarysource=` options " "to every command line or setting the `VCPKG_BINARY_SOURCES` environment variable to a set of sources (Ex: " "\";;...\"). Command line sources are interpreted after environment sources."); tbl.blank();