From d97eae3338f1a4657c35ad136ea426f66d6936f2 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Mon, 21 Dec 2020 09:19:57 -0800 Subject: [PATCH] Print failing upload attempts in !debug. (#15206) Other changes: * Changed Strings::replace_all to use more StringView * Introduced Strings::inplace_replace_all for the common x = replace_all(move(x) pattern and tests -> Also fixed bug if the search string was empty consuming infinite memory! * Added many missing {}s in binarycaching.cpp --- toolsrc/include/vcpkg/base/strings.h | 6 +- toolsrc/include/vcpkg/base/stringview.h | 31 +-- toolsrc/src/vcpkg-test/strings.cpp | 41 +++ toolsrc/src/vcpkg/base/strings.cpp | 21 +- toolsrc/src/vcpkg/base/stringview.cpp | 10 +- toolsrc/src/vcpkg/binarycaching.cpp | 303 +++++++++++++++++++---- toolsrc/src/vcpkg/commands.integrate.cpp | 6 +- toolsrc/src/vcpkg/export.chocolatey.cpp | 30 ++- toolsrc/src/vcpkg/export.prefab.cpp | 34 +-- 9 files changed, 369 insertions(+), 113 deletions(-) diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 5113ff2fc3..16b39001bd 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -188,7 +188,11 @@ namespace vcpkg::Strings return join(delimiter, v, [](const Element& x) -> const Element& { return x; }); } - std::string replace_all(std::string&& s, const std::string& search, StringView rep); + std::string replace_all(std::string&& s, StringView search, StringView rep); + + void inplace_replace_all(std::string& s, StringView search, StringView rep); + + void inplace_replace_all(std::string& s, char search, char rep) noexcept; std::string trim(std::string&& s); diff --git a/toolsrc/include/vcpkg/base/stringview.h b/toolsrc/include/vcpkg/base/stringview.h index 9238990fef..aba27f9d13 100644 --- a/toolsrc/include/vcpkg/base/stringview.h +++ b/toolsrc/include/vcpkg/base/stringview.h @@ -14,43 +14,44 @@ namespace vcpkg { static std::vector find_all_enclosed(const StringView& input, const std::string& left_delim, - const std::string& right_delim); + const std::string& right_delim) noexcept; static StringView find_exactly_one_enclosed(const StringView& input, const std::string& left_tag, - const std::string& right_tag); + const std::string& right_tag) noexcept; static Optional find_at_most_one_enclosed(const StringView& input, const std::string& left_tag, - const std::string& right_tag); + const std::string& right_tag) noexcept; constexpr StringView() = default; - StringView(const std::string& s); // Implicit by design + StringView(const std::string& s) noexcept; // Implicit by design // NOTE: we do this instead of the delegating constructor since delegating ctors are a perf footgun template - constexpr StringView(const char (&arr)[Sz]) : m_ptr(arr), m_size(Sz - 1) + constexpr StringView(const char (&arr)[Sz]) noexcept : m_ptr(arr), m_size(Sz - 1) { } - constexpr StringView(const char* ptr, size_t size) : m_ptr(ptr), m_size(size) { } - constexpr StringView(const char* b, const char* e) : m_ptr(b), m_size(static_cast(e - b)) { } + constexpr StringView(const char* ptr, size_t size) noexcept : m_ptr(ptr), m_size(size) { } + constexpr StringView(const char* b, const char* e) noexcept : m_ptr(b), m_size(static_cast(e - b)) { } - constexpr const char* begin() const { return m_ptr; } - constexpr const char* end() const { return m_ptr + m_size; } + constexpr const char* begin() const noexcept { return m_ptr; } + constexpr const char* end() const noexcept { return m_ptr + m_size; } - std::reverse_iterator rbegin() const { return std::make_reverse_iterator(end()); } - std::reverse_iterator rend() const { return std::make_reverse_iterator(begin()); } + std::reverse_iterator rbegin() const noexcept { return std::make_reverse_iterator(end()); } + std::reverse_iterator rend() const noexcept { return std::make_reverse_iterator(begin()); } - constexpr const char* data() const { return m_ptr; } - constexpr size_t size() const { return m_size; } + constexpr const char* data() const noexcept { return m_ptr; } + constexpr size_t size() const noexcept { return m_size; } + constexpr bool empty() const noexcept { return m_size == 0; } std::string to_string() const; void to_string(std::string& out) const; - StringView substr(size_t pos, size_t count = std::numeric_limits::max()) const; + StringView substr(size_t pos, size_t count = std::numeric_limits::max()) const noexcept; - constexpr char byte_at_index(size_t pos) const { return m_ptr[pos]; } + constexpr char byte_at_index(size_t pos) const noexcept { return m_ptr[pos]; } private: const char* m_ptr = 0; diff --git a/toolsrc/src/vcpkg-test/strings.cpp b/toolsrc/src/vcpkg-test/strings.cpp index ecf0524bc7..32fe5b3afa 100644 --- a/toolsrc/src/vcpkg-test/strings.cpp +++ b/toolsrc/src/vcpkg-test/strings.cpp @@ -71,3 +71,44 @@ TEST_CASE ("edit distance", "[strings]") REQUIRE(byte_edit_distance("", "hello") == 5); REQUIRE(byte_edit_distance("world", "") == 5); } + +TEST_CASE ("replace_all", "[strings]") +{ + REQUIRE(vcpkg::Strings::replace_all("literal", "ter", "x") == "lixal"); +} + +TEST_CASE ("inplace_replace_all", "[strings]") +{ + using vcpkg::Strings::inplace_replace_all; + std::string target; + inplace_replace_all(target, "", "content"); + REQUIRE(target.empty()); + target = "aa"; + inplace_replace_all(target, "a", "content"); + REQUIRE(target == "contentcontent"); + inplace_replace_all(target, "content", ""); + REQUIRE(target.empty()); + target = "ababababa"; + inplace_replace_all(target, "aba", "X"); + REQUIRE(target == "XbXba"); + target = "ababababa"; + inplace_replace_all(target, "aba", "aba"); + REQUIRE(target == "ababababa"); +} + +TEST_CASE ("inplace_replace_all(char)", "[strings]") +{ + using vcpkg::Strings::inplace_replace_all; + static_assert(noexcept(inplace_replace_all(std::declval(), 'a', 'a'))); + + std::string target; + inplace_replace_all(target, ' ', '?'); + REQUIRE(target.empty()); + target = "hello"; + inplace_replace_all(target, 'l', 'w'); + REQUIRE(target == "hewwo"); + inplace_replace_all(target, 'w', 'w'); + REQUIRE(target == "hewwo"); + inplace_replace_all(target, 'x', '?'); + REQUIRE(target == "hewwo"); +} diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index a59cdaceb7..d8377e3944 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -126,15 +126,30 @@ bool Strings::starts_with(StringView s, StringView pattern) return std::equal(s.begin(), s.begin() + pattern.size(), pattern.begin(), pattern.end()); } -std::string Strings::replace_all(std::string&& s, const std::string& search, StringView rep) +std::string Strings::replace_all(std::string&& s, StringView search, StringView rep) { + inplace_replace_all(s, search, rep); + return std::move(s); +} + +void Strings::inplace_replace_all(std::string& s, StringView search, StringView rep) +{ + if (search.empty()) + { + return; + } + size_t pos = 0; - while ((pos = s.find(search, pos)) != std::string::npos) + while ((pos = s.find(search.data(), pos, search.size())) != std::string::npos) { s.replace(pos, search.size(), rep.data(), rep.size()); pos += rep.size(); } - return std::move(s); +} + +void Strings::inplace_replace_all(std::string& s, char search, char rep) noexcept +{ + std::replace(s.begin(), s.end(), search, rep); } std::string Strings::trim(std::string&& s) diff --git a/toolsrc/src/vcpkg/base/stringview.cpp b/toolsrc/src/vcpkg/base/stringview.cpp index 6e03ba543e..3b980a1955 100644 --- a/toolsrc/src/vcpkg/base/stringview.cpp +++ b/toolsrc/src/vcpkg/base/stringview.cpp @@ -8,7 +8,7 @@ namespace vcpkg { std::vector StringView::find_all_enclosed(const StringView& input, const std::string& left_delim, - const std::string& right_delim) + const std::string& right_delim) noexcept { auto it_left = input.begin(); auto it_right = input.begin(); @@ -35,7 +35,7 @@ namespace vcpkg StringView StringView::find_exactly_one_enclosed(const StringView& input, const std::string& left_tag, - const std::string& right_tag) + const std::string& right_tag) noexcept { std::vector result = find_all_enclosed(input, left_tag, right_tag); Checks::check_exit(VCPKG_LINE_INFO, @@ -50,7 +50,7 @@ namespace vcpkg Optional StringView::find_at_most_one_enclosed(const StringView& input, const std::string& left_tag, - const std::string& right_tag) + const std::string& right_tag) noexcept { std::vector result = find_all_enclosed(input, left_tag, right_tag); Checks::check_exit(VCPKG_LINE_INFO, @@ -69,12 +69,12 @@ namespace vcpkg return result.front(); } - StringView::StringView(const std::string& s) : m_ptr(s.data()), m_size(s.size()) { } + StringView::StringView(const std::string& s) noexcept : m_ptr(s.data()), m_size(s.size()) { } std::string StringView::to_string() const { return std::string(m_ptr, m_size); } void StringView::to_string(std::string& s) const { s.append(m_ptr, m_size); } - StringView StringView::substr(size_t pos, size_t count) const + StringView StringView::substr(size_t pos, size_t count) const noexcept { if (pos > m_size) { diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp index c73e789d0a..cf4888e1be 100644 --- a/toolsrc/src/vcpkg/binarycaching.cpp +++ b/toolsrc/src/vcpkg/binarycaching.cpp @@ -38,7 +38,10 @@ std::unordered_map vcpkg: { std::unordered_map checked; for (auto&& action : plan.install_actions) + { checked.emplace(&action, RestoreResult::missing); + } + provider.precheck(paths, checked); return checked; } @@ -107,10 +110,12 @@ namespace { ArchivesBinaryProvider(std::vector&& read_dirs, std::vector&& write_dirs, - std::vector&& put_url_templates) + std::vector&& put_url_templates, + std::vector&& secrets) : m_read_dirs(std::move(read_dirs)) , m_write_dirs(std::move(write_dirs)) , m_put_url_templates(std::move(put_url_templates)) + , m_secrets(std::move(secrets)) { } @@ -177,12 +182,21 @@ namespace if (code >= 200 && code < 300) { http_remotes_pushed++; + continue; } - else + + auto safe_url = url; + if (!Debug::g_debugging.load(std::memory_order_relaxed)) { - Debug::print("Failed to upload to ", url, ": ", code, '\n'); + for (const auto& secret : m_secrets) + { + Strings::inplace_replace_all(safe_url, secret, "*** SECRET ***"); + } } + + System::print2(System::Color::warning, "Failed to upload to ", safe_url, ": ", code, '\n'); } + if (!m_put_url_templates.empty()) { System::print2("Uploaded binaries to ", http_remotes_pushed, " HTTP remotes.\n"); @@ -197,9 +211,14 @@ namespace fs.create_directories(archive_path.parent_path(), ignore_errors); std::error_code ec; if (m_write_dirs.size() > 1) + { fs.copy_file(tmp_archive_path, archive_path, fs::copy_options::overwrite_existing, ec); + } else + { fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + } + if (ec) { System::printf(System::Color::warning, @@ -208,7 +227,9 @@ namespace ec.message()); } else + { System::printf("Stored binary cache: %s\n", fs::u8string(archive_path)); + } } // In the case of 1 write dir, the file will be moved instead of copied if (m_write_dirs.size() != 1) @@ -223,7 +244,10 @@ namespace for (auto&& result_pair : results_map) { - if (result_pair.second != RestoreResult::missing) continue; + if (result_pair.second != RestoreResult::missing) + { + continue; + } const auto& abi_tag = result_pair.first->abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi; std::error_code ec; @@ -246,6 +270,7 @@ namespace std::vector m_read_dirs; std::vector m_write_dirs; std::vector m_put_url_templates; + std::vector m_secrets; std::set m_restored; }; @@ -266,7 +291,10 @@ namespace for (auto&& action : actions) { auto abi = action->package_abi(); - if (!abi) continue; + if (!abi) + { + continue; + } specs.push_back(action->spec); auto pkgdir = paths.package_dir(action->spec); @@ -311,9 +339,11 @@ namespace RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override { if (Util::Sets::contains(m_restored, action.spec)) + { return RestoreResult::success; - else - return RestoreResult::missing; + } + + return RestoreResult::missing; } void precheck(const VcpkgPaths&, std::unordered_map& results_map) override @@ -332,7 +362,12 @@ namespace urls.push_back(Strings::replace_all(std::string(url_template), "", *abi.get())); url_actions.push_back(result_pair.first); } - if (urls.empty()) break; + + if (urls.empty()) + { + break; + } + auto codes = Downloads::url_heads(urls); Checks::check_exit(VCPKG_LINE_INFO, codes.size() == url_actions.size()); for (size_t i = 0; i < codes.size(); ++i) @@ -360,6 +395,7 @@ namespace { v.erase(0, n); } + return v; } @@ -390,6 +426,7 @@ namespace { System::print2(res.output); } + if (res.output.find("Authentication may require manual action.") != std::string::npos) { System::print2(System::Color::warning, @@ -412,14 +449,19 @@ namespace { System::print2(res2.output); } + return res2.exit_code; } + return res.exit_code; } void prefetch(const VcpkgPaths& paths, std::vector& actions) override { - if (m_read_sources.empty() && m_read_configs.empty()) return; + if (m_read_sources.empty() && m_read_configs.empty()) + { + return; + } auto& fs = paths.get_filesystem(); @@ -427,7 +469,10 @@ namespace for (auto&& action : actions) { - if (!action->has_package_abi()) continue; + if (!action->has_package_abi()) + { + continue; + } auto& spec = action->spec; fs.remove_all(paths.package_dir(spec), VCPKG_LINE_INFO); @@ -435,7 +480,10 @@ namespace nuget_refs.emplace_back(spec, NugetReference(*action)); } - if (nuget_refs.empty()) return; + if (nuget_refs.empty()) + { + return; + } System::print2("Attempting to fetch ", nuget_refs.size(), " packages from nuget.\n"); @@ -447,11 +495,13 @@ namespace xml.open_tag("packages").line_break(); for (auto&& nuget_ref : nuget_refs) + { xml.start_complex_open_tag("package") .text_attr("id", nuget_ref.second.id) .text_attr("version", nuget_ref.second.version) .finish_self_closing_complex_tag() .line_break(); + } xml.close_tag("packages").line_break(); paths.get_filesystem().write_contents(packages_config, xml.buf, VCPKG_LINE_INFO); @@ -483,7 +533,11 @@ namespace .string_arg("-Verbosity") .string_arg("detailed") .string_arg("-ForceEnglishOutput"); - if (!m_interactive) cmdline.string_arg("-NonInteractive"); + if (!m_interactive) + { + cmdline.string_arg("-NonInteractive"); + } + cmdlines.push_back(cmdline.extract()); } for (auto&& cfg : m_read_configs) @@ -509,7 +563,11 @@ namespace .string_arg("-Verbosity") .string_arg("detailed") .string_arg("-ForceEnglishOutput"); - if (!m_interactive) cmdline.string_arg("-NonInteractive"); + if (!m_interactive) + { + cmdline.string_arg("-NonInteractive"); + } + cmdlines.push_back(cmdline.extract()); } @@ -517,7 +575,10 @@ namespace for (const auto& cmdline : cmdlines) { - if (nuget_refs.empty()) break; + if (nuget_refs.empty()) + { + break; + } [&] { generate_packages_config(); @@ -536,10 +597,8 @@ namespace m_restored.emplace(nuget_ref.first); return true; } - else - { - return false; - } + + return false; }); } @@ -554,13 +613,19 @@ namespace RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override { if (Util::Sets::contains(m_restored, action.spec)) + { return RestoreResult::success; - else - return RestoreResult::missing; + } + + return RestoreResult::missing; } void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override { - if (m_write_sources.empty() && m_write_configs.empty()) return; + if (m_write_sources.empty() && m_write_configs.empty()) + { + return; + } + auto& spec = action.spec; NugetReference nuget_ref(action); @@ -603,12 +668,15 @@ namespace .string_arg("-ForceEnglishOutput") .string_arg("-Source") .string_arg(write_src); - if (!m_interactive) cmd.string_arg("-NonInteractive"); + + if (!m_interactive) + { + cmd.string_arg("-NonInteractive"); + } System::print2("Uploading binaries for ", spec, " to NuGet source ", write_src, ".\n"); auto rc = run_nuget_commandline(cmd.extract()); - if (rc != 0) { System::print2(System::Color::error, @@ -629,7 +697,10 @@ namespace .string_arg("-ForceEnglishOutput") .string_arg("-ConfigFile") .path_arg(write_cfg); - if (!m_interactive) cmd.string_arg("-NonInteractive"); + if (!m_interactive) + { + cmd.string_arg("-NonInteractive"); + } System::print2( "Uploading binaries for ", spec, " using NuGet config ", fs::u8string(write_cfg), ".\n"); @@ -644,6 +715,7 @@ namespace " failed. Use --debug for more information.\n"); } } + paths.get_filesystem().remove(nupkg_path, ignore_errors); } } @@ -731,16 +803,25 @@ namespace path.make_preferred(); const auto status = fs::stdfs::status(path); if (!fs::stdfs::exists(status)) + { return {"Path to VCPKG_DEFAULT_BINARY_CACHE does not exist: " + fs::u8string(path), expected_right_tag}; + } + if (!fs::stdfs::is_directory(status)) + { return {"Value of environment variable VCPKG_DEFAULT_BINARY_CACHE is not a directory: " + fs::u8string(path), expected_right_tag}; + } + if (!path.is_absolute()) + { return {"Value of environment variable VCPKG_DEFAULT_BINARY_CACHE is not absolute: " + fs::u8string(path), expected_right_tag}; + } + return {std::move(path), expected_left_tag}; } p /= fs::u8path("vcpkg/archives"); @@ -774,6 +855,8 @@ namespace std::vector configs_to_read; std::vector configs_to_write; + std::vector secrets; + void clear() { m_cleared = true; @@ -786,6 +869,7 @@ namespace sources_to_write.clear(); configs_to_read.clear(); configs_to_write.clear(); + secrets.clear(); } }; @@ -814,37 +898,61 @@ namespace Strings::append(segment, n); auto ch = cur(); if (ch == Unicode::end_of_file || ch == ',' || ch == ';') + { break; - else if (ch == '`') + } + + if (ch == '`') { ch = next(); if (ch == Unicode::end_of_file) + { add_error("unexpected eof: trailing unescaped backticks (`) are not allowed"); + } else + { Unicode::utf8_append_code_point(segment, ch); + } + next(); } else + { Checks::unreachable(VCPKG_LINE_INFO); + } } segments.emplace_back(std::move(loc), std::move(segment)); auto ch = cur(); if (ch == Unicode::end_of_file || ch == ';') + { break; - else if (ch == ',') + } + + if (ch == ',') { next(); continue; } - else - Checks::unreachable(VCPKG_LINE_INFO); + + Checks::unreachable(VCPKG_LINE_INFO); + } + + if (segments.size() != 1 || !segments[0].second.empty()) + { + handle_segments(std::move(segments)); } - if (segments.size() != 1 || !segments[0].second.empty()) handle_segments(std::move(segments)); segments.clear(); - if (get_error()) return; - if (cur() == ';') next(); + if (get_error()) + { + return; + } + + if (cur() == ';') + { + next(); + } } } @@ -889,8 +997,11 @@ namespace if (segments[0].second == "clear") { if (segments.size() != 1) + { return add_error("unexpected arguments: binary config 'clear' does not take arguments", segments[1].first); + } + state->clear(); } else if (segments[0].second == "files") @@ -907,48 +1018,67 @@ namespace return add_error("expected arguments: path arguments for binary config strings must be absolute", segments[1].first); } + handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 2); if (segments.size() > 3) + { return add_error("unexpected arguments: binary config 'files' requires 1 or 2 arguments", segments[3].first); + } } else if (segments[0].second == "interactive") { if (segments.size() > 1) + { return add_error("unexpected arguments: binary config 'interactive' does not accept any arguments", segments[1].first); + } + state->interactive = true; } else if (segments[0].second == "nugetconfig") { if (segments.size() < 2) + { return add_error( "expected arguments: binary config 'nugetconfig' requires at least a source argument", segments[0].first); + } auto p = fs::u8path(segments[1].second); if (!p.is_absolute()) + { return add_error("expected arguments: path arguments for binary config strings must be absolute", segments[1].first); + } + handle_readwrite(state->configs_to_read, state->configs_to_write, std::move(p), segments, 2); if (segments.size() > 3) + { return add_error("unexpected arguments: binary config 'nugetconfig' requires 1 or 2 arguments", segments[3].first); + } } else if (segments[0].second == "nuget") { if (segments.size() < 2) + { return add_error("expected arguments: binary config 'nuget' requires at least a source argument", segments[0].first); + } auto&& p = segments[1].second; if (p.empty()) + { return add_error("unexpected arguments: binary config 'nuget' requires non-empty source"); + } handle_readwrite(state->sources_to_read, state->sources_to_write, std::move(p), segments, 2); if (segments.size() > 3) + { return add_error("unexpected arguments: binary config 'nuget' requires 1 or 2 arguments", segments[3].first); + } } else if (segments[0].second == "default") { @@ -959,7 +1089,10 @@ namespace } const auto& maybe_home = default_cache_path(); - if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first); + if (!maybe_home.has_value()) + { + return add_error(maybe_home.error(), segments[0].first); + } handle_readwrite( state->archives_to_read, state->archives_to_write, fs::path(*maybe_home.get()), segments, 1); @@ -973,30 +1106,43 @@ namespace "expected arguments: binary config 'azblob' requires at least a base-url and a SAS token", segments[0].first); } + if (!Strings::starts_with(segments[1].second, "https://")) { return add_error( "invalid argument: binary config 'azblob' requires an https base url as the first argument", segments[1].first); } + if (Strings::starts_with(segments[2].second, "?")) { return add_error("invalid argument: binary config 'azblob' requires a SAS token without a " "preceeding '?' as the second argument", segments[2].first); } - auto p = segments[1].second; - if (p.back() != '/') p.push_back('/'); - p.append(".zip"); - if (!Strings::starts_with(segments[2].second, "?")) p.push_back('?'); - p.append(segments[2].second); - handle_readwrite( - state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3); + if (segments.size() > 4) { return add_error("unexpected arguments: binary config 'azblob' requires 2 or 3 arguments", segments[4].first); } + + auto p = segments[1].second; + if (p.back() != '/') + { + p.push_back('/'); + } + + p.append(".zip"); + if (!Strings::starts_with(segments[2].second, "?")) + { + p.push_back('?'); + } + + p.append(segments[2].second); + state->secrets.push_back(segments[2].second); + handle_readwrite( + state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3); } else { @@ -1016,9 +1162,13 @@ ExpectedS> vcpkg::create_binary_provider_from_c { const auto& cachepath = default_cache_path(); if (cachepath.has_value()) + { Debug::print("Default binary cache path is: ", fs::u8string(*cachepath.get()), '\n'); + } else + { Debug::print("No binary cache path. Reason: ", cachepath.error(), '\n'); + } } return create_binary_provider_from_configs_pure(env_string, args); @@ -1029,39 +1179,63 @@ ExpectedS> vcpkg::create_binary_provider_from_c { { auto metrics = Metrics::g_metrics.lock(); - if (!env_string.empty()) metrics->track_property("VCPKG_BINARY_SOURCES", "defined"); - if (args.size() != 0) metrics->track_property("binarycaching-source", "defined"); + if (!env_string.empty()) + { + metrics->track_property("VCPKG_BINARY_SOURCES", "defined"); + } + + if (args.size() != 0) + { + metrics->track_property("binarycaching-source", "defined"); + } } State s; BinaryConfigParser default_parser("default,readwrite", "", &s); default_parser.parse(); - if (auto err = default_parser.get_error()) return err->get_message(); + if (auto err = default_parser.get_error()) + { + return err->get_message(); + } BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES", &s); env_parser.parse(); - if (auto err = env_parser.get_error()) return err->format(); + if (auto err = env_parser.get_error()) + { + return err->format(); + } + for (auto&& arg : args) { BinaryConfigParser arg_parser(arg, "", &s); arg_parser.parse(); - if (auto err = arg_parser.get_error()) return err->format(); + if (auto err = arg_parser.get_error()) + { + return err->format(); + } } - if (s.m_cleared) Metrics::g_metrics.lock()->track_property("binarycaching-clear", "defined"); + if (s.m_cleared) + { + Metrics::g_metrics.lock()->track_property("binarycaching-clear", "defined"); + } std::vector> providers; if (!s.archives_to_read.empty() || !s.archives_to_write.empty() || !s.azblob_templates_to_put.empty()) { - providers.push_back(std::make_unique( - std::move(s.archives_to_read), std::move(s.archives_to_write), std::move(s.azblob_templates_to_put))); + providers.push_back(std::make_unique(std::move(s.archives_to_read), + std::move(s.archives_to_write), + std::move(s.azblob_templates_to_put), + std::move(s.secrets))); } + if (!s.url_templates_to_get.empty()) { Metrics::g_metrics.lock()->track_property("binarycaching-url-get", "defined"); providers.push_back(std::make_unique(std::move(s.url_templates_to_get))); } + if (!s.sources_to_read.empty() || !s.sources_to_write.empty() || !s.configs_to_read.empty() || !s.configs_to_write.empty()) { @@ -1112,13 +1286,20 @@ details::NuGetRepoInfo details::get_nuget_repo_info_from_env() Metrics::g_metrics.lock()->track_property("VCPKG_NUGET_REPOSITORY", "defined"); return {std::move(*p)}; } + auto gh_repo = System::get_environment_variable("GITHUB_REPOSITORY").value_or(""); - if (gh_repo.empty()) return {}; + if (gh_repo.empty()) + { + return {}; + } + auto gh_server = System::get_environment_variable("GITHUB_SERVER_URL").value_or(""); - if (gh_server.empty()) return {}; + if (gh_server.empty()) + { + return {}; + } Metrics::g_metrics.lock()->track_property("GITHUB_REPOSITORY", "defined"); - return {Strings::concat(gh_server, '/', gh_repo, ".git"), System::get_environment_variable("GITHUB_REF").value_or(""), System::get_environment_variable("GITHUB_SHA").value_or("")}; @@ -1155,12 +1336,17 @@ std::string vcpkg::generate_nuspec(const VcpkgPaths& paths, { Strings::append(description, " ", dep.name(), '\n'); } + XmlSerializer xml; xml.open_tag("package").line_break(); xml.open_tag("metadata").line_break(); xml.simple_tag("id", ref.id).line_break(); xml.simple_tag("version", ref.version).line_break(); - if (!scf.core_paragraph->homepage.empty()) xml.simple_tag("projectUrl", scf.core_paragraph->homepage); + if (!scf.core_paragraph->homepage.empty()) + { + xml.simple_tag("projectUrl", scf.core_paragraph->homepage); + } + xml.simple_tag("authors", "vcpkg").line_break(); xml.simple_tag("description", description).line_break(); xml.open_tag("packageTypes"); @@ -1169,10 +1355,19 @@ std::string vcpkg::generate_nuspec(const VcpkgPaths& paths, if (!rinfo.repo.empty()) { xml.start_complex_open_tag("repository").text_attr("type", "git").text_attr("url", rinfo.repo); - if (!rinfo.branch.empty()) xml.text_attr("branch", rinfo.branch); - if (!rinfo.commit.empty()) xml.text_attr("commit", rinfo.commit); + if (!rinfo.branch.empty()) + { + xml.text_attr("branch", rinfo.branch); + } + + if (!rinfo.commit.empty()) + { + xml.text_attr("commit", rinfo.commit); + } + xml.finish_self_closing_complex_tag().line_break(); } + xml.close_tag("metadata").line_break(); xml.open_tag("files"); xml.start_complex_open_tag("file") @@ -1237,6 +1432,7 @@ void vcpkg::help_topic_binary_caching(const VcpkgPaths&) fs::u8string(*p), "\nThis consults %LOCALAPPDATA%/%APPDATA% on Windows and $XDG_CACHE_HOME or $HOME on other platforms.\n"); } + System::print2("\nExtended documentation is available at " "https://github.com/Microsoft/vcpkg/tree/master/docs/users/binarycaching.md \n"); } @@ -1256,6 +1452,7 @@ std::string vcpkg::generate_nuget_packages_config(const Dependencies::ActionPlan .finish_self_closing_complex_tag() .line_break(); } + xml.close_tag("packages").line_break(); return std::move(xml.buf); } diff --git a/toolsrc/src/vcpkg/commands.integrate.cpp b/toolsrc/src/vcpkg/commands.integrate.cpp index ce15222558..1b1dee6901 100644 --- a/toolsrc/src/vcpkg/commands.integrate.cpp +++ b/toolsrc/src/vcpkg/commands.integrate.cpp @@ -112,8 +112,8 @@ namespace vcpkg::Commands::Integrate )"; std::string content = Strings::replace_all(CONTENT_TEMPLATE, "@NUGET_ID@", nuget_id); - content = Strings::replace_all(std::move(content), "@VCPKG_DIR@", vcpkg_root_dir.string()); - content = Strings::replace_all(std::move(content), "@VERSION@", nupkg_version); + Strings::inplace_replace_all(content, "@VCPKG_DIR@", vcpkg_root_dir.string()); + Strings::inplace_replace_all(content, "@VERSION@", nupkg_version); return content; } #endif @@ -393,7 +393,7 @@ CMake projects should use: "-DCMAKE_TOOLCHAIN_FILE=%s" System::print2(System::Color::success, "Created nupkg: ", fs::u8string(nuget_package), '\n'); auto source_path = fs::u8string(buildsystems_dir); - source_path = Strings::replace_all(std::move(source_path), "`", "``"); + Strings::inplace_replace_all(source_path, "`", "``"); System::printf(R"( With a project open, go to Tools->NuGet Package Manager->Package Manager Console and paste: diff --git a/toolsrc/src/vcpkg/export.chocolatey.cpp b/toolsrc/src/vcpkg/export.chocolatey.cpp index 099bcf400e..b9f14a4d1e 100644 --- a/toolsrc/src/vcpkg/export.chocolatey.cpp +++ b/toolsrc/src/vcpkg/export.chocolatey.cpp @@ -27,7 +27,7 @@ namespace vcpkg::Export::Chocolatey Checks::exit_with_message(VCPKG_LINE_INFO, "Cannot find desired dependency version."); } std::string nuspec_dependency = Strings::replace_all(CONTENT_TEMPLATE, "@PACKAGE_ID@", depend); - nuspec_dependency = Strings::replace_all(std::move(nuspec_dependency), "@PACKAGE_VERSION@", found->second); + Strings::inplace_replace_all(nuspec_dependency, "@PACKAGE_VERSION@", found->second); nuspec_dependencies += nuspec_dependency; } return nuspec_dependencies; @@ -64,17 +64,15 @@ namespace vcpkg::Export::Chocolatey } std::string nuspec_file_content = Strings::replace_all(CONTENT_TEMPLATE, "@PACKAGE_ID@", binary_paragraph.spec.name()); - nuspec_file_content = - Strings::replace_all(std::move(nuspec_file_content), "@PACKAGE_VERSION@", package_version->second); - nuspec_file_content = Strings::replace_all( - std::move(nuspec_file_content), "@PACKAGE_MAINTAINER@", chocolatey_options.maybe_maintainer.value_or("")); - nuspec_file_content = Strings::replace_all( - std::move(nuspec_file_content), "@PACKAGE_DESCRIPTION@", Strings::join("\n", binary_paragraph.description)); - nuspec_file_content = - Strings::replace_all(std::move(nuspec_file_content), "@EXPORTED_ROOT_DIR@", exported_root_dir); - nuspec_file_content = Strings::replace_all(std::move(nuspec_file_content), - "@PACKAGE_DEPENDENCIES@", - create_nuspec_dependencies(binary_paragraph, packages_version)); + Strings::inplace_replace_all(nuspec_file_content, "@PACKAGE_VERSION@", package_version->second); + Strings::inplace_replace_all( + nuspec_file_content, "@PACKAGE_MAINTAINER@", chocolatey_options.maybe_maintainer.value_or("")); + Strings::inplace_replace_all( + nuspec_file_content, "@PACKAGE_DESCRIPTION@", Strings::join("\n", binary_paragraph.description)); + Strings::inplace_replace_all(nuspec_file_content, "@EXPORTED_ROOT_DIR@", exported_root_dir); + Strings::inplace_replace_all(nuspec_file_content, + "@PACKAGE_DEPENDENCIES@", + create_nuspec_dependencies(binary_paragraph, packages_version)); return nuspec_file_content; } @@ -179,10 +177,10 @@ if (Test-Path $installedDir) const BinaryParagraph& binary_paragraph = action.core_paragraph().value_or_exit(VCPKG_LINE_INFO); auto norm_version = binary_paragraph.version; - // normalize the version string to be separated by dots to be compliant with Nusepc. - norm_version = Strings::replace_all(std::move(norm_version), "-", "."); - norm_version = Strings::replace_all(std::move(norm_version), "_", "."); - norm_version = norm_version + chocolatey_options.maybe_version_suffix.value_or(""); + // normalize the version string to be separated by dots to be compliant with Nuspec. + Strings::inplace_replace_all(norm_version, '-', '.'); + Strings::inplace_replace_all(norm_version, '_', '.'); + norm_version += chocolatey_options.maybe_version_suffix.value_or(""); packages_version.insert(std::make_pair(binary_paragraph.spec.name(), norm_version)); } diff --git a/toolsrc/src/vcpkg/export.prefab.cpp b/toolsrc/src/vcpkg/export.prefab.cpp index ce4e0cf6a8..38d4568a8c 100644 --- a/toolsrc/src/vcpkg/export.prefab.cpp +++ b/toolsrc/src/vcpkg/export.prefab.cpp @@ -103,9 +103,9 @@ namespace vcpkg::Export::Prefab "stl":"@STL@" })"; std::string json = Strings::replace_all(std::move(TEMPLATE), "@ABI@", abi); - json = Strings::replace_all(std::move(json), "@API@", std::to_string(api)); - json = Strings::replace_all(std::move(json), "@NDK@", std::to_string(ndk)); - json = Strings::replace_all(std::move(json), "@STL@", stl); + Strings::inplace_replace_all(json, "@API@", std::to_string(api)); + Strings::inplace_replace_all(json, "@NDK@", std::to_string(ndk)); + Strings::inplace_replace_all(json, "@STL@", stl); return json; } @@ -117,7 +117,7 @@ namespace vcpkg::Export::Prefab })"; std::string json = Strings::replace_all(std::move(TEMPLATE), "@LIBRARY_NAME@", null_if_empty(library_name)); - json = Strings::replace_all(std::move(json), "@LIBRARIES@", null_if_empty_array(jsonify(export_libraries))); + Strings::inplace_replace_all(json, "@LIBRARIES@", null_if_empty_array(jsonify(export_libraries))); return json; } @@ -130,8 +130,8 @@ namespace vcpkg::Export::Prefab })"; std::string json = Strings::replace_all(std::move(TEMPLATE), "@LIBRARY_NAME@", null_if_empty(library_name)); - json = Strings::replace_all(std::move(json), "@LIBRARIES@", jsonify(export_libraries)); - json = Strings::replace_all(std::move(json), "@ANDROID_METADATA@", android.to_json()); + Strings::inplace_replace_all(json, "@LIBRARIES@", jsonify(export_libraries)); + Strings::inplace_replace_all(json, "@ANDROID_METADATA@", android.to_json()); return json; } @@ -146,9 +146,9 @@ namespace vcpkg::Export::Prefab "version":"@PACKAGE_VERSION@" })"; std::string json = Strings::replace_all(std::move(TEMPLATE), "@PACKAGE_NAME@", name); - json = Strings::replace_all(std::move(json), "@PACKAGE_SCHEMA@", std::to_string(schema)); - json = Strings::replace_all(std::move(json), "@PACKAGE_DEPS@", deps); - json = Strings::replace_all(std::move(json), "@PACKAGE_VERSION@", version); + Strings::inplace_replace_all(json, "@PACKAGE_SCHEMA@", std::to_string(schema)); + Strings::inplace_replace_all(json, "@PACKAGE_DEPS@", deps); + Strings::inplace_replace_all(json, "@PACKAGE_VERSION@", version); return json; } @@ -426,9 +426,9 @@ namespace vcpkg::Export::Prefab )"; std::string manifest = Strings::replace_all(std::move(MANIFEST_TEMPLATE), "@GROUP_ID@", group_id); - manifest = Strings::replace_all(std::move(manifest), "@ARTIFACT_ID@", artifact_id); - manifest = Strings::replace_all(std::move(manifest), "@MIN_SDK_VERSION@", sdk_min_version); - manifest = Strings::replace_all(std::move(manifest), "@SDK_TARGET_VERSION@", sdk_target_version); + Strings::inplace_replace_all(manifest, "@ARTIFACT_ID@", artifact_id); + Strings::inplace_replace_all(manifest, "@MIN_SDK_VERSION@", sdk_min_version); + Strings::inplace_replace_all(manifest, "@SDK_TARGET_VERSION@", sdk_target_version); fs::path manifest_path = package_directory / "AndroidManifest.xml"; fs::path prefab_path = prefab_directory / "prefab.json"; @@ -484,8 +484,8 @@ namespace vcpkg::Export::Prefab runtime )"; std::string pom = Strings::replace_all(std::move(maven_pom), "@GROUP_ID@", group_id); - pom = Strings::replace_all(std::move(pom), "@ARTIFACT_ID@", it.name()); - pom = Strings::replace_all(std::move(pom), "@VERSION@", version_map[it.name()]); + Strings::inplace_replace_all(pom, "@ARTIFACT_ID@", it.name()); + Strings::inplace_replace_all(pom, "@VERSION@", version_map[it.name()]); pom_dependencies.push_back(pom); pm.dependencies.push_back(it.name()); } @@ -670,9 +670,9 @@ namespace vcpkg::Export::Prefab )"; std::string pom = Strings::replace_all(std::move(POM), "@GROUP_ID@", group_id); - pom = Strings::replace_all(std::move(pom), "@ARTIFACT_ID@", artifact_id); - pom = Strings::replace_all(std::move(pom), "@DEPENDENCIES@", Strings::join("\n", pom_dependencies)); - pom = Strings::replace_all(std::move(pom), "@VERSION@", norm_version); + Strings::inplace_replace_all(pom, "@ARTIFACT_ID@", artifact_id); + Strings::inplace_replace_all(pom, "@DEPENDENCIES@", Strings::join("\n", pom_dependencies)); + Strings::inplace_replace_all(pom, "@VERSION@", norm_version); utils.write_contents(pom_path, pom, VCPKG_LINE_INFO);