From e9f8cc67a5e5541973e53ac03f88adb45cc1b21b Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Mon, 26 Oct 2020 10:24:30 -0700 Subject: [PATCH] [vcpkg] Implement versions db generator (#13777) * [vcpkg] Add script to generate ports versions history * [vcpkg] Fix formatting * Fetch port versions from commit ID * Use global --x-json switch * Use --no-checkout when cloning secondary instance * Clone from local repository instead of from GitHub * Use CmdLineBuilder to build git commands * Use CmdLineBuilder and reduce repeated code * Fetch version at baseline and code cleanup * Guess version scheme from old CONTROL files * Rename version db generator script * Simplify x-history json output * Use CONTROL/manifest parsers on x-history * Use git-tree instaed of commit-id * Remove 'ports' field from root object * Clean up code * More code cleanup * Improve port version detection * Improve generator logging * Do not ignore parsing errors in CONTROL files * PR review comments in Python script * Fix subprocess.run() calls * Make `canonicalize()` return error instead of terminating * [vcpkg] Add tests for new test_parse_control_file paths * Remove unnecessary std::move() calls * Fix formatting * Python formatting Co-authored-by: Robert Schumacher --- scripts/generatePortVersionsDb.py | 71 ++++ toolsrc/include/vcpkg/paragraphs.h | 6 + toolsrc/include/vcpkg/sourceparagraph.h | 5 +- toolsrc/include/vcpkg/versions.h | 6 +- toolsrc/src/vcpkg-test/binarycaching.cpp | 6 +- toolsrc/src/vcpkg-test/manifests.cpp | 6 +- toolsrc/src/vcpkg-test/paragraph.cpp | 50 +++ .../src/vcpkg/commands.format-manifest.cpp | 4 +- toolsrc/src/vcpkg/commands.porthistory.cpp | 332 ++++++++++++++---- toolsrc/src/vcpkg/paragraphs.cpp | 64 +++- toolsrc/src/vcpkg/sourceparagraph.cpp | 95 +++-- toolsrc/windows-bootstrap/vcpkg.vcxproj | 1 + 12 files changed, 539 insertions(+), 107 deletions(-) create mode 100644 scripts/generatePortVersionsDb.py diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py new file mode 100644 index 0000000000..cefd61e1c9 --- /dev/null +++ b/scripts/generatePortVersionsDb.py @@ -0,0 +1,71 @@ +import os +import os.path +import sys +import subprocess +import json +import time + +from subprocess import CalledProcessError +from json.decoder import JSONDecodeError +from pathlib import Path + + +SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + +def get_current_git_ref(): + output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], + capture_output=True, + encoding='utf-8') + if output.returncode == 0: + return output.stdout.strip() + print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr) + return None + + +def generate_port_versions_db(ports_path, db_path, revision): + start_time = time.time() + port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))] + total_count = len(port_names) + for counter, port_name in enumerate(port_names): + containing_dir = os.path.join(db_path, f'{port_name[0]}-') + os.makedirs(containing_dir, exist_ok=True) + output_filepath = os.path.join(containing_dir, f'{port_name}.json') + if not os.path.exists(output_filepath): + output = subprocess.run( + [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], + capture_output=True, encoding='utf-8') + if output.returncode == 0: + try: + versions_object = json.loads(output.stdout) + with open(output_filepath, 'w') as output_file: + json.dump(versions_object, output_file) + except JSONDecodeError: + print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr) + else: + print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr) + # This should be replaced by a progress bar + if counter > 0 and counter % 100 == 0: + elapsed_time = time.time() - start_time + print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds') + rev_file = os.path.join(db_path, revision) + Path(rev_file).touch() + elapsed_time = time.time() - start_time + print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds') + + +def main(ports_path, db_path): + revision = get_current_git_ref() + if not revision: + print('Couldn\'t fetch current Git revision', file=sys.stderr) + sys.exit(1) + rev_file = os.path.join(db_path, revision) + if os.path.exists(rev_file): + print(f'Database files already exist for commit {revision}') + sys.exit(0) + generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision) + + +if __name__ == "__main__": + main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), + db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions')) diff --git a/toolsrc/include/vcpkg/paragraphs.h b/toolsrc/include/vcpkg/paragraphs.h index f4ea36edef..0293f0733f 100644 --- a/toolsrc/include/vcpkg/paragraphs.h +++ b/toolsrc/include/vcpkg/paragraphs.h @@ -17,12 +17,18 @@ namespace vcpkg::Paragraphs ExpectedS parse_single_paragraph(const std::string& str, const std::string& origin); ExpectedS get_single_paragraph(const Files::Filesystem& fs, const fs::path& control_path); + ExpectedS> get_paragraphs(const Files::Filesystem& fs, const fs::path& control_path); + ExpectedS> get_paragraphs_text(const std::string& text, const std::string& origin); + ExpectedS> parse_paragraphs(const std::string& str, const std::string& origin); bool is_port_directory(const Files::Filesystem& fs, const fs::path& path); Parse::ParseExpected try_load_port(const Files::Filesystem& fs, const fs::path& path); + Parse::ParseExpected try_load_port_text(const std::string& text, + const std::string& origin, + bool is_manifest); ExpectedS try_load_cached_package(const VcpkgPaths& paths, const PackageSpec& spec); diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 21e5a4ef9f..d88c0b0fd8 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -91,11 +91,14 @@ namespace vcpkg return ret; } + static Parse::ParseExpected parse_manifest_object(const std::string& origin, + const Json::Object& object); + static Parse::ParseExpected parse_manifest_file(const fs::path& path_to_manifest, const Json::Object& object); static Parse::ParseExpected parse_control_file( - const fs::path& path_to_control, std::vector&& control_paragraphs); + const std::string& origin, std::vector&& control_paragraphs); // Always non-null in non-error cases std::unique_ptr core_paragraph; diff --git a/toolsrc/include/vcpkg/versions.h b/toolsrc/include/vcpkg/versions.h index f0304c7885..331bf7f8b5 100644 --- a/toolsrc/include/vcpkg/versions.h +++ b/toolsrc/include/vcpkg/versions.h @@ -1,14 +1,12 @@ #pragma once -#include - namespace vcpkg::Versions { enum class Scheme { - String, Relaxed, Semver, - Date + Date, + String }; } diff --git a/toolsrc/src/vcpkg-test/binarycaching.cpp b/toolsrc/src/vcpkg-test/binarycaching.cpp index 33f63e2ea3..cb0d7cb99d 100644 --- a/toolsrc/src/vcpkg-test/binarycaching.cpp +++ b/toolsrc/src/vcpkg-test/binarycaching.cpp @@ -75,7 +75,7 @@ Build-Depends: bzip )", ""); REQUIRE(pghs.has_value()); - auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get())); + auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get())); REQUIRE(maybe_scf.has_value()); SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()}; @@ -255,7 +255,7 @@ Description: a spiffy compression library wrapper )", ""); REQUIRE(pghs.has_value()); - auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get())); + auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get())); REQUIRE(maybe_scf.has_value()); SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()}; plan.install_actions.push_back(Dependencies::InstallPlanAction()); @@ -278,7 +278,7 @@ Description: a spiffy compression library wrapper )", ""); REQUIRE(pghs2.has_value()); - auto maybe_scf2 = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs2.get())); + auto maybe_scf2 = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs2.get())); REQUIRE(maybe_scf2.has_value()); SourceControlFileLocation scfl2{std::move(*maybe_scf2.get()), fs::path()}; plan.install_actions.push_back(Dependencies::InstallPlanAction()); diff --git a/toolsrc/src/vcpkg-test/manifests.cpp b/toolsrc/src/vcpkg-test/manifests.cpp index 75b14771f0..5ec3f160a3 100644 --- a/toolsrc/src/vcpkg-test/manifests.cpp +++ b/toolsrc/src/vcpkg-test/manifests.cpp @@ -331,9 +331,9 @@ TEST_CASE ("Serialize all the ports", "[manifests]") auto pghs = Paragraphs::parse_paragraphs(contents, fs::u8string(control)); REQUIRE(pghs); - scfs.push_back(std::move( - *SourceControlFile::parse_control_file(control, std::move(pghs).value_or_exit(VCPKG_LINE_INFO)) - .value_or_exit(VCPKG_LINE_INFO))); + scfs.push_back(std::move(*SourceControlFile::parse_control_file( + fs::u8string(control), std::move(pghs).value_or_exit(VCPKG_LINE_INFO)) + .value_or_exit(VCPKG_LINE_INFO))); } else if (fs.exists(manifest)) { diff --git a/toolsrc/src/vcpkg-test/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp index 05ba8cfba3..356ee88f8b 100644 --- a/toolsrc/src/vcpkg-test/paragraph.cpp +++ b/toolsrc/src/vcpkg-test/paragraph.cpp @@ -51,6 +51,38 @@ TEST_CASE ("SourceParagraph construct minimum", "[paragraph]") REQUIRE(pgh.core_paragraph->dependencies.size() == 0); } +TEST_CASE ("SourceParagraph construct invalid", "[paragraph]") +{ + auto m_pgh = test_parse_control_file({{ + {"Source", "zlib"}, + {"Version", "1.2.8"}, + {"Build-Depends", "1.2.8"}, + }}); + + REQUIRE(!m_pgh.has_value()); + + m_pgh = test_parse_control_file({{ + {"Source", "zlib"}, + {"Version", "1.2.8"}, + {"Default-Features", "1.2.8"}, + }}); + + REQUIRE(!m_pgh.has_value()); + + m_pgh = test_parse_control_file({ + { + {"Source", "zlib"}, + {"Version", "1.2.8"}, + }, + { + {"Feature", "a"}, + {"Build-Depends", "1.2.8"}, + }, + }); + + REQUIRE(!m_pgh.has_value()); +} + TEST_CASE ("SourceParagraph construct maximum", "[paragraph]") { auto m_pgh = test_parse_control_file({{ @@ -76,6 +108,24 @@ TEST_CASE ("SourceParagraph construct maximum", "[paragraph]") REQUIRE(pgh.core_paragraph->default_features[0] == "df"); } +TEST_CASE ("SourceParagraph construct feature", "[paragraph]") +{ + auto m_pgh = test_parse_control_file({ + { + {"Source", "s"}, + {"Version", "v"}, + }, + {{"Feature", "f"}, {"Description", "d2"}, {"Build-Depends", "bd2"}}, + }); + REQUIRE(m_pgh.has_value()); + auto& pgh = **m_pgh.get(); + + REQUIRE(pgh.feature_paragraphs.size() == 1); + REQUIRE(pgh.feature_paragraphs[0]->name == "f"); + REQUIRE(pgh.feature_paragraphs[0]->description == std::vector{"d2"}); + REQUIRE(pgh.feature_paragraphs[0]->dependencies.size() == 1); +} + TEST_CASE ("SourceParagraph two dependencies", "[paragraph]") { auto m_pgh = test_parse_control_file({{ diff --git a/toolsrc/src/vcpkg/commands.format-manifest.cpp b/toolsrc/src/vcpkg/commands.format-manifest.cpp index 1be6ecbeb8..8d4be1ce98 100644 --- a/toolsrc/src/vcpkg/commands.format-manifest.cpp +++ b/toolsrc/src/vcpkg/commands.format-manifest.cpp @@ -80,8 +80,8 @@ namespace paragraphs.error()); return {}; } - auto scf_res = - SourceControlFile::parse_control_file(control_path, std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO)); + auto scf_res = SourceControlFile::parse_control_file(fs::u8string(control_path), + std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO)); if (!scf_res) { System::printf(System::Color::error, "Failed to parse control file: %s\n", control_path_string); diff --git a/toolsrc/src/vcpkg/commands.porthistory.cpp b/toolsrc/src/vcpkg/commands.porthistory.cpp index 4313ab647e..aa011b9993 100644 --- a/toolsrc/src/vcpkg/commands.porthistory.cpp +++ b/toolsrc/src/vcpkg/commands.porthistory.cpp @@ -1,77 +1,257 @@ +#include #include #include #include #include #include +#include #include #include #include +#include namespace vcpkg::Commands::PortHistory { - struct PortControlVersion + namespace { - std::string commit_id; - std::string version; - std::string date; - }; - - static System::ExitCodeAndOutput run_git_command(const VcpkgPaths& paths, const std::string& cmd) - { - const fs::path& git_exe = paths.get_tool_exe(Tools::GIT); - const fs::path dot_git_dir = paths.root / ".git"; - - const std::string full_cmd = - Strings::format(R"("%s" --git-dir="%s" %s)", fs::u8string(git_exe), fs::u8string(dot_git_dir), cmd); - - auto output = System::cmd_execute_and_capture_output(full_cmd); - Checks::check_exit(VCPKG_LINE_INFO, output.exit_code == 0, "Failed to run command: %s", full_cmd); - return output; - } - - static std::string get_version_from_commit(const VcpkgPaths& paths, - const std::string& commit_id, - const std::string& port_name) - { - const std::string cmd = Strings::format(R"(show %s:ports/%s/CONTROL)", commit_id, port_name); - auto output = run_git_command(paths, cmd); - - const auto version = Strings::find_at_most_one_enclosed(output.output, "\nVersion: ", "\n"); - const auto port_version = Strings::find_at_most_one_enclosed(output.output, "\nPort-Version: ", "\n"); - Checks::check_exit(VCPKG_LINE_INFO, version.has_value(), "CONTROL file does not have a 'Version' field"); - if (auto pv = port_version.get()) + struct HistoryVersion { - return Strings::format("%s#%s", version.get()->to_string(), pv->to_string()); + std::string port_name; + std::string git_tree; + std::string commit_id; + std::string commit_date; + std::string version_string; + std::string version; + int port_version; + Versions::Scheme scheme; + }; + + const System::ExitCodeAndOutput run_git_command_inner(const VcpkgPaths& paths, + const fs::path& dot_git_directory, + const fs::path& working_directory, + const std::string& cmd) + { + const fs::path& git_exe = paths.get_tool_exe(Tools::GIT); + + System::CmdLineBuilder builder; + builder.path_arg(git_exe) + .string_arg(Strings::concat("--git-dir=", fs::u8string(dot_git_directory))) + .string_arg(Strings::concat("--work-tree=", fs::u8string(working_directory))); + const std::string full_cmd = Strings::concat(builder.extract(), " ", cmd); + + const auto output = System::cmd_execute_and_capture_output(full_cmd); + return output; } - return version.get()->to_string(); - } - - static std::vector read_versions_from_log(const VcpkgPaths& paths, const std::string& port_name) - { - const std::string cmd = - Strings::format(R"(log --format="%%H %%cd" --date=short --left-only -- ports/%s/.)", port_name); - auto output = run_git_command(paths, cmd); - - auto commits = Util::fmap( - Strings::split(output.output, '\n'), [](const std::string& line) -> auto { - auto parts = Strings::split(line, ' '); - return std::make_pair(parts[0], parts[1]); - }); - - std::vector ret; - std::string last_version; - for (auto&& commit_date_pair : commits) + const System::ExitCodeAndOutput run_git_command(const VcpkgPaths& paths, const std::string& cmd) { - const std::string version = get_version_from_commit(paths, commit_date_pair.first, port_name); - if (last_version != version) + const fs::path& work_dir = paths.root; + const fs::path dot_git_dir = paths.root / ".git"; + + return run_git_command_inner(paths, dot_git_dir, work_dir, cmd); + } + + bool is_date(const std::string& version_string) + { + // The date regex is not complete, it matches strings that look like dates, + // e.g.: 2020-99-99. + // + // The regex has two capture groups: + // * Date: "^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})", it matches strings that resemble YYYY-MM-DD. + // It does not validate that MM <= 12, or that DD is possible with the given MM. + // YYYY should be AT LEAST 4 digits, for some kind of "future proofing". + std::regex re("^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})((?:[.|-][0-9a-zA-Z]+)*)$"); + return std::regex_match(version_string, re); + } + + bool is_date_without_tags(const std::string& version_string) + { + std::regex re("^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})$"); + return std::regex_match(version_string, re); + } + + bool is_semver(const std::string& version_string) + { + // This is the "official" SemVer regex, taken from: + // https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + std::regex re("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*" + ")(?:\\.(?:0|[" + "1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + return std::regex_match(version_string, re); + } + + bool is_semver_relaxed(const std::string& version_string) + { + std::regex re("^(?:[0-9a-zA-Z]+)\\.(?:[0-9a-zA-Z]+)\\.(?:[0-9a-zA-Z]+)(?:[\\.|-|\\+][0-9a-zA-Z]+)*$"); + return std::regex_match(version_string, re); + } + + const Versions::Scheme guess_version_scheme(const std::string& version_string) + { + if (is_date(version_string)) { - ret.emplace_back(PortControlVersion{commit_date_pair.first, version, commit_date_pair.second}); - last_version = version; + return Versions::Scheme::Date; } + + if (is_semver(version_string) || is_semver_relaxed(version_string)) + { + return Versions::Scheme::Relaxed; + } + + return Versions::Scheme::String; + } + + std::pair clean_version_string(const std::string& version_string, + int port_version, + bool from_manifest) + { + // Manifest files and ports that use the `Port-Version` field are assumed to have a clean version string + // already. + if (from_manifest || port_version > 0) + { + return std::make_pair(version_string, port_version); + } + + std::string clean_version = version_string; + int clean_port_version = 0; + + const auto index = version_string.find_last_of('-'); + if (index != std::string::npos) + { + // Very lazy check to keep date versions untouched + if (!is_date_without_tags(version_string)) + { + auto maybe_port_version = version_string.substr(index + 1); + clean_version.resize(index); + + try + { + clean_port_version = std::stoi(maybe_port_version); + } + catch (std::exception&) + { + // If not convertible to int consider last fragment as part of version string + clean_version = version_string; + } + } + } + + return std::make_pair(clean_version, clean_port_version); + } + + vcpkg::Optional get_version_from_text(const std::string& text, + const std::string& git_tree, + const std::string& commit_id, + const std::string& commit_date, + const std::string& port_name, + bool is_manifest) + { + auto res = Paragraphs::try_load_port_text(text, Strings::concat(commit_id, ":", port_name), is_manifest); + if (const auto& maybe_scf = res.get()) + { + if (const auto& scf = maybe_scf->get()) + { + // TODO: Get clean version name and port version + const auto version_string = scf->core_paragraph->version; + const auto clean_version = + clean_version_string(version_string, scf->core_paragraph->port_version, is_manifest); + + // SCF to HistoryVersion + return HistoryVersion{ + port_name, + git_tree, + commit_id, + commit_date, + Strings::concat(clean_version.first, "#", std::to_string(clean_version.second)), + clean_version.first, + clean_version.second, + guess_version_scheme(clean_version.first)}; + } + } + + return nullopt; + } + + vcpkg::Optional get_version_from_commit(const VcpkgPaths& paths, + const std::string& commit_id, + const std::string& commit_date, + const std::string& port_name) + { + const std::string rev_parse_cmd = Strings::format("rev-parse %s:ports/%s", commit_id, port_name); + auto rev_parse_output = run_git_command(paths, rev_parse_cmd); + if (rev_parse_output.exit_code == 0) + { + // Remove newline character + const auto git_tree = Strings::trim(std::move(rev_parse_output.output)); + + // Do we have a manifest file? + const std::string manifest_cmd = Strings::format(R"(show %s:vcpkg.json)", git_tree, port_name); + auto manifest_output = run_git_command(paths, manifest_cmd); + if (manifest_output.exit_code == 0) + { + return get_version_from_text( + manifest_output.output, git_tree, commit_id, commit_date, port_name, true); + } + + const std::string cmd = Strings::format(R"(show %s:CONTROL)", git_tree, commit_id, port_name); + auto control_output = run_git_command(paths, cmd); + + if (control_output.exit_code == 0) + { + return get_version_from_text( + control_output.output, git_tree, commit_id, commit_date, port_name, false); + } + } + + return nullopt; + } + + std::vector read_versions_from_log(const VcpkgPaths& paths, const std::string& port_name) + { + // log --format="%H %cd" --date=short --left-only -- ports/{port_name}/. + System::CmdLineBuilder builder; + builder.string_arg("log"); + builder.string_arg("--format=%H %cd"); + builder.string_arg("--date=short"); + builder.string_arg("--left-only"); + builder.string_arg("--"); // Begin pathspec + builder.string_arg(Strings::format("ports/%s/.", port_name)); + const auto output = run_git_command(paths, builder.extract()); + + auto commits = Util::fmap( + Strings::split(output.output, '\n'), [](const std::string& line) -> auto { + auto parts = Strings::split(line, ' '); + return std::make_pair(parts[0], parts[1]); + }); + + std::vector ret; + std::string last_version; + for (auto&& commit_date_pair : commits) + { + auto maybe_version = + get_version_from_commit(paths, commit_date_pair.first, commit_date_pair.second, port_name); + if (maybe_version.has_value()) + { + const auto version = maybe_version.value_or_exit(VCPKG_LINE_INFO); + + // Keep latest port with the current version string + if (last_version != version.version_string) + { + last_version = version.version_string; + ret.emplace_back(version); + } + } + // NOTE: Uncomment this code if you're looking for edge cases to patch in the generation. + // Otherwise, x-history simply skips "bad" versions, which is OK behavior. + // else + //{ + // Checks::exit_with_message(VCPKG_LINE_INFO, "Failed to get version from %s:%s", + // commit_date_pair.first, port_name); + //} + } + return ret; } - return ret; } const CommandStructure COMMAND_STRUCTURE = { @@ -84,13 +264,47 @@ namespace vcpkg::Commands::PortHistory void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) { - (void)args.parse_arguments(COMMAND_STRUCTURE); + const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); + std::string port_name = args.command_arguments.at(0); - std::vector versions = read_versions_from_log(paths, port_name); - System::print2(" version date vcpkg commit\n"); - for (auto&& version : versions) + std::vector versions = read_versions_from_log(paths, port_name); + + if (args.output_json()) { - System::printf("%20.20s %s %s\n", version.version, version.date, version.commit_id); + Json::Array versions_json; + for (auto&& version : versions) + { + Json::Object object; + object.insert("git-tree", Json::Value::string(version.git_tree)); + switch (version.scheme) + { + case Versions::Scheme::Semver: // falls through + case Versions::Scheme::Relaxed: + object.insert("version", Json::Value::string(version.version)); + break; + case Versions::Scheme::Date: + object.insert("version-date", Json::Value::string(version.version)); + break; + case Versions::Scheme::String: // falls through + default: object.insert("version-string", Json::Value::string(version.version)); break; + } + object.insert("port-version", Json::Value::integer(version.port_version)); + versions_json.push_back(std::move(object)); + } + + Json::Object root; + root.insert("versions", versions_json); + + auto json_string = Json::stringify(root, vcpkg::Json::JsonStyle::with_spaces(2)); + System::printf("%s\n", json_string); + } + else + { + System::print2(" version date vcpkg commit\n"); + for (auto&& version : versions) + { + System::printf("%20.20s %s %s\n", version.version_string, version.commit_date, version.commit_id); + } } Checks::exit_success(VCPKG_LINE_INFO); } diff --git a/toolsrc/src/vcpkg/paragraphs.cpp b/toolsrc/src/vcpkg/paragraphs.cpp index 90073317cf..ba28acccfd 100644 --- a/toolsrc/src/vcpkg/paragraphs.cpp +++ b/toolsrc/src/vcpkg/paragraphs.cpp @@ -241,6 +241,11 @@ namespace vcpkg::Paragraphs return contents.error().message(); } + ExpectedS> get_paragraphs_text(const std::string& text, const std::string& origin) + { + return parse_paragraphs(text, origin); + } + ExpectedS> get_paragraphs(const Files::Filesystem& fs, const fs::path& control_path) { const Expected contents = fs.read_contents(control_path); @@ -262,36 +267,73 @@ namespace vcpkg::Paragraphs return fs.exists(path / fs::u8path("CONTROL")) || fs.exists(path / fs::u8path("vcpkg.json")); } - static ParseExpected try_load_manifest(const Files::Filesystem& fs, - const std::string& port_name, - const fs::path& path_to_manifest, - std::error_code& ec) + static ParseExpected try_load_manifest_object( + const std::string& origin, + const ExpectedT, std::unique_ptr>& + res) { auto error_info = std::make_unique(); - auto res = Json::parse_file(fs, path_to_manifest, ec); - if (ec) return error_info; - if (auto val = res.get()) { if (val->first.is_object()) { - return SourceControlFile::parse_manifest_file(path_to_manifest, val->first.object()); + return SourceControlFile::parse_manifest_object(origin, val->first.object()); } else { - error_info->name = port_name; + error_info->name = origin; error_info->error = "Manifest files must have a top-level object"; return error_info; } } else { - error_info->name = port_name; + error_info->name = origin; error_info->error = res.error()->format(); return error_info; } } + static ParseExpected try_load_manifest_text(const std::string& text, const std::string& origin) + { + auto res = Json::parse(text); + return try_load_manifest_object(origin, res); + } + + static ParseExpected try_load_manifest(const Files::Filesystem& fs, + const std::string& port_name, + const fs::path& path_to_manifest, + std::error_code& ec) + { + (void)port_name; + + auto error_info = std::make_unique(); + auto res = Json::parse_file(fs, path_to_manifest, ec); + if (ec) return error_info; + + return try_load_manifest_object(fs::u8string(path_to_manifest), res); + } + + ParseExpected try_load_port_text(const std::string& text, + const std::string& origin, + bool is_manifest) + { + if (is_manifest) + { + return try_load_manifest_text(text, origin); + } + + ExpectedS> pghs = get_paragraphs_text(text, origin); + if (auto vector_pghs = pghs.get()) + { + return SourceControlFile::parse_control_file(origin, std::move(*vector_pghs)); + } + auto error_info = std::make_unique(); + error_info->name = fs::u8string(origin); + error_info->error = pghs.error(); + return error_info; + } + ParseExpected try_load_port(const Files::Filesystem& fs, const fs::path& path) { const auto path_to_manifest = path / fs::u8path("vcpkg.json"); @@ -318,7 +360,7 @@ namespace vcpkg::Paragraphs ExpectedS> pghs = get_paragraphs(fs, path_to_control); if (auto vector_pghs = pghs.get()) { - return SourceControlFile::parse_control_file(path_to_control, std::move(*vector_pghs)); + return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs)); } auto error_info = std::make_unique(); error_info->name = fs::u8string(path.filename()); diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 802e6d6a39..74858a7386 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -221,7 +221,7 @@ namespace vcpkg fpgh.extra_info.sort_keys(); } - void operator()(SourceControlFile& scf) const + [[nodiscard]] std::unique_ptr operator()(SourceControlFile& scf) const { (*this)(*scf.core_paragraph); std::for_each(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), *this); @@ -231,20 +231,21 @@ namespace vcpkg std::adjacent_find(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), FeatureEqual{}); if (adjacent_equal != scf.feature_paragraphs.end()) { - Checks::exit_with_message(VCPKG_LINE_INFO, - R"(Multiple features with the same name for port %s: %s + auto error_info = std::make_unique(); + error_info->name = scf.core_paragraph->name; + error_info->error = Strings::format(R"(Multiple features with the same name for port %s: %s This is invalid; please make certain that features have distinct names.)", - scf.core_paragraph->name, - (*adjacent_equal)->name); + scf.core_paragraph->name, + (*adjacent_equal)->name); + return error_info; } + return nullptr; } } canonicalize{}; } - static ParseExpected parse_source_paragraph(const fs::path& path_to_control, Paragraph&& fields) + static ParseExpected parse_source_paragraph(const std::string& origin, Paragraph&& fields) { - auto origin = fs::u8string(path_to_control); - ParagraphParser parser(std::move(fields)); auto spgh = std::make_unique(); @@ -276,10 +277,35 @@ namespace vcpkg TextRowCol textrowcol; std::string buf; parser.optional_field(SourceParagraphFields::BUILD_DEPENDS, {buf, textrowcol}); - spgh->dependencies = parse_dependencies_list(buf, origin, textrowcol).value_or_exit(VCPKG_LINE_INFO); + + auto maybe_dependencies = parse_dependencies_list(buf, origin, textrowcol); + if (maybe_dependencies.has_value()) + { + spgh->dependencies = maybe_dependencies.value_or_exit(VCPKG_LINE_INFO); + } + else + { + auto error_info = std::make_unique(); + error_info->name = origin; + error_info->error = maybe_dependencies.error(); + return error_info; + } + buf.clear(); parser.optional_field(SourceParagraphFields::DEFAULT_FEATURES, {buf, textrowcol}); - spgh->default_features = parse_default_features_list(buf, origin, textrowcol).value_or_exit(VCPKG_LINE_INFO); + + auto maybe_default_features = parse_default_features_list(buf, origin, textrowcol); + if (maybe_default_features.has_value()) + { + spgh->default_features = maybe_default_features.value_or_exit(VCPKG_LINE_INFO); + } + else + { + auto error_info = std::make_unique(); + error_info->name = origin; + error_info->error = maybe_default_features.error(); + return error_info; + } auto supports_expr = parser.optional_field(SourceParagraphFields::SUPPORTS); if (!supports_expr.empty()) @@ -304,9 +330,8 @@ namespace vcpkg return spgh; } - static ParseExpected parse_feature_paragraph(const fs::path& path_to_control, Paragraph&& fields) + static ParseExpected parse_feature_paragraph(const std::string& origin, Paragraph&& fields) { - auto origin = fs::u8string(path_to_control); ParagraphParser parser(std::move(fields)); auto fpgh = std::make_unique(); @@ -315,9 +340,19 @@ namespace vcpkg fpgh->description = Strings::split(parser.required_field(SourceParagraphFields::DESCRIPTION), '\n'); trim_all(fpgh->description); - fpgh->dependencies = - parse_dependencies_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS), origin) - .value_or_exit(VCPKG_LINE_INFO); + auto maybe_dependencies = + parse_dependencies_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS), origin); + if (maybe_dependencies.has_value()) + { + fpgh->dependencies = maybe_dependencies.value_or_exit(VCPKG_LINE_INFO); + } + else + { + auto error_info = std::make_unique(); + error_info->name = origin; + error_info->error = maybe_dependencies.error(); + return error_info; + } auto err = parser.error_info(fpgh->name.empty() ? origin : fpgh->name); if (err) @@ -327,18 +362,18 @@ namespace vcpkg } ParseExpected SourceControlFile::parse_control_file( - const fs::path& path_to_control, std::vector&& control_paragraphs) + const std::string& origin, std::vector&& control_paragraphs) { if (control_paragraphs.size() == 0) { auto ret = std::make_unique(); - ret->name = fs::u8string(path_to_control); + ret->name = origin; return ret; } auto control_file = std::make_unique(); - auto maybe_source = parse_source_paragraph(path_to_control, std::move(control_paragraphs.front())); + auto maybe_source = parse_source_paragraph(origin, std::move(control_paragraphs.front())); if (const auto source = maybe_source.get()) control_file->core_paragraph = std::move(*source); else @@ -348,14 +383,17 @@ namespace vcpkg for (auto&& feature_pgh : control_paragraphs) { - auto maybe_feature = parse_feature_paragraph(path_to_control, std::move(feature_pgh)); + auto maybe_feature = parse_feature_paragraph(origin, std::move(feature_pgh)); if (const auto feature = maybe_feature.get()) control_file->feature_paragraphs.emplace_back(std::move(*feature)); else return std::move(maybe_feature).error(); } - canonicalize(*control_file); + if (auto maybe_error = canonicalize(*control_file)) + { + return std::move(maybe_error); + } return control_file; } @@ -854,7 +892,10 @@ namespace vcpkg r.optional_object_field( obj, FEATURES, control_file->feature_paragraphs, FeaturesFieldDeserializer::instance); - canonicalize(*control_file); + if (auto maybe_error = canonicalize(*control_file)) + { + Checks::exit_with_message(VCPKG_LINE_INFO, maybe_error->error); + } return std::move(control_file); } @@ -879,8 +920,8 @@ namespace vcpkg constexpr StringLiteral ManifestDeserializer::DEFAULT_FEATURES; constexpr StringLiteral ManifestDeserializer::SUPPORTS; - Parse::ParseExpected SourceControlFile::parse_manifest_file(const fs::path& path_to_manifest, - const Json::Object& manifest) + Parse::ParseExpected SourceControlFile::parse_manifest_object(const std::string& origin, + const Json::Object& manifest) { Json::Reader reader; @@ -889,7 +930,7 @@ namespace vcpkg if (!reader.errors().empty()) { auto err = std::make_unique(); - err->name = fs::u8string(path_to_manifest); + err->name = origin; err->other_errors = std::move(reader.errors()); return std::move(err); } @@ -903,6 +944,12 @@ namespace vcpkg } } + Parse::ParseExpected SourceControlFile::parse_manifest_file(const fs::path& path_to_manifest, + const Json::Object& manifest) + { + return parse_manifest_object(fs::u8string(path_to_manifest), manifest); + } + void print_error_message(Span> error_info_list) { Checks::check_exit(VCPKG_LINE_INFO, error_info_list.size() > 0); diff --git a/toolsrc/windows-bootstrap/vcpkg.vcxproj b/toolsrc/windows-bootstrap/vcpkg.vcxproj index 259d1a3302..5f897bc673 100644 --- a/toolsrc/windows-bootstrap/vcpkg.vcxproj +++ b/toolsrc/windows-bootstrap/vcpkg.vcxproj @@ -267,6 +267,7 @@ +