[vcpkg] Implement VersionedPortfileProvider and BaselineProvider (#14123)

* WIP: Get versions from database files

* Fix formatting

* Provider inherits ResourceBase

* Correct versions JSON file location

* Fix formatting

* Fix formatting

* Fix include in versions.h

* Fetch port versions using git tree object

* Undo changes to x-history

* Remove unnecesary moves

Co-authored-by: nicole mazzuca <mazzucan@outlook.com>

* Extract Git manipulation code

* [WIP] Review comments

* [WIP] Review comments pt. 2

* [WIP] Review comments / fix formatting

* Generate baseline.json

* Extract deserializers from registries source file

* BaselineProvider initial implementation

* Modify gitignore

* Update .gitignore again

* Use JSON deserializer for versions db

* Lazy load baseline file

* Fetch baseline.json from baseline commit

* More git abstractions

* Clean up code

* Path helpers

* Formatting

* Move data into impl object

* Use implementation object for VersionedPortfileProvider

* Reuse cloned instance for checkouts

* Code cleanup and formatting

* Fix returning dangling reference

* Prepare to remove files in port_versions/

* Remove files in port_versions/

* Update .gitignore

* Some PR review comments

* Use StringView

* More StringView conversions

* More refactoring

* Make some implementation members private

* Functions for parsing baseline and version files

* Hide deserializers implementation

* Check for `versions` feature flag in registries.

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: nicole mazzuca <mazzucan@outlook.com>

.gitignore

@@ -295,6 +295,8 @@ __pycache__/
 /toolsrc/windows-bootstrap/msbuild.x86.release/
 /toolsrc/windows-bootstrap/msbuild.x64.debug/
 /toolsrc/windows-bootstrap/msbuild.x64.release/
+#ignore db
+/port_versions/
 #ignore custom triplets
 /triplets/*
 #add vcpkg-designed triplets back in

scripts/generatePortVersionsDb.py

@@ -4,6 +4,7 @@ import sys
 import subprocess
 import json
 import time
+import shutil
 
 from subprocess import CalledProcessError
 from json.decoder import JSONDecodeError
@@ -14,7 +15,7 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
 def get_current_git_ref():
-    output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], 
+    output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'],
                             capture_output=True,
                             encoding='utf-8')
     if output.returncode == 0:
@@ -25,33 +26,65 @@ def get_current_git_ref():
 
 def generate_port_versions_db(ports_path, db_path, revision):
     start_time = time.time()
-    port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))]
+
+    # Assume each directory in ${VCPKG_ROOT}/ports is a different port
+    port_names = [item for item in os.listdir(
+        ports_path) if os.path.isdir(os.path.join(ports_path, item))]
+    port_names.sort()
     total_count = len(port_names)
+
+    # Dictionary to collect the latest version of each port as baseline
+    baseline_objects = {}
+    baseline_objects['default'] = {}
+
     for counter, port_name in enumerate(port_names):
         containing_dir = os.path.join(db_path, f'{port_name[0]}-')
         os.makedirs(containing_dir, exist_ok=True)
+
         output_filepath = os.path.join(containing_dir, f'{port_name}.json')
         if not os.path.exists(output_filepath):
             output = subprocess.run(
-                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], 
+                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'),
+                 'x-history', port_name, '--x-json'],
                 capture_output=True, encoding='utf-8')
+
             if output.returncode == 0:
                 try:
                     versions_object = json.loads(output.stdout)
+
+                    # Put latest version in baseline dictionary
+                    latest_version = versions_object["versions"][0]
+                    baseline_objects['default'][port_name] = {
+                        "version-string": latest_version["version-string"],
+                        "port-version": latest_version["port-version"]
+                    }
                     with open(output_filepath, 'w') as output_file:
                         json.dump(versions_object, output_file)
                 except JSONDecodeError:
-                    print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
+                    print(
+                        f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
             else:
-                print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr)
+                print(f'x-history {port_name} failed: ',
+                      output.stdout.strip(), file=sys.stderr)
+
         # This should be replaced by a progress bar
         if counter > 0 and counter % 100 == 0:
             elapsed_time = time.time() - start_time
-            print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
+            print(
+                f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
+
+    # Generate baseline.json
+    baseline_file_path = os.path.join(db_path, 'baseline.json')
+    with open(baseline_file_path, 'w') as baseline_output_file:
+        json.dump(baseline_objects, baseline_output_file)
+
+    # Generate timestamp
     rev_file = os.path.join(db_path, revision)
     Path(rev_file).touch()
+
     elapsed_time = time.time() - start_time
-    print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
+    print(
+        f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
 
 
 def main(ports_path, db_path):
@@ -59,11 +92,21 @@ def main(ports_path, db_path):
     if not revision:
         print('Couldn\'t fetch current Git revision', file=sys.stderr)
         sys.exit(1)
+
     rev_file = os.path.join(db_path, revision)
     if os.path.exists(rev_file):
         print(f'Database files already exist for commit {revision}')
         sys.exit(0)
-    generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision)
+
+    if (os.path.exists(db_path)):
+        try:
+            shutil.rmtree(db_path)
+        except OSError as e:
+            print(f'Could not delete folder: {db_path}.\nError: {e.strerror}')
+
+    generate_port_versions_db(ports_path=ports_path,
+                              db_path=db_path,
+                              revision=revision)
 
 
 if __name__ == "__main__":

toolsrc/include/vcpkg/base/jsonreader.h

@@ -23,7 +23,7 @@ namespace vcpkg::Json
         Optional<Type> visit(Reader&, const Value&);
         Optional<Type> visit(Reader&, const Object&);
 
-    protected:
+    public:
         virtual Optional<Type> visit_null(Reader&);
         virtual Optional<Type> visit_boolean(Reader&, bool);
         virtual Optional<Type> visit_integer(Reader& r, int64_t i);
@@ -33,6 +33,7 @@ namespace vcpkg::Json
         virtual Optional<Type> visit_object(Reader&, const Object&);
         virtual View<StringView> valid_fields() const;
 
+    protected:
         IDeserializer() = default;
         IDeserializer(const IDeserializer&) = default;
         IDeserializer& operator=(const IDeserializer&) = default;

toolsrc/include/vcpkg/portfileprovider.h

@@ -6,6 +6,7 @@
 #include <vcpkg/base/util.h>
 
 #include <vcpkg/sourceparagraph.h>
+#include <vcpkg/versions.h>
 
 namespace vcpkg::PortFileProvider
 {
@@ -36,4 +37,48 @@ namespace vcpkg::PortFileProvider
         std::vector<fs::path> overlay_ports;
         mutable std::unordered_map<std::string, SourceControlFileLocation> cache;
     };
+
+    struct IVersionedPortfileProvider
+    {
+        virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const = 0;
+
+        virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
+            const vcpkg::Versions::VersionSpec& version_spec) const = 0;
+    };
+
+    struct IBaselineProvider
+    {
+        virtual Optional<VersionT> get_baseline_version(StringView port_name) const = 0;
+    };
+
+    namespace details
+    {
+        struct BaselineProviderImpl;
+        struct VersionedPortfileProviderImpl;
+    }
+
+    struct VersionedPortfileProvider : IVersionedPortfileProvider, Util::ResourceBase
+    {
+        explicit VersionedPortfileProvider(const vcpkg::VcpkgPaths& paths);
+        ~VersionedPortfileProvider();
+
+        const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const override;
+
+        ExpectedS<const SourceControlFileLocation&> get_control_file(
+            const vcpkg::Versions::VersionSpec& version_spec) const override;
+
+    private:
+        std::unique_ptr<details::VersionedPortfileProviderImpl> m_impl;
+    };
+
+    struct BaselineProvider : IBaselineProvider, Util::ResourceBase
+    {
+        explicit BaselineProvider(const vcpkg::VcpkgPaths& paths, const std::string& baseline);
+        ~BaselineProvider();
+
+        Optional<VersionT> get_baseline_version(StringView port_name) const override;
+
+    private:
+        std::unique_ptr<details::BaselineProviderImpl> m_impl;
+    };
 }

toolsrc/include/vcpkg/vcpkgpaths.h

@@ -102,11 +102,23 @@ namespace vcpkg
         fs::path vcpkg_dir_info;
         fs::path vcpkg_dir_updates;
 
+        fs::path baselines_dot_git_dir;
+        fs::path baselines_work_tree;
+        fs::path baselines_output;
+
+        fs::path versions_dot_git_dir;
+        fs::path versions_work_tree;
+        fs::path versions_output;
+
         fs::path ports_cmake;
 
         const fs::path& get_tool_exe(const std::string& tool) const;
         const std::string& get_tool_version(const std::string& tool) const;
 
+        // Git manipulation
+        fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const;
+        fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const;
+
         Optional<const Json::Object&> get_manifest() const;
         Optional<const fs::path&> get_manifest_path() const;
         const Configuration& get_configuration() const;
@@ -133,5 +145,20 @@ namespace vcpkg
 
     private:
         std::unique_ptr<details::VcpkgPathsImpl> m_pimpl;
+
+        static void git_checkout_subpath(const VcpkgPaths& paths,
+                                         StringView commit_sha,
+                                         const fs::path& subpath,
+                                         const fs::path& local_repo,
+                                         const fs::path& destination,
+                                         const fs::path& dot_git_dir,
+                                         const fs::path& work_tree);
+
+        static void git_checkout_object(const VcpkgPaths& paths,
+                                        StringView git_object,
+                                        const fs::path& local_repo,
+                                        const fs::path& destination,
+                                        const fs::path& dot_git_dir,
+                                        const fs::path& work_tree);
     };
 }

toolsrc/include/vcpkg/versiondeserializers.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <vcpkg/base/fwd/stringview.h>
+
+#include <vcpkg/base/jsonreader.h>
+#include <vcpkg/base/stringliteral.h>
+
+#include <vcpkg/versions.h>
+#include <vcpkg/versiont.h>
+
+namespace vcpkg
+{
+    struct VersionDbEntry
+    {
+        VersionT version;
+        Versions::Scheme scheme;
+        std::string git_tree;
+
+        VersionDbEntry(const std::string& version_string,
+                       int port_version,
+                       Versions::Scheme scheme,
+                       const std::string& git_tree)
+            : version(VersionT(version_string, port_version)), scheme(scheme), git_tree(git_tree)
+        {
+        }
+    };
+
+    Json::IDeserializer<VersionT>& get_versiont_deserializer_instance();
+
+    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
+                                                                                StringView baseline_name,
+                                                                                const fs::path& baseline_file_path);
+
+    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
+                                                               StringView port_name,
+                                                               const fs::path& versions_file_path);
+}

toolsrc/include/vcpkg/versions.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/versiont.h>
+
 namespace vcpkg::Versions
 {
     enum class Scheme
@@ -10,6 +12,25 @@ namespace vcpkg::Versions
         String
     };
 
+    struct VersionSpec
+    {
+        std::string port_name;
+        VersionT version;
+        Scheme scheme;
+
+        VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme);
+
+        VersionSpec(const std::string& port_name, const std::string& version_string, int port_version, Scheme scheme);
+
+        friend bool operator==(const VersionSpec& lhs, const VersionSpec& rhs);
+        friend bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs);
+    };
+
+    struct VersionSpecHasher
+    {
+        std::size_t operator()(const VersionSpec& key) const;
+    };
+
     struct Constraint
     {
         enum class Type

toolsrc/src/vcpkg/portfileprovider.cpp

@@ -1,3 +1,6 @@
+
+
+#include <vcpkg/base/json.h>
 #include <vcpkg/base/system.debug.h>
 
 #include <vcpkg/configuration.h>
@@ -7,6 +10,33 @@
 #include <vcpkg/sourceparagraph.h>
 #include <vcpkg/vcpkgcmdarguments.h>
 #include <vcpkg/vcpkgpaths.h>
+#include <vcpkg/versiondeserializers.h>
+
+#include <regex>
+
+using namespace vcpkg;
+using namespace Versions;
+
+namespace
+{
+    Optional<fs::path> get_versions_json_path(const VcpkgPaths& paths, StringView port_name)
+    {
+        const auto port_versions_dir_path = paths.root / fs::u8path("port_versions");
+        const auto subpath = Strings::concat(port_name.substr(0, 1), "-/", port_name, ".json");
+        const auto json_path = port_versions_dir_path / subpath;
+        if (paths.get_filesystem().exists(json_path))
+        {
+            return json_path;
+        }
+        return nullopt;
+    }
+
+    Optional<fs::path> get_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha)
+    {
+        const auto baseline_json = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha);
+        return paths.get_filesystem().exists(baseline_json) ? make_optional(baseline_json) : nullopt;
+    }
+}
 
 namespace vcpkg::PortFileProvider
 {
@@ -27,7 +57,7 @@ namespace vcpkg::PortFileProvider
         return Util::fmap(ports, [](auto&& kvpair) -> const SourceControlFileLocation* { return &kvpair.second; });
     }
 
-    PathsPortFileProvider::PathsPortFileProvider(const vcpkg::VcpkgPaths& paths_,
+    PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths_,
                                                  const std::vector<std::string>& overlay_ports_)
         : paths(paths_)
     {
@@ -80,7 +110,7 @@ namespace vcpkg::PortFileProvider
                 }
                 else
                 {
-                    vcpkg::print_error_message(maybe_scf.error());
+                    print_error_message(maybe_scf.error());
                     Checks::exit_with_message(
                         VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
                 }
@@ -106,7 +136,7 @@ namespace vcpkg::PortFileProvider
                 }
                 else
                 {
-                    vcpkg::print_error_message(found_scf.error());
+                    print_error_message(found_scf.error());
                     Checks::exit_with_message(
                         VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
                 }
@@ -148,7 +178,7 @@ namespace vcpkg::PortFileProvider
                 }
                 else
                 {
-                    vcpkg::print_error_message(found_scf.error());
+                    print_error_message(found_scf.error());
                     Checks::exit_with_message(
                         VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(port_directory));
                 }
@@ -224,7 +254,7 @@ namespace vcpkg::PortFileProvider
                 }
                 else
                 {
-                    vcpkg::print_error_message(maybe_scf.error());
+                    print_error_message(maybe_scf.error());
                     Checks::exit_with_message(
                         VCPKG_LINE_INFO, "Error: Failed to load port from %s", fs::u8string(ports_dir));
                 }
@@ -257,4 +287,154 @@ namespace vcpkg::PortFileProvider
 
         return ret;
     }
+
+    namespace details
+    {
+        struct BaselineProviderImpl
+        {
+            BaselineProviderImpl(const VcpkgPaths& paths, const std::string& baseline)
+                : paths(paths), baseline(baseline)
+            {
+            }
+            ~BaselineProviderImpl() { }
+
+            const std::map<std::string, VersionT, std::less<>>& get_baseline_cache() const
+            {
+                return baseline_cache.get_lazy([&]() -> auto {
+                    auto maybe_baseline_file = get_baseline_json_path(paths, baseline);
+                    Checks::check_exit(VCPKG_LINE_INFO, maybe_baseline_file.has_value(), "Couldn't find baseline.json");
+                    auto baseline_file = maybe_baseline_file.value_or_exit(VCPKG_LINE_INFO);
+
+                    auto maybe_baselines_map = parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
+                    Checks::check_exit(VCPKG_LINE_INFO,
+                                       maybe_baselines_map.has_value(),
+                                       "Error: Couldn't parse baseline `%s` from `%s`",
+                                       "default",
+                                       fs::u8string(baseline_file));
+                    auto baselines_map = *maybe_baselines_map.get();
+                    return std::move(baselines_map);
+                });
+            }
+
+        private:
+            const VcpkgPaths& paths;
+            const std::string baseline;
+            Lazy<std::map<std::string, VersionT, std::less<>>> baseline_cache;
+        };
+
+        struct VersionedPortfileProviderImpl
+        {
+            std::map<std::string, std::vector<VersionSpec>> versions_cache;
+            std::unordered_map<VersionSpec, std::string, VersionSpecHasher> git_tree_cache;
+            std::unordered_map<VersionSpec, SourceControlFileLocation, VersionSpecHasher> control_cache;
+
+            VersionedPortfileProviderImpl(const VcpkgPaths& paths) : paths(paths) { }
+            ~VersionedPortfileProviderImpl() { }
+
+            const VcpkgPaths& get_paths() const { return paths; }
+            Files::Filesystem& get_filesystem() const { return paths.get_filesystem(); }
+
+        private:
+            const VcpkgPaths& paths;
+        };
+    }
+
+    VersionedPortfileProvider::VersionedPortfileProvider(const VcpkgPaths& paths)
+        : m_impl(std::make_unique<details::VersionedPortfileProviderImpl>(paths))
+    {
+    }
+    VersionedPortfileProvider::~VersionedPortfileProvider() { }
+
+    const std::vector<VersionSpec>& VersionedPortfileProvider::get_port_versions(StringView port_name) const
+    {
+        auto cache_it = m_impl->versions_cache.find(port_name.to_string());
+        if (cache_it != m_impl->versions_cache.end())
+        {
+            return cache_it->second;
+        }
+
+        auto maybe_versions_file_path = get_versions_json_path(m_impl->get_paths(), port_name);
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           maybe_versions_file_path.has_value(),
+                           "Error: Couldn't find a versions database file: %s.json.",
+                           port_name);
+        auto versions_file_path = maybe_versions_file_path.value_or_exit(VCPKG_LINE_INFO);
+
+        auto maybe_version_entries = parse_versions_file(m_impl->get_filesystem(), port_name, versions_file_path);
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           maybe_version_entries.has_value(),
+                           "Error: Couldn't parse versions from file: %s",
+                           fs::u8string(versions_file_path));
+        auto version_entries = maybe_version_entries.value_or_exit(VCPKG_LINE_INFO);
+
+        auto port = port_name.to_string();
+        for (auto&& version_entry : version_entries)
+        {
+            VersionSpec spec(port, version_entry.version, version_entry.scheme);
+            m_impl->versions_cache[port].push_back(spec);
+            m_impl->git_tree_cache.emplace(std::move(spec), std::move(version_entry.git_tree));
+        }
+        return m_impl->versions_cache.at(port);
+    }
+
+    ExpectedS<const SourceControlFileLocation&> VersionedPortfileProvider::get_control_file(
+        const VersionSpec& version_spec) const
+    {
+        auto cache_it = m_impl->control_cache.find(version_spec);
+        if (cache_it != m_impl->control_cache.end())
+        {
+            return cache_it->second;
+        }
+
+        // Pre-populate versions cache.
+        get_port_versions(version_spec.port_name);
+
+        auto git_tree_cache_it = m_impl->git_tree_cache.find(version_spec);
+        if (git_tree_cache_it == m_impl->git_tree_cache.end())
+        {
+            return Strings::concat("No git object SHA for entry %s at version %s.",
+                                   version_spec.port_name,
+                                   version_spec.version.to_string());
+        }
+
+        const std::string git_tree = git_tree_cache_it->second;
+        auto port_directory =
+            m_impl->get_paths().git_checkout_port(m_impl->get_filesystem(), version_spec.port_name, git_tree);
+
+        auto maybe_control_file = Paragraphs::try_load_port(m_impl->get_filesystem(), port_directory);
+        if (auto scf = maybe_control_file.get())
+        {
+            if (scf->get()->core_paragraph->name == version_spec.port_name)
+            {
+                return m_impl->control_cache
+                    .emplace(version_spec, SourceControlFileLocation{std::move(*scf), std::move(port_directory)})
+                    .first->second;
+            }
+            return Strings::format("Error: Failed to load port from %s: names did not match: '%s' != '%s'",
+                                   fs::u8string(port_directory),
+                                   version_spec.port_name,
+                                   scf->get()->core_paragraph->name);
+        }
+
+        print_error_message(maybe_control_file.error());
+        return Strings::format(
+            "Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory));
+    }
+
+    BaselineProvider::BaselineProvider(const VcpkgPaths& paths, const std::string& baseline)
+        : m_impl(std::make_unique<details::BaselineProviderImpl>(paths, baseline))
+    {
+    }
+    BaselineProvider::~BaselineProvider() { }
+
+    Optional<VersionT> BaselineProvider::get_baseline_version(StringView port_name) const
+    {
+        const auto& cache = m_impl->get_baseline_cache();
+        auto it = cache.find(port_name.to_string());
+        if (it != cache.end())
+        {
+            return it->second;
+        }
+        return nullopt;
+    }
 }

toolsrc/src/vcpkg/registries.cpp

@@ -5,7 +5,9 @@
 
 #include <vcpkg/configurationdeserializer.h>
 #include <vcpkg/registries.h>
+#include <vcpkg/vcpkgcmdarguments.h>
 #include <vcpkg/vcpkgpaths.h>
+#include <vcpkg/versiondeserializers.h>
 #include <vcpkg/versiont.h>
 
 #include <map>
@@ -70,32 +72,6 @@ namespace
         }
     };
 
-    struct VersionTDeserializer final : Json::IDeserializer<VersionT>
-    {
-        StringView type_name() const override { return "a version object"; }
-        View<StringView> valid_fields() const override
-        {
-            static const StringView t[] = {"version-string", "port-version"};
-            return t;
-        }
-
-        Optional<VersionT> visit_object(Json::Reader& r, const Json::Object& obj) override
-        {
-            std::string version;
-            int port_version = 0;
-
-            r.required_object_field(type_name(), obj, "version-string", version, version_deserializer);
-            r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance);
-
-            return VersionT{std::move(version), port_version};
-        }
-
-        static Json::StringDeserializer version_deserializer;
-        static VersionTDeserializer instance;
-    };
-    Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"};
-    VersionTDeserializer VersionTDeserializer::instance;
-
     struct FilesystemVersionEntryDeserializer final : Json::IDeserializer<std::pair<VersionT, fs::path>>
     {
         StringView type_name() const override { return "a version entry object"; }
@@ -109,7 +85,7 @@ namespace
         {
             fs::path registry_path;
 
-            auto version = VersionTDeserializer::instance.visit_object(r, obj);
+            auto version = get_versiont_deserializer_instance().visit_object(r, obj);
 
             r.required_object_field(
                 "version entry", obj, "registry-path", registry_path, Json::PathDeserializer::instance);
@@ -162,30 +138,6 @@ namespace
         const fs::path& registry_root;
     };
 
-    struct BaselineDeserializer final : Json::IDeserializer<std::map<std::string, VersionT, std::less<>>>
-    {
-        StringView type_name() const override { return "a baseline object"; }
-
-        Optional<type> visit_object(Json::Reader& r, const Json::Object& obj) override
-        {
-            std::map<std::string, VersionT, std::less<>> result;
-
-            for (auto pr : obj)
-            {
-                const auto& version_value = pr.second;
-                VersionT version;
-                r.visit_in_key(version_value, pr.first, version, VersionTDeserializer::instance);
-
-                result.emplace(pr.first.to_string(), std::move(version));
-            }
-
-            return std::move(result);
-        }
-
-        static BaselineDeserializer instance;
-    };
-    BaselineDeserializer BaselineDeserializer::instance;
-
     struct FilesystemRegistry final : RegistryImpl
     {
         std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths& paths, StringView port_name) const override
@@ -267,6 +219,12 @@ namespace
 
         Optional<VersionT> get_baseline_version(const VcpkgPaths& paths, StringView port_name) const override
         {
+            if (!paths.get_feature_flags().versions)
+            {
+                Checks::check_exit(VCPKG_LINE_INFO,
+                                   "This invocation failed because the `versions` feature flag is not enabled.");
+            }
+
             const auto& baseline_cache = baseline.get([this, &paths] { return load_baseline_versions(paths); });
             auto it = baseline_cache.find(port_name);
             if (it != baseline_cache.end())
@@ -310,26 +268,17 @@ namespace
                 Checks::exit_with_message(VCPKG_LINE_INFO, "Error: `baseline.json` does not have a top-level object.");
             }
 
-            const auto& obj = value.first.object();
-            auto baseline_value = obj.get("default");
-            if (!baseline_value)
+            auto maybe_baseline_versions = parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
+            if (auto baseline_versions = maybe_baseline_versions.get())
             {
-                Checks::exit_with_message(
-                    VCPKG_LINE_INFO, "Error: `baseline.json` does not contain the baseline \"%s\"", "default");
-            }
-
-            Json::Reader r;
-            std::map<std::string, VersionT, std::less<>> result;
-            r.visit_in_key(*baseline_value, "default", result, BaselineDeserializer::instance);
-
-            if (r.errors().empty())
-            {
-                return result;
+                return std::move(*baseline_versions);
             }
             else
             {
-                Checks::exit_with_message(
-                    VCPKG_LINE_INFO, "Error: failed to parse `baseline.json`:\n%s", Strings::join("\n", r.errors()));
+                Checks::exit_with_message(VCPKG_LINE_INFO,
+                                          "Error: failed to parse `%s`:\n%s",
+                                          fs::u8string(baseline_file),
+                                          maybe_baseline_versions.error());
             }
         }
 

toolsrc/src/vcpkg/vcpkgpaths.cpp

@@ -71,6 +71,15 @@ namespace
         return result;
     }
 
+    System::CmdLineBuilder git_cmd_builder(const VcpkgPaths& paths,
+                                           const fs::path& dot_git_dir,
+                                           const fs::path& work_tree)
+    {
+        return System::CmdLineBuilder()
+            .path_arg(paths.get_tool_exe(Tools::GIT))
+            .string_arg(Strings::concat("--git-dir=", fs::u8string(dot_git_dir)))
+            .string_arg(Strings::concat("--work-tree=", fs::u8string(work_tree)));
+    }
 } // unnamed namespace
 
 namespace vcpkg
@@ -353,6 +362,18 @@ If you wish to silence this error and use classic mode, you can:
         vcpkg_dir_info = vcpkg_dir / fs::u8path("info");
         vcpkg_dir_updates = vcpkg_dir / fs::u8path("updates");
 
+        // Versioning paths
+        const auto versioning_tmp = buildtrees / fs::u8path("versioning_tmp");
+        const auto versioning_output = buildtrees / fs::u8path("versioning");
+
+        baselines_dot_git_dir = versioning_tmp / fs::u8path(".baselines.git");
+        baselines_work_tree = versioning_tmp / fs::u8path("baselines-worktree");
+        baselines_output = versioning_output / fs::u8path("baselines");
+
+        versions_dot_git_dir = versioning_tmp / fs::u8path(".versions.git");
+        versions_work_tree = versioning_tmp / fs::u8path("versions-worktree");
+        versions_output = versioning_output / fs::u8path("versions");
+
         ports_cmake = filesystem.canonical(VCPKG_LINE_INFO, scripts / fs::u8path("ports.cmake"));
 
         for (auto&& overlay_triplets_dir : args.overlay_triplets)
@@ -456,6 +477,178 @@ If you wish to silence this error and use classic mode, you can:
         return m_pimpl->m_tool_cache->get_tool_version(*this, tool);
     }
 
+    void VcpkgPaths::git_checkout_subpath(const VcpkgPaths& paths,
+                                          StringView commit_sha,
+                                          const fs::path& subpath,
+                                          const fs::path& local_repo,
+                                          const fs::path& destination,
+                                          const fs::path& dot_git_dir,
+                                          const fs::path& work_tree)
+    {
+        Files::Filesystem& fs = paths.get_filesystem();
+        fs.remove_all(work_tree, VCPKG_LINE_INFO);
+        fs.remove_all(destination, VCPKG_LINE_INFO);
+        fs.remove_all(dot_git_dir, VCPKG_LINE_INFO);
+
+        // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp}
+        // git clone --no-checkout --local {vcpkg_root} {dot_git_dir}
+        System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
+                                                       .string_arg("clone")
+                                                       .string_arg("--no-checkout")
+                                                       .string_arg("--local")
+                                                       .path_arg(local_repo)
+                                                       .path_arg(dot_git_dir);
+        const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           clone_output.exit_code == 0,
+                           "Failed to clone temporary vcpkg instance.\n%s\n",
+                           clone_output.output);
+
+        // git checkout {commit-sha} -- {subpath}
+        System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
+                                                          .string_arg("checkout")
+                                                          .string_arg(commit_sha)
+                                                          .string_arg("--")
+                                                          .path_arg(subpath);
+        const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract());
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           checkout_output.exit_code == 0,
+                           "Error: Failed to checkout %s:%s\n%s\n",
+                           commit_sha,
+                           fs::u8string(subpath),
+                           checkout_output.output);
+
+        const fs::path checked_out_path = work_tree / subpath;
+        const auto& containing_folder = destination.parent_path();
+        if (!fs.exists(containing_folder))
+        {
+            fs.create_directories(containing_folder, VCPKG_LINE_INFO);
+        }
+
+        std::error_code ec;
+        fs.rename_or_copy(checked_out_path, destination, ".tmp", ec);
+        fs.remove_all(work_tree, VCPKG_LINE_INFO);
+        fs.remove_all(dot_git_dir, VCPKG_LINE_INFO);
+        if (ec)
+        {
+            System::printf(System::Color::error,
+                           "Error: Couldn't move checked out files from %s to destination %s",
+                           fs::u8string(checked_out_path),
+                           fs::u8string(destination));
+            Checks::exit_fail(VCPKG_LINE_INFO);
+        }
+    }
+
+    void VcpkgPaths::git_checkout_object(const VcpkgPaths& paths,
+                                         StringView git_object,
+                                         const fs::path& local_repo,
+                                         const fs::path& destination,
+                                         const fs::path& dot_git_dir,
+                                         const fs::path& work_tree)
+    {
+        Files::Filesystem& fs = paths.get_filesystem();
+        fs.remove_all(work_tree, VCPKG_LINE_INFO);
+        fs.remove_all(destination, VCPKG_LINE_INFO);
+
+        if (!fs.exists(dot_git_dir))
+        {
+            // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp}
+            // git clone --no-checkout --local {vcpkg_root} {dot_git_dir}
+            System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
+                                                           .string_arg("clone")
+                                                           .string_arg("--no-checkout")
+                                                           .string_arg("--local")
+                                                           .path_arg(local_repo)
+                                                           .path_arg(dot_git_dir);
+            const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
+            Checks::check_exit(VCPKG_LINE_INFO,
+                               clone_output.exit_code == 0,
+                               "Failed to clone temporary vcpkg instance.\n%s\n",
+                               clone_output.output);
+        }
+        else
+        {
+            System::CmdLineBuilder fetch_cmd_builder =
+                git_cmd_builder(paths, dot_git_dir, work_tree).string_arg("fetch");
+            const auto fetch_output = System::cmd_execute_and_capture_output(fetch_cmd_builder.extract());
+            Checks::check_exit(VCPKG_LINE_INFO,
+                               fetch_output.exit_code == 0,
+                               "Failed to update refs on temporary vcpkg repository.\n%s\n",
+                               fetch_output.output);
+        }
+
+        if (!fs.exists(work_tree))
+        {
+            fs.create_directories(work_tree, VCPKG_LINE_INFO);
+        }
+
+        // git checkout {tree_object} .
+        System::CmdLineBuilder checkout_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
+                                                          .string_arg("checkout")
+                                                          .string_arg(git_object)
+                                                          .string_arg(".");
+        const auto checkout_output = System::cmd_execute_and_capture_output(checkout_cmd_builder.extract());
+        Checks::check_exit(VCPKG_LINE_INFO, checkout_output.exit_code == 0, "Failed to checkout %s", git_object);
+
+        const auto& containing_folder = destination.parent_path();
+        if (!fs.exists(containing_folder))
+        {
+            fs.create_directories(containing_folder, VCPKG_LINE_INFO);
+        }
+
+        std::error_code ec;
+        fs.rename_or_copy(work_tree, destination, ".tmp", ec);
+        fs.remove_all(work_tree, VCPKG_LINE_INFO);
+        if (ec)
+        {
+            System::printf(System::Color::error,
+                           "Error: Couldn't move checked out files from %s to destination %s",
+                           fs::u8string(work_tree),
+                           fs::u8string(destination));
+            Checks::exit_fail(VCPKG_LINE_INFO);
+        }
+    }
+
+    fs::path VcpkgPaths::git_checkout_baseline(Files::Filesystem& fs, StringView commit_sha) const
+    {
+        const fs::path local_repo = this->root;
+        const fs::path destination = this->baselines_output / fs::u8path(commit_sha) / fs::u8path("baseline.json");
+        const fs::path baseline_subpath = fs::u8path("port_versions") / fs::u8path("baseline.json");
+
+        if (!fs.exists(destination))
+        {
+            git_checkout_subpath(*this,
+                                 commit_sha,
+                                 baseline_subpath,
+                                 local_repo,
+                                 destination,
+                                 this->baselines_dot_git_dir,
+                                 this->baselines_work_tree);
+        }
+        return destination;
+    }
+
+    fs::path VcpkgPaths::git_checkout_port(Files::Filesystem& fs, StringView port_name, StringView git_tree) const
+    {
+        /* Clone a new vcpkg repository instance using the local instance as base.
+         *
+         * The `--git-dir` directory will store all the Git metadata files,
+         * and the  `--work-tree` is the directory where files will be checked out.
+         *
+         * Since we are checking a git tree object, all files will be checked out to the root of `work-tree`.
+         * Because of that, it makes sense to use the git hash as the name for the directory.
+         */
+        const fs::path local_repo = this->root;
+        const fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name);
+
+        if (!fs.exists(destination / "CONTROL") && !fs.exists(destination / "vcpkg.json"))
+        {
+            git_checkout_object(
+                *this, git_tree, local_repo, destination, this->versions_dot_git_dir, this->versions_work_tree);
+        }
+        return destination;
+    }
+
     Optional<const Json::Object&> VcpkgPaths::get_manifest() const
     {
         if (auto p = m_pimpl->m_manifest_doc.get())

toolsrc/src/vcpkg/versiondeserializers.cpp

@@ -0,0 +1,210 @@
+#include <vcpkg/versiondeserializers.h>
+
+using namespace vcpkg;
+using namespace vcpkg::Versions;
+
+namespace
+{
+    struct VersionDbEntryDeserializer final : Json::IDeserializer<VersionDbEntry>
+    {
+        static constexpr StringLiteral VERSION_RELAXED = "version";
+        static constexpr StringLiteral VERSION_SEMVER = "version-semver";
+        static constexpr StringLiteral VERSION_STRING = "version-string";
+        static constexpr StringLiteral VERSION_DATE = "version-date";
+        static constexpr StringLiteral PORT_VERSION = "port-version";
+        static constexpr StringLiteral GIT_TREE = "git-tree";
+
+        StringView type_name() const override { return "a version database entry"; }
+        View<StringView> valid_fields() const override
+        {
+            static const StringView t[] = {
+                VERSION_RELAXED, VERSION_SEMVER, VERSION_STRING, VERSION_DATE, PORT_VERSION, GIT_TREE};
+            return t;
+        }
+
+        Optional<VersionDbEntry> visit_object(Json::Reader& r, const Json::Object& obj) override
+        {
+            std::string version;
+            int port_version = 0;
+            std::string git_tree;
+            Versions::Scheme version_scheme = Versions::Scheme::String;
+
+            // Code copy-and-paste'd from sourceparagraph.cpp
+            static Json::StringDeserializer version_exact_deserializer{"an exact version string"};
+            static Json::StringDeserializer version_relaxed_deserializer{"a relaxed version string"};
+            static Json::StringDeserializer version_semver_deserializer{"a semantic version string"};
+            static Json::StringDeserializer version_date_deserializer{"a date version string"};
+            static Json::StringDeserializer git_tree_deserializer("a git object SHA");
+
+            bool has_exact = r.optional_object_field(obj, VERSION_STRING, version, version_exact_deserializer);
+            bool has_relax = r.optional_object_field(obj, VERSION_RELAXED, version, version_relaxed_deserializer);
+            bool has_semver = r.optional_object_field(obj, VERSION_SEMVER, version, version_semver_deserializer);
+            bool has_date = r.optional_object_field(obj, VERSION_DATE, version, version_date_deserializer);
+            int num_versions = (int)has_exact + (int)has_relax + (int)has_semver + (int)has_date;
+            if (num_versions == 0)
+            {
+                r.add_generic_error(type_name(), "expected a versioning field (example: ", VERSION_STRING, ")");
+            }
+            else if (num_versions > 1)
+            {
+                r.add_generic_error(type_name(), "expected only one versioning field");
+            }
+            else
+            {
+                if (has_exact)
+                    version_scheme = Versions::Scheme::String;
+                else if (has_relax)
+                    version_scheme = Versions::Scheme::Relaxed;
+                else if (has_semver)
+                    version_scheme = Versions::Scheme::Semver;
+                else if (has_date)
+                    version_scheme = Versions::Scheme::Date;
+                else
+                    Checks::unreachable(VCPKG_LINE_INFO);
+            }
+            r.optional_object_field(obj, PORT_VERSION, port_version, Json::NaturalNumberDeserializer::instance);
+            r.required_object_field(type_name(), obj, GIT_TREE, git_tree, git_tree_deserializer);
+
+            return VersionDbEntry(version, port_version, version_scheme, git_tree);
+        }
+
+        static VersionDbEntryDeserializer instance;
+    };
+
+    struct VersionDbEntryArrayDeserializer final : Json::IDeserializer<std::vector<VersionDbEntry>>
+    {
+        virtual StringView type_name() const override { return "an array of versions"; }
+
+        virtual Optional<std::vector<VersionDbEntry>> visit_array(Json::Reader& r, const Json::Array& arr) override
+        {
+            return r.array_elements(arr, VersionDbEntryDeserializer::instance);
+        }
+
+        static VersionDbEntryArrayDeserializer instance;
+    };
+
+    VersionDbEntryDeserializer VersionDbEntryDeserializer::instance;
+    VersionDbEntryArrayDeserializer VersionDbEntryArrayDeserializer::instance;
+
+    struct BaselineDeserializer final : Json::IDeserializer<std::map<std::string, VersionT, std::less<>>>
+    {
+        StringView type_name() const override { return "a baseline object"; }
+
+        Optional<type> visit_object(Json::Reader& r, const Json::Object& obj) override
+        {
+            std::map<std::string, VersionT, std::less<>> result;
+
+            for (auto&& pr : obj)
+            {
+                const auto& version_value = pr.second;
+                VersionT version;
+                r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance());
+
+                result.emplace(pr.first.to_string(), std::move(version));
+            }
+
+            return std::move(result);
+        }
+
+        static BaselineDeserializer instance;
+    };
+    BaselineDeserializer BaselineDeserializer::instance;
+
+    struct VersionTDeserializer final : Json::IDeserializer<VersionT>
+    {
+        StringView type_name() const override { return "a version object"; }
+        View<StringView> valid_fields() const override
+        {
+            static const StringView t[] = {"version-string", "port-version"};
+            return t;
+        }
+
+        Optional<VersionT> visit_object(Json::Reader& r, const Json::Object& obj) override
+        {
+            std::string version;
+            int port_version = 0;
+
+            r.required_object_field(type_name(), obj, "version-string", version, version_deserializer);
+            r.optional_object_field(obj, "port-version", port_version, Json::NaturalNumberDeserializer::instance);
+
+            return VersionT{std::move(version), port_version};
+        }
+
+        static Json::StringDeserializer version_deserializer;
+        static VersionTDeserializer instance;
+    };
+    Json::StringDeserializer VersionTDeserializer::version_deserializer{"version"};
+    VersionTDeserializer VersionTDeserializer::instance;
+}
+
+namespace vcpkg
+{
+    Json::IDeserializer<VersionT>& get_versiont_deserializer_instance() { return VersionTDeserializer::instance; }
+
+    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
+                                                                                StringView baseline_name,
+                                                                                const fs::path& baseline_file_path)
+    {
+        if (!fs.exists(baseline_file_path))
+        {
+            return Strings::format("Couldn't find `%s`", fs::u8string(baseline_file_path));
+        }
+
+        auto value = Json::parse_file(VCPKG_LINE_INFO, fs, baseline_file_path);
+        if (!value.first.is_object())
+        {
+            return Strings::format("Error: `%s` does not have a top-level object.", fs::u8string(baseline_file_path));
+        }
+
+        const auto& obj = value.first.object();
+        auto baseline_value = obj.get(baseline_name);
+        if (!baseline_value)
+        {
+            return Strings::format(
+                "Error: `%s` does not contain the baseline \"%s\"", fs::u8string(baseline_file_path), baseline_name);
+        }
+
+        Json::Reader r;
+        std::map<std::string, VersionT, std::less<>> result;
+        r.visit_in_key(*baseline_value, baseline_name, result, BaselineDeserializer::instance);
+        if (!r.errors().empty())
+        {
+            return Strings::format(
+                "Error: failed to parse `%s`:\n%s", fs::u8string(baseline_file_path), Strings::join("\n", r.errors()));
+        }
+        return result;
+    }
+
+    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
+                                                               StringView port_name,
+                                                               const fs::path& versions_file_path)
+    {
+        (void)port_name;
+        if (!fs.exists(versions_file_path))
+        {
+            return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path));
+        }
+
+        auto versions_json = Json::parse_file(VCPKG_LINE_INFO, fs, versions_file_path);
+        if (!versions_json.first.is_object())
+        {
+            return Strings::format("Error: `%s` does not have a top level object.", fs::u8string(versions_file_path));
+        }
+
+        const auto& versions_object = versions_json.first.object();
+        auto maybe_versions_array = versions_object.get("versions");
+        if (!maybe_versions_array || !maybe_versions_array->is_array())
+        {
+            return Strings::format("Error: `%s` does not contain a versions array.", fs::u8string(versions_file_path));
+        }
+
+        std::vector<VersionDbEntry> db_entries;
+        // Avoid warning treated as error.
+        if (maybe_versions_array != nullptr)
+        {
+            Json::Reader r;
+            r.visit_in_key(*maybe_versions_array, "versions", db_entries, VersionDbEntryArrayDeserializer::instance);
+        }
+        return db_entries;
+    }
+}

toolsrc/src/vcpkg/versions.cpp

@@ -0,0 +1,34 @@
+#include <vcpkg/versions.h>
+
+namespace vcpkg::Versions
+{
+    VersionSpec::VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme)
+        : port_name(port_name), version(version), scheme(scheme)
+    {
+    }
+
+    VersionSpec::VersionSpec(const std::string& port_name,
+                             const std::string& version_string,
+                             int port_version,
+                             Scheme scheme)
+        : port_name(port_name), version(version_string, port_version), scheme(scheme)
+    {
+    }
+
+    bool operator==(const VersionSpec& lhs, const VersionSpec& rhs)
+    {
+        return std::tie(lhs.port_name, lhs.version, lhs.scheme) == std::tie(rhs.port_name, rhs.version, rhs.scheme);
+    }
+
+    bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs) { return !(lhs == rhs); }
+
+    std::size_t VersionSpecHasher::operator()(const VersionSpec& key) const
+    {
+        using std::hash;
+        using std::size_t;
+        using std::string;
+
+        return ((hash<string>()(key.port_name) ^ (hash<string>()(key.version.to_string()) << 1)) >> 1) ^
+               (hash<int>()(static_cast<int>(key.scheme)) << 1);
+    }
+}

toolsrc/windows-bootstrap/vcpkg.vcxproj

@@ -267,6 +267,7 @@
     <ClInclude Include="..\include\vcpkg\vcpkgcmdarguments.h" />
     <ClInclude Include="..\include\vcpkg\vcpkglib.h" />
     <ClInclude Include="..\include\vcpkg\vcpkgpaths.h" />
+    <ClInclude Include="..\include\vcpkg\versiondeserializers.h" />
     <ClInclude Include="..\include\vcpkg\versions.h" />
     <ClInclude Include="..\include\vcpkg\versiont.h" />
     <ClInclude Include="..\include\vcpkg\visualstudio.h" />
@@ -355,6 +356,8 @@
     <ClCompile Include="..\src\vcpkg\vcpkgcmdarguments.cpp" />
     <ClCompile Include="..\src\vcpkg\vcpkglib.cpp" />
     <ClCompile Include="..\src\vcpkg\vcpkgpaths.cpp" />
+    <ClCompile Include="..\src\vcpkg\versiondeserializers.cpp" />
+    <ClCompile Include="..\src\vcpkg\versions.cpp" />
     <ClCompile Include="..\src\vcpkg\versiont.cpp" />
     <ClCompile Include="..\src\vcpkg\visualstudio.cpp" />
     <ClCompile Include="..\src\vcpkg.cpp" />