[vcpkg registries] support versions (#15114)

* [vcpkg registries] support versions

This PR merges the Registries changes and the versioning changes, so that one can use both at the same time.

There is one major difference between this PR and the RFC (#13590), which is that instead of version files looking like:

```json
[
  ...
]
```

version files look like:

```
{
  "versions": [
    ...
  ]
}
```

this is to support interop between this PR and existing demos and the like;
fixing this, along with perhaps renaming `port_versions` to `port-versions` should be done after this is merged,
should be a trivial change.

scripts/azure-pipelines/end-to-end-tests-dir/create.ps1

@@ -1,8 +1,8 @@
 . $PSScriptRoot/../end-to-end-tests-prelude.ps1
 
 # Test vcpkg create
-$CurrentTest = "create zlib"
-Write-Host $CurrentTest
+$Script:CurrentTest = "create zlib"
+Write-Host $Script:CurrentTest
 ./vcpkg --x-builtin-ports-root=$TestingRoot/ports create zlib https://github.com/madler/zlib/archive/v1.2.11.tar.gz zlib-1.2.11.tar.gz
 Throw-IfFailed
 

scripts/azure-pipelines/end-to-end-tests-dir/integrate-install.ps1

@@ -2,24 +2,24 @@ if (-not $IsLinux -and -not $IsMacOS) {
     . $PSScriptRoot/../end-to-end-tests-prelude.ps1
 
     # Test msbuild props and targets
-    $CurrentTest = "zlib:x86-windows-static msbuild scripts\testing\integrate-install\..."
-    Write-Host $CurrentTest
+    $Script:CurrentTest = "zlib:x86-windows-static msbuild scripts\testing\integrate-install\..."
+    Write-Host $Script:CurrentTest
     ./vcpkg $commonArgs install zlib:x86-windows-static --x-binarysource=clear
     Throw-IfFailed
     foreach ($project in @("VcpkgTriplet", "VcpkgTriplet2", "VcpkgUseStatic", "VcpkgUseStatic2")) {
-        $CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj"
+        $Script:CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj"
         ./vcpkg $commonArgs env "msbuild scripts\testing\integrate-install\$project.vcxproj /p:VcpkgRoot=$TestingRoot /p:IntDir=$TestingRoot\int\ /p:OutDir=$TestingRoot\out\ "
         Throw-IfFailed
         Remove-Item -Recurse -Force $TestingRoot\int
         Remove-Item -Recurse -Force $TestingRoot\out
     }
-    $CurrentTest = "zlib:x86-windows msbuild scripts\testing\integrate-install\..."
-    Write-Host $CurrentTest
+    $Script:CurrentTest = "zlib:x86-windows msbuild scripts\testing\integrate-install\..."
+    Write-Host $Script:CurrentTest
     ./vcpkg $commonArgs install zlib:x86-windows --x-binarysource=clear
     Throw-IfFailed
     foreach ($project in @("Project1", "NoProps")) {
-        $CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj"
-        Write-Host $CurrentTest
+        $Script:CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj"
+        Write-Host $Script:CurrentTest
         ./vcpkg $commonArgs env "msbuild scripts\testing\integrate-install\$project.vcxproj /p:VcpkgRoot=$TestingRoot /p:IntDir=$TestingRoot\int\ /p:OutDir=$TestingRoot\out\ "
         Throw-IfFailed
         Remove-Item -Recurse -Force $TestingRoot\int

scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1

@@ -0,0 +1,13 @@
+. "$PSScriptRoot/../end-to-end-tests-prelude.ps1"
+
+
+$commonArgs += @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions")
+
+Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port'
+Throw-IfNotFailed
+
+Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
+Throw-IfFailed
+
+Run-Vcpkg install @commonArgs --feature-flags=registries 'zlib'
+Throw-IfFailed

scripts/azure-pipelines/end-to-end-tests-dir/spaces.ps1

@@ -1,8 +1,8 @@
 . $PSScriptRoot/../end-to-end-tests-prelude.ps1
 
 ##### Test spaces in the path
-$CurrentTest = "zlib with spaces in path"
-Write-Host $CurrentTest
+$Script:CurrentTest = "zlib with spaces in path"
+Write-Host $Script:CurrentTest
 ./vcpkg install zlib "--triplet" $Triplet `
     "--no-binarycaching" `
     "--x-buildtrees-root=$TestingRoot/build Trees" `

scripts/azure-pipelines/end-to-end-tests-prelude.ps1

@@ -12,9 +12,9 @@ $commonArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",
     "--x-install-root=$installRoot",
     "--x-packages-root=$packagesRoot",
-    "--overlay-ports=scripts/e2e_ports"
+    "--overlay-ports=scripts/e2e_ports/overlays"
 )
-$CurrentTest = 'unassigned'
+$Script:CurrentTest = 'unassigned'
 
 function Refresh-TestRoot {
     Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue
@@ -28,7 +28,7 @@ function Require-FileExists {
         [string]$File
     )
     if (-Not (Test-Path $File)) {
-        throw "'$CurrentTest' failed to create file '$File'"
+        throw "'$Script:CurrentTest' failed to create file '$File'"
     }
 }
 
@@ -38,26 +38,29 @@ function Require-FileNotExists {
         [string]$File
     )
     if (Test-Path $File) {
-        throw "'$CurrentTest' should not have created file '$File'"
+        throw "'$Script:CurrentTest' should not have created file '$File'"
     }
 }
 
 function Throw-IfFailed {
     if ($LASTEXITCODE -ne 0) {
-        throw "'$CurrentTest' had a step with a nonzero exit code"
+        throw "'$Script:CurrentTest' had a step with a nonzero exit code"
     }
 }
 
 function Throw-IfNotFailed {
     if ($LASTEXITCODE -eq 0) {
-        throw "'$CurrentTest' had a step with an unexpectedly zero exit code"
+        throw "'$Script:CurrentTest' had a step with an unexpectedly zero exit code"
     }
 }
 
 function Run-Vcpkg {
-    param([string[]]$TestArgs)
-    $CurrentTest = "./vcpkg $($testArgs -join ' ')"
-    Write-Host $CurrentTest
+    Param(
+        [Parameter(ValueFromRemainingArguments)]
+        [string[]]$TestArgs
+    )
+    $Script:CurrentTest = "./vcpkg $($testArgs -join ' ')"
+    Write-Host $Script:CurrentTest
     ./vcpkg @testArgs
 }
 

scripts/azure-pipelines/end-to-end-tests.ps1

@@ -34,7 +34,7 @@ $ErrorActionPreference = "Stop"
 
 $AllTests = Get-ChildItem $PSScriptRoot/end-to-end-tests-dir/*.ps1
 if ($Filter -ne $Null) {
-    $AllTests = $AllTests | ? { $_ -match $Filter }
+    $AllTests = $AllTests | ? { $_.Name -match $Filter }
 }
 $n = 1
 $m = $AllTests.Count

scripts/e2e_ports/port_versions/baseline.json

@@ -0,0 +1,3 @@
+{
+	"vcpkg-internal-e2e-test-port": { "version-string": "1.0.0" }
+}

scripts/e2e_ports/port_versions/v-/vcpkg-internal-e2e-test-port.json

@@ -0,0 +1,8 @@
+{
+  "versions": [
+    {
+      "version-string": "1.0.0",
+      "git-tree": "1dc3e42a3c0cafe2884d379af4399273238b986e"
+    }
+  ]
+}

scripts/e2e_ports/vcpkg-internal-e2e-test-port/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

scripts/e2e_ports/vcpkg-internal-e2e-test-port/vcpkg.json

@@ -0,0 +1,4 @@
+{
+	"name": "vcpkg-internal-e2e-test-port",
+	"version-string": "1.0.0"
+}

toolsrc/include/vcpkg/base/files.h

@@ -239,6 +239,12 @@ namespace vcpkg::Files
         virtual void current_path(const fs::path& path, std::error_code&) = 0;
         void current_path(const fs::path& path, LineInfo li);
 
+        // if the path does not exist, then (try_|)take_exclusive_file_lock attempts to create the file
+        // (but not any path members above the file itself)
+        // in other words, if `/a/b` is a directory, and you're attempting to lock `/a/b/c`,
+        // then these lock functions create `/a/b/c` if it doesn't exist;
+        // however, if `/a/b` doesn't exist, then the functions will fail.
+
         // waits forever for the file lock
         virtual fs::SystemHandle take_exclusive_file_lock(const fs::path& path, std::error_code&) = 0;
         // waits, at most, 1.5 seconds, for the file lock

toolsrc/include/vcpkg/base/jsonreader.h

@@ -75,13 +75,13 @@ namespace vcpkg::Json
         };
         std::vector<Path> m_path;
 
+    public:
         // checks that an object doesn't contain any fields which both:
         // * don't start with a `$`
         // * are not in `valid_fields`
         // if known_fields.empty(), then it's treated as if all field names are valid
         void check_for_unexpected_fields(const Object& obj, View<StringView> valid_fields, StringView type_name);
 
-    public:
         template<class Type>
         void required_object_field(
             StringView type, const Object& obj, StringView key, Type& place, IDeserializer<Type>& visitor)

toolsrc/include/vcpkg/configuration.h

@@ -1,6 +1,10 @@
 #pragma once
 
 #include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/json.h>
 
 #include <vcpkg/registries.h>
 
@@ -12,5 +16,10 @@ namespace vcpkg
         // `registries` and `default_registry`. The fall back logic is
         // taken care of in RegistrySet.
         RegistrySet registry_set;
+
+        void validate_feature_flags(const FeatureFlagSettings& flags);
     };
+
+    std::unique_ptr<Json::IDeserializer<Configuration>> make_configuration_deserializer(
+        const fs::path& config_directory);
 }

toolsrc/include/vcpkg/configurationdeserializer.h

@@ -1,66 +0,0 @@
-#pragma once
-
-#include <vcpkg/base/fwd/json.h>
-
-#include <vcpkg/fwd/vcpkgcmdarguments.h>
-
-#include <vcpkg/base/json.h>
-#include <vcpkg/base/optional.h>
-#include <vcpkg/base/stringliteral.h>
-#include <vcpkg/base/stringview.h>
-#include <vcpkg/base/view.h>
-
-#include <vcpkg/configuration.h>
-#include <vcpkg/registries.h>
-
-namespace vcpkg
-{
-    struct RegistryImplDeserializer : Json::IDeserializer<std::unique_ptr<RegistryImpl>>
-    {
-        constexpr static StringLiteral KIND = "kind";
-        constexpr static StringLiteral PATH = "path";
-
-        constexpr static StringLiteral KIND_BUILTIN = "builtin";
-        constexpr static StringLiteral KIND_FILESYSTEM = "filesystem";
-
-        virtual StringView type_name() const override;
-        virtual View<StringView> valid_fields() const override;
-
-        virtual Optional<std::unique_ptr<RegistryImpl>> visit_null(Json::Reader&) override;
-        virtual Optional<std::unique_ptr<RegistryImpl>> visit_object(Json::Reader&, const Json::Object&) override;
-
-        static RegistryImplDeserializer instance;
-    };
-
-    struct RegistryDeserializer final : Json::IDeserializer<Registry>
-    {
-        constexpr static StringLiteral PACKAGES = "packages";
-
-        virtual StringView type_name() const override;
-        virtual View<StringView> valid_fields() const override;
-
-        virtual Optional<Registry> visit_object(Json::Reader&, const Json::Object&) override;
-    };
-
-    struct ConfigurationDeserializer final : Json::IDeserializer<Configuration>
-    {
-        virtual StringView type_name() const override { return "a configuration object"; }
-
-        constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry";
-        constexpr static StringLiteral REGISTRIES = "registries";
-        virtual View<StringView> valid_fields() const override
-        {
-            constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES};
-            return t;
-        }
-
-        virtual Optional<Configuration> visit_object(Json::Reader& r, const Json::Object& obj) override;
-
-        ConfigurationDeserializer(const VcpkgCmdArguments& args);
-
-    private:
-        bool print_json;
-
-        bool registries_enabled;
-    };
-}

toolsrc/include/vcpkg/fwd/registries.h

@@ -2,7 +2,8 @@
 
 namespace vcpkg
 {
-    struct RegistryImpl;
+    struct RegistryEntry;
+    struct RegistryImplementation;
     struct Registry;
     struct RegistrySet;
 }

toolsrc/include/vcpkg/portfileprovider.h

@@ -5,6 +5,7 @@
 #include <vcpkg/base/expected.h>
 #include <vcpkg/base/util.h>
 
+#include <vcpkg/registries.h>
 #include <vcpkg/sourceparagraph.h>
 #include <vcpkg/versions.h>
 
@@ -41,21 +42,19 @@ namespace vcpkg::PortFileProvider
 
     struct IVersionedPortfileProvider
     {
-        virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const = 0;
+        virtual View<VersionT> get_port_versions(StringView port_name) const = 0;
         virtual ~IVersionedPortfileProvider() = default;
 
         virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
-            const vcpkg::Versions::VersionSpec& version_spec) const = 0;
+            const Versions::VersionSpec& version_spec) const = 0;
     };
 
     struct IBaselineProvider
     {
-        virtual ~IBaselineProvider() = default;
-
         virtual Optional<VersionT> get_baseline_version(StringView port_name) const = 0;
+        virtual ~IBaselineProvider() = default;
     };
 
-    std::unique_ptr<IBaselineProvider> make_baseline_provider(const vcpkg::VcpkgPaths& paths);
-    std::unique_ptr<IBaselineProvider> make_baseline_provider(const vcpkg::VcpkgPaths& paths, StringView baseline);
-    std::unique_ptr<IVersionedPortfileProvider> make_versioned_portfile_provider(const vcpkg::VcpkgPaths& paths);
+    std::unique_ptr<IBaselineProvider> make_baseline_provider(const VcpkgPaths&);
+    std::unique_ptr<IVersionedPortfileProvider> make_versioned_portfile_provider(const VcpkgPaths&);
 }

toolsrc/include/vcpkg/registries.h

@@ -1,8 +1,11 @@
 #pragma once
 
+#include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/registries.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/jsonreader.h>
 #include <vcpkg/base/stringview.h>
 #include <vcpkg/base/view.h>
 
@@ -17,40 +20,45 @@ namespace vcpkg
 {
     struct RegistryEntry
     {
-        // returns fs::path() if version doesn't exist
-        virtual fs::path get_port_directory(const VcpkgPaths& paths, const VersionT& version) const = 0;
+        virtual View<VersionT> get_port_versions() const = 0;
+
+        virtual ExpectedS<fs::path> get_path_to_version(const VcpkgPaths& paths, const VersionT& version) const = 0;
 
         virtual ~RegistryEntry() = default;
     };
 
-    struct RegistryImpl
+    struct RegistryImplementation
     {
         // returns nullptr if the port doesn't exist
         virtual std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths& paths, StringView port_name) const = 0;
+
         // appends the names of the ports to the out parameter
+        // may result in duplicated port names; make sure to Util::sort_unique_erase at the end
         virtual void get_all_port_names(std::vector<std::string>& port_names, const VcpkgPaths& paths) const = 0;
 
         virtual Optional<VersionT> get_baseline_version(const VcpkgPaths& paths, StringView port_name) const = 0;
 
-        virtual ~RegistryImpl() = default;
+        virtual ~RegistryImplementation() = default;
     };
 
     struct Registry
     {
         // requires: static_cast<bool>(implementation)
-        Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImpl>&& implementation);
+        Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImplementation>&& implementation);
 
         Registry(std::vector<std::string>&&, std::nullptr_t) = delete;
 
         // always ordered lexicographically
         View<std::string> packages() const { return packages_; }
-        const RegistryImpl& implementation() const { return *implementation_; }
+        const RegistryImplementation& implementation() const { return *implementation_; }
 
-        static std::unique_ptr<RegistryImpl> builtin_registry();
+        static std::unique_ptr<RegistryImplementation> builtin_registry(std::string&& baseline = {});
+
+        friend RegistrySet; // for experimental_set_builtin_registry_baseline
 
     private:
         std::vector<std::string> packages_;
-        std::unique_ptr<RegistryImpl> implementation_;
+        std::unique_ptr<RegistryImplementation> implementation_;
     };
 
     // this type implements the registry fall back logic from the registries RFC:
@@ -65,20 +73,34 @@ namespace vcpkg
 
         // finds the correct registry for the port name
         // Returns the null pointer if there is no registry set up for that name
-        const RegistryImpl* registry_for_port(StringView port_name) const;
+        const RegistryImplementation* registry_for_port(StringView port_name) const;
+        Optional<VersionT> baseline_for_port(const VcpkgPaths& paths, StringView port_name) const;
 
         View<Registry> registries() const { return registries_; }
 
-        const RegistryImpl* default_registry() const { return default_registry_.get(); }
+        const RegistryImplementation* default_registry() const { return default_registry_.get(); }
 
         // TODO: figure out how to get this to return an error (or maybe it should be a warning?)
         void add_registry(Registry&& r);
-        void set_default_registry(std::unique_ptr<RegistryImpl>&& r);
+        void set_default_registry(std::unique_ptr<RegistryImplementation>&& r);
         void set_default_registry(std::nullptr_t r);
 
+        // this exists in order to allow versioning and registries to be developed and tested separately
+        void experimental_set_builtin_registry_baseline(StringView baseline) const;
+
+        // returns whether the registry set has any modifications to the default
+        // (i.e., whether `default_registry` was set, or `registries` had any entries)
+        // for checking against the registry feature flag.
+        bool has_modifications() const;
+
     private:
-        std::unique_ptr<RegistryImpl> default_registry_;
+        std::unique_ptr<RegistryImplementation> default_registry_;
         std::vector<Registry> registries_;
     };
 
+    std::unique_ptr<Json::IDeserializer<std::unique_ptr<RegistryImplementation>>>
+    get_registry_implementation_deserializer(const fs::path& configuration_directory);
+
+    std::unique_ptr<Json::IDeserializer<std::vector<Registry>>> get_registry_array_deserializer(
+        const fs::path& configuration_directory);
 }

toolsrc/include/vcpkg/sourceparagraph.h

@@ -116,8 +116,8 @@ namespace vcpkg
     Json::Object serialize_debug_manifest(const SourceControlFile& scf);
 
     /// <summary>
-    /// Full metadata of a package: core and other features. As well as the location the SourceControlFile was
-    /// loaded from.
+    /// Full metadata of a package: core and other features,
+    /// as well as the port directory the SourceControlFile was loaded from
     /// </summary>
     struct SourceControlFileLocation
     {

toolsrc/include/vcpkg/vcpkgcmdarguments.h

@@ -134,6 +134,8 @@ namespace vcpkg
         std::unique_ptr<std::string> scripts_root_dir;
         constexpr static StringLiteral BUILTIN_PORTS_ROOT_DIR_ARG = "x-builtin-ports-root";
         std::unique_ptr<std::string> builtin_ports_root_dir;
+        constexpr static StringLiteral BUILTIN_PORT_VERSIONS_DIR_ARG = "x-builtin-port-versions-dir";
+        std::unique_ptr<std::string> builtin_port_versions_dir;
 
         constexpr static StringLiteral DEFAULT_VISUAL_STUDIO_PATH_ENV = "VCPKG_VISUAL_STUDIO_PATH";
         std::unique_ptr<std::string> default_visual_studio_path;

toolsrc/include/vcpkg/vcpkgpaths.h

@@ -92,6 +92,7 @@ namespace vcpkg
         fs::path scripts;
         fs::path prefab;
         fs::path builtin_ports;
+        fs::path builtin_port_versions;
 
         fs::path tools;
         fs::path buildsystems;
@@ -116,7 +117,7 @@ namespace vcpkg
         const fs::path& get_tool_exe(const std::string& tool) const;
         const std::string& get_tool_version(const std::string& tool) const;
 
-        // Git manipulation
+        // Git manipulation in the vcpkg directory
         fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const;
         fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const;
         ExpectedS<std::string> git_show(const std::string& treeish, const fs::path& dot_git_dir) const;

toolsrc/include/vcpkg/versiondeserializers.h

@@ -10,13 +10,6 @@
 
 namespace vcpkg
 {
-    struct VersionDbEntry
-    {
-        VersionT version;
-        Versions::Scheme scheme = Versions::Scheme::String;
-        std::string git_tree;
-    };
-
     Json::IDeserializer<VersionT>& get_versiont_deserializer_instance();
     std::unique_ptr<Json::IDeserializer<std::string>> make_version_deserializer(StringLiteral type_name);
 
@@ -44,12 +37,4 @@ namespace vcpkg
                                    const std::string& version,
                                    int port_version,
                                    bool always_emit_port_version = false);
-
-    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
-                                                                                StringView baseline_name,
-                                                                                const fs::path& baseline_file_path);
-
-    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
-                                                               StringView port_name,
-                                                               const fs::path& versions_file_path);
 }

toolsrc/src/vcpkg-test/dependencies.cpp

@@ -54,10 +54,7 @@ struct MockVersionedPortfileProvider : PortFileProvider::IVersionedPortfileProvi
         return it2->second;
     }
 
-    virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView) const override
-    {
-        Checks::unreachable(VCPKG_LINE_INFO);
-    }
+    virtual View<vcpkg::VersionT> get_port_versions(StringView) const override { Checks::unreachable(VCPKG_LINE_INFO); }
 
     SourceControlFileLocation& emplace(std::string&& name,
                                        Versions::Version&& version,

toolsrc/src/vcpkg-test/registries.cpp

@@ -0,0 +1,132 @@
+#include <catch2/catch.hpp>
+
+#include <vcpkg/base/jsonreader.h>
+
+#include <vcpkg/registries.h>
+
+using namespace vcpkg;
+
+namespace
+{
+    struct TestRegistryImplementation final : RegistryImplementation
+    {
+        std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths&, StringView) const override { return nullptr; }
+
+        void get_all_port_names(std::vector<std::string>&, const VcpkgPaths&) const override { }
+
+        Optional<VersionT> get_baseline_version(const VcpkgPaths&, StringView) const override { return nullopt; }
+
+        int number;
+
+        TestRegistryImplementation(int n) : number(n) { }
+    };
+
+    Registry make_registry(int n, std::vector<std::string>&& port_names)
+    {
+        return {std::move(port_names), std::make_unique<TestRegistryImplementation>(n)};
+    }
+
+    int get_tri_num(const RegistryImplementation& r)
+    {
+        if (auto tri = dynamic_cast<const TestRegistryImplementation*>(&r))
+        {
+            return tri->number;
+        }
+        else
+        {
+            return -1;
+        }
+    }
+
+    // test functions which parse string literals, so no concerns about failure
+    Json::Value parse_json(StringView sv) { return Json::parse(sv).value_or_exit(VCPKG_LINE_INFO).first; }
+}
+
+TEST_CASE ("registry_set_selects_registry", "[registries]")
+{
+    RegistrySet set;
+    set.set_default_registry(std::make_unique<TestRegistryImplementation>(0));
+
+    set.add_registry(make_registry(1, {"p1", "q1", "r1"}));
+    set.add_registry(make_registry(2, {"p2", "q2", "r2"}));
+
+    auto reg = set.registry_for_port("p1");
+    REQUIRE(reg);
+    CHECK(get_tri_num(*reg) == 1);
+    reg = set.registry_for_port("r2");
+    REQUIRE(reg);
+    CHECK(get_tri_num(*reg) == 2);
+    reg = set.registry_for_port("a");
+    REQUIRE(reg);
+    CHECK(get_tri_num(*reg) == 0);
+
+    set.set_default_registry(nullptr);
+
+    reg = set.registry_for_port("q1");
+    REQUIRE(reg);
+    CHECK(get_tri_num(*reg) == 1);
+    reg = set.registry_for_port("p2");
+    REQUIRE(reg);
+    CHECK(get_tri_num(*reg) == 2);
+    reg = set.registry_for_port("a");
+    CHECK_FALSE(reg);
+}
+
+TEST_CASE ("registry_parsing", "[registries]")
+{
+    Json::Reader r;
+    auto registry_impl_des = get_registry_implementation_deserializer({});
+
+    auto test_json = parse_json(R"json(
+{
+    "kind": "builtin"
+}
+    )json");
+    auto registry_impl = r.visit(test_json, *registry_impl_des);
+    REQUIRE(registry_impl);
+    CHECK(*registry_impl.get());
+    CHECK(r.errors().empty());
+
+    test_json = parse_json(R"json(
+{
+    "kind": "builtin",
+    "baseline": "hi"
+}
+    )json");
+    registry_impl = r.visit(test_json, *registry_impl_des);
+    REQUIRE(registry_impl);
+    CHECK(*registry_impl.get());
+    CHECK(r.errors().empty());
+
+    test_json = parse_json(R"json(
+{
+    "kind": "builtin",
+    "path": "a/b"
+}
+    )json");
+    registry_impl = r.visit(test_json, *registry_impl_des);
+    CHECK_FALSE(r.errors().empty());
+    r.errors().clear();
+
+    test_json = parse_json(R"json(
+{
+    "kind": "filesystem",
+    "path": "a/b/c"
+}
+    )json");
+    registry_impl = r.visit(test_json, *registry_impl_des);
+    REQUIRE(registry_impl);
+    CHECK(*registry_impl.get());
+    CHECK(r.errors().empty());
+
+    test_json = parse_json(R"json(
+{
+    "kind": "filesystem",
+    "path": "/a/b/c"
+}
+    )json");
+    registry_impl = r.visit(test_json, *registry_impl_des);
+    REQUIRE(registry_impl);
+    CHECK(*registry_impl.get());
+    CHECK(r.errors().empty());
+}

toolsrc/src/vcpkg/base/files.cpp

@@ -1216,7 +1216,7 @@ namespace vcpkg::Files
             {
             }
 
-#if defined(WIN32)
+#if defined(_WIN32)
             void assign_busy_error(std::error_code& ec) { ec.assign(ERROR_BUSY, std::system_category()); }
 
             bool operator()(std::error_code& ec)
@@ -1242,7 +1242,7 @@ namespace vcpkg::Files
                 res.system_handle = reinterpret_cast<intptr_t>(handle);
                 return true;
             }
-#else // ^^^ WIN32 / !WIN32 vvv
+#else // ^^^ _WIN32 / !_WIN32 vvv
             int fd = -1;
 
             void assign_busy_error(std::error_code& ec) { ec.assign(EBUSY, std::generic_category()); }
@@ -1252,7 +1252,7 @@ namespace vcpkg::Files
                 ec.clear();
                 if (fd == -1)
                 {
-                    fd = ::open(native.c_str(), 0);
+                    fd = ::open(native.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
                     if (fd < 0)
                     {
                         ec.assign(errno, std::generic_category());
@@ -1335,7 +1335,7 @@ namespace vcpkg::Files
 
         virtual void unlock_file_lock(fs::SystemHandle handle, std::error_code& ec) override
         {
-#if defined(WIN32)
+#if defined(_WIN32)
             if (CloseHandle(reinterpret_cast<HANDLE>(handle.system_handle)) == 0)
             {
                 ec.assign(GetLastError(), std::system_category());

toolsrc/src/vcpkg/configuration.cpp

@@ -2,66 +2,83 @@
 #include <vcpkg/base/system.print.h>
 
 #include <vcpkg/configuration.h>
-#include <vcpkg/configurationdeserializer.h>
 #include <vcpkg/vcpkgcmdarguments.h>
 
-namespace vcpkg
+namespace
 {
+    using namespace vcpkg;
+
+    struct ConfigurationDeserializer final : Json::IDeserializer<Configuration>
+    {
+        virtual StringView type_name() const override { return "a configuration object"; }
+
+        constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry";
+        constexpr static StringLiteral REGISTRIES = "registries";
+        virtual View<StringView> valid_fields() const override
+        {
+            constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES};
+            return t;
+        }
+
+        virtual Optional<Configuration> visit_object(Json::Reader& r, const Json::Object& obj) override;
+
+        ConfigurationDeserializer(const fs::path& configuration_directory);
+
+    private:
+        fs::path configuration_directory;
+    };
+
+    constexpr StringLiteral ConfigurationDeserializer::DEFAULT_REGISTRY;
+    constexpr StringLiteral ConfigurationDeserializer::REGISTRIES;
+
     Optional<Configuration> ConfigurationDeserializer::visit_object(Json::Reader& r, const Json::Object& obj)
     {
         RegistrySet registries;
 
-        bool registries_feature_flags_warning = false;
+        auto impl_des = get_registry_implementation_deserializer(configuration_directory);
 
+        std::unique_ptr<RegistryImplementation> default_registry;
+        if (r.optional_object_field(obj, DEFAULT_REGISTRY, default_registry, *impl_des))
         {
-            std::unique_ptr<RegistryImpl> default_registry;
-            if (r.optional_object_field(obj, DEFAULT_REGISTRY, default_registry, RegistryImplDeserializer::instance))
-            {
-                if (!registries_enabled)
-                {
-                    registries_feature_flags_warning = true;
-                }
-                else
-                {
-                    registries.set_default_registry(std::move(default_registry));
-                }
-            }
+            registries.set_default_registry(std::move(default_registry));
         }
 
-        static Json::ArrayDeserializer<RegistryDeserializer> array_of_registries{"an array of registries"};
-
+        auto reg_des = get_registry_array_deserializer(configuration_directory);
         std::vector<Registry> regs;
-        r.optional_object_field(obj, REGISTRIES, regs, array_of_registries);
-
-        if (!regs.empty() && !registries_enabled)
-        {
-            registries_feature_flags_warning = true;
-            regs.clear();
-        }
+        r.optional_object_field(obj, REGISTRIES, regs, *reg_des);
 
         for (Registry& reg : regs)
         {
             registries.add_registry(std::move(reg));
         }
 
-        if (registries_feature_flags_warning && !print_json)
+        return Configuration{std::move(registries)};
+    }
+
+    ConfigurationDeserializer::ConfigurationDeserializer(const fs::path& configuration_directory)
+        : configuration_directory(configuration_directory)
+    {
+    }
+
+}
+
+std::unique_ptr<Json::IDeserializer<Configuration>> vcpkg::make_configuration_deserializer(
+    const fs::path& config_directory)
+{
+    return std::make_unique<ConfigurationDeserializer>(config_directory);
+}
+
+namespace vcpkg
+{
+    void Configuration::validate_feature_flags(const FeatureFlagSettings& flags)
+    {
+        if (!flags.registries && registry_set.has_modifications())
         {
             System::printf(System::Color::warning,
                            "Warning: configuration specified the \"registries\" or \"default-registries\" field, but "
                            "the %s feature flag was not enabled.\n",
                            VcpkgCmdArguments::REGISTRIES_FEATURE);
+            registry_set = RegistrySet();
         }
-
-        return Configuration{std::move(registries)};
     }
-
-    constexpr StringLiteral ConfigurationDeserializer::DEFAULT_REGISTRY;
-    constexpr StringLiteral ConfigurationDeserializer::REGISTRIES;
-
-    ConfigurationDeserializer::ConfigurationDeserializer(const VcpkgCmdArguments& args)
-    {
-        registries_enabled = args.registries_enabled();
-        print_json = args.output_json();
-    }
-
 }

toolsrc/src/vcpkg/install.cpp

@@ -7,6 +7,7 @@
 #include <vcpkg/build.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.setinstalled.h>
+#include <vcpkg/configuration.h>
 #include <vcpkg/dependencies.h>
 #include <vcpkg/globalstate.h>
 #include <vcpkg/help.h>
@@ -16,6 +17,7 @@
 #include <vcpkg/paragraphs.h>
 #include <vcpkg/remove.h>
 #include <vcpkg/vcpkglib.h>
+#include <vcpkg/vcpkgpaths.h>
 
 namespace vcpkg::Install
 {
@@ -841,16 +843,12 @@ namespace vcpkg::Install
             if (args.versions_enabled())
             {
                 auto verprovider = PortFileProvider::make_versioned_portfile_provider(paths);
-                auto baseprovider = [&]() -> std::unique_ptr<PortFileProvider::IBaselineProvider> {
-                    if (auto p_baseline = manifest_scf.core_paragraph->extra_info.get("$x-default-baseline"))
-                    {
-                        return PortFileProvider::make_baseline_provider(paths, p_baseline->string().to_string());
-                    }
-                    else
-                    {
-                        return PortFileProvider::make_baseline_provider(paths);
-                    }
-                }();
+                auto baseprovider = PortFileProvider::make_baseline_provider(paths);
+                if (auto p_baseline = manifest_scf.core_paragraph->extra_info.get("$x-default-baseline"))
+                {
+                    paths.get_configuration().registry_set.experimental_set_builtin_registry_baseline(
+                        p_baseline->string());
+                }
 
                 auto install_plan =
                     Dependencies::create_versioned_install_plan(*verprovider,

toolsrc/src/vcpkg/paragraphs.cpp

@@ -357,14 +357,23 @@ namespace vcpkg::Paragraphs
 
             return res;
         }
-        ExpectedS<std::vector<Paragraph>> pghs = get_paragraphs(fs, path_to_control);
-        if (auto vector_pghs = pghs.get())
+
+        if (fs.exists(path_to_control))
         {
-            return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs));
+            ExpectedS<std::vector<Paragraph>> pghs = get_paragraphs(fs, path_to_control);
+            if (auto vector_pghs = pghs.get())
+            {
+                return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs));
+            }
+            auto error_info = std::make_unique<ParseControlErrorInfo>();
+            error_info->name = fs::u8string(path.filename());
+            error_info->error = pghs.error();
+            return error_info;
         }
+
         auto error_info = std::make_unique<ParseControlErrorInfo>();
         error_info->name = fs::u8string(path.filename());
-        error_info->error = pghs.error();
+        error_info->error = "Failed to find either a CONTROL file or vcpkg.json file.";
         return error_info;
     }
 
@@ -423,15 +432,8 @@ namespace vcpkg::Paragraphs
             auto baseline_version = impl->get_baseline_version(paths, port_name);
             if (port_entry && baseline_version)
             {
-                auto port_path = port_entry->get_port_directory(paths, *baseline_version.get());
-                if (port_path.empty())
-                {
-                    Debug::print("Registry for port `",
-                                 port_name,
-                                 "` is incorrect - baseline port version `",
-                                 baseline_version.get()->to_string(),
-                                 "` not found.");
-                }
+                auto port_path =
+                    port_entry->get_path_to_version(paths, *baseline_version.get()).value_or_exit(VCPKG_LINE_INFO);
                 auto maybe_spgh = try_load_port(fs, port_path);
                 if (const auto spgh = maybe_spgh.get())
                 {

toolsrc/src/vcpkg/portfileprovider.cpp

@@ -17,29 +17,25 @@ using namespace Versions;
 
 namespace
 {
-    ExpectedS<fs::path> get_versions_json_path(const VcpkgPaths& paths, StringView port_name)
-    {
-        auto json_path = paths.root / fs::u8path("port_versions") /
-                         fs::u8path(Strings::concat(port_name.substr(0, 1), "-")) /
-                         fs::u8path(Strings::concat(port_name, ".json"));
-        if (paths.get_filesystem().exists(json_path))
-        {
-            return std::move(json_path);
-        }
-        return {Strings::concat("Error: Versions database file does not exist: ", fs::u8string(json_path)),
-                expected_right_tag};
-    }
+    using namespace vcpkg;
 
-    ExpectedS<fs::path> get_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha)
+    struct OverlayRegistryEntry final : RegistryEntry
     {
-        auto baseline_path = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha);
-        if (paths.get_filesystem().exists(baseline_path))
+        OverlayRegistryEntry(fs::path&& p, VersionT&& v) : path(p), version(v) { }
+
+        View<VersionT> get_port_versions() const override { return {&version, 1}; }
+        ExpectedS<fs::path> get_path_to_version(const VcpkgPaths&, const VersionT& v) const override
         {
-            return std::move(baseline_path);
+            if (v == version)
+            {
+                return path;
+            }
+            return Strings::format("Version %s not found; only %s is available.", v.to_string(), version.to_string());
         }
-        return {Strings::concat("Error: Baseline database file does not exist: ", fs::u8string(baseline_path)),
-                expected_right_tag};
-    }
+
+        fs::path path;
+        VersionT version;
+    };
 }
 
 namespace vcpkg::PortFileProvider
@@ -95,9 +91,9 @@ namespace vcpkg::PortFileProvider
         }
     }
 
-    static Optional<SourceControlFileLocation> try_load_overlay_port(const Files::Filesystem& fs,
-                                                                     View<fs::path> overlay_ports,
-                                                                     const std::string& spec)
+    static std::unique_ptr<OverlayRegistryEntry> try_load_overlay_port(const Files::Filesystem& fs,
+                                                                       View<fs::path> overlay_ports,
+                                                                       const std::string& spec)
     {
         for (auto&& ports_dir : overlay_ports)
         {
@@ -105,11 +101,12 @@ namespace vcpkg::PortFileProvider
             if (Paragraphs::is_port_directory(fs, ports_dir))
             {
                 auto maybe_scf = Paragraphs::try_load_port(fs, ports_dir);
-                if (auto scf = maybe_scf.get())
+                if (auto scfp = maybe_scf.get())
                 {
-                    if (scf->get()->core_paragraph->name == spec)
+                    auto& scf = *scfp;
+                    if (scf->core_paragraph->name == spec)
                     {
-                        return SourceControlFileLocation{std::move(*scf), ports_dir};
+                        return std::make_unique<OverlayRegistryEntry>(fs::path(ports_dir), scf->to_versiont());
                     }
                 }
                 else
@@ -126,17 +123,18 @@ namespace vcpkg::PortFileProvider
             if (Paragraphs::is_port_directory(fs, ports_spec))
             {
                 auto found_scf = Paragraphs::try_load_port(fs, ports_spec);
-                if (auto scf = found_scf.get())
+                if (auto scfp = found_scf.get())
                 {
-                    if (scf->get()->core_paragraph->name == spec)
+                    auto& scf = *scfp;
+                    if (scf->core_paragraph->name == spec)
                     {
-                        return SourceControlFileLocation{std::move(*scf), std::move(ports_spec)};
+                        return std::make_unique<OverlayRegistryEntry>(std::move(ports_spec), scf->to_versiont());
                     }
                     Checks::exit_with_message(VCPKG_LINE_INFO,
                                               "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
                                               fs::u8string(ports_spec),
                                               spec,
-                                              scf->get()->core_paragraph->name);
+                                              scf->core_paragraph->name);
                 }
                 else
                 {
@@ -146,61 +144,38 @@ namespace vcpkg::PortFileProvider
                 }
             }
         }
-        return nullopt;
+        return nullptr;
     }
 
-    static Optional<SourceControlFileLocation> try_load_registry_port(const VcpkgPaths& paths, const std::string& spec)
+    static std::pair<std::unique_ptr<RegistryEntry>, Optional<VersionT>> try_load_registry_port_and_baseline(
+        const VcpkgPaths& paths, const std::string& spec)
     {
-        const auto& fs = paths.get_filesystem();
         if (auto registry = paths.get_configuration().registry_set.registry_for_port(spec))
         {
-            auto baseline_version = registry->get_baseline_version(paths, spec);
             auto entry = registry->get_port_entry(paths, spec);
-            if (entry && baseline_version)
+            auto maybe_baseline = registry->get_baseline_version(paths, spec);
+            if (entry)
             {
-                auto port_directory = entry->get_port_directory(paths, *baseline_version.get());
-                if (port_directory.empty())
+                if (!maybe_baseline)
                 {
-                    Checks::exit_with_message(VCPKG_LINE_INFO,
-                                              "Error: registry is incorrect. Baseline version for port `%s` is `%s`, "
-                                              "but that version is not in the registry.\n",
-                                              spec,
-                                              baseline_version.get()->to_string());
-                }
-                auto found_scf = Paragraphs::try_load_port(fs, port_directory);
-                if (auto scf = found_scf.get())
-                {
-                    if (scf->get()->core_paragraph->name == spec)
+                    if (entry->get_port_versions().size() == 1)
                     {
-                        return SourceControlFileLocation{std::move(*scf), std::move(port_directory)};
+                        maybe_baseline = entry->get_port_versions()[0];
                     }
-                    Checks::exit_with_message(VCPKG_LINE_INFO,
-                                              "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
-                                              fs::u8string(port_directory),
-                                              spec,
-                                              scf->get()->core_paragraph->name);
-                }
-                else
-                {
-                    print_error_message(found_scf.error());
-                    Checks::exit_with_message(
-                        VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(port_directory));
                 }
+                return {std::move(entry), std::move(maybe_baseline)};
             }
             else
             {
-                Debug::print("Failed to find port `",
-                             spec,
-                             "` in registry:",
-                             entry ? " entry found;" : " no entry found;",
-                             baseline_version ? " baseline version found\n" : " no baseline version found\n");
+                Debug::print("Failed to find port `", spec, "` in registry: no entry found.\n");
             }
         }
         else
         {
             Debug::print("Failed to find registry for port: `", spec, "`.\n");
         }
-        return nullopt;
+
+        return {nullptr, nullopt};
     }
 
     ExpectedS<const SourceControlFileLocation&> PathsPortFileProvider::get_control_file(const std::string& spec) const
@@ -209,18 +184,49 @@ namespace vcpkg::PortFileProvider
         if (cache_it == cache.end())
         {
             const auto& fs = paths.get_filesystem();
-            auto maybe_port = try_load_overlay_port(fs, overlay_ports, spec);
-            if (!maybe_port)
-            {
-                maybe_port = try_load_registry_port(paths, spec);
-            }
-            if (auto p = maybe_port.get())
-            {
-                auto maybe_error =
-                    p->source_control_file->check_against_feature_flags(p->source_location, paths.get_feature_flags());
-                if (maybe_error) return std::move(*maybe_error.get());
 
-                cache_it = cache.emplace(spec, std::move(*p)).first;
+            std::unique_ptr<RegistryEntry> port;
+            VersionT port_version;
+
+            auto maybe_overlay_port = try_load_overlay_port(fs, overlay_ports, spec);
+            if (maybe_overlay_port)
+            {
+                port_version = maybe_overlay_port->version;
+                port = std::move(maybe_overlay_port);
+            }
+            else
+            {
+                auto maybe_registry_port = try_load_registry_port_and_baseline(paths, spec);
+                port = std::move(maybe_registry_port.first);
+                if (auto version = maybe_registry_port.second.get())
+                {
+                    port_version = std::move(*version);
+                }
+                else if (port)
+                {
+                    return std::string("No baseline version available.");
+                }
+            }
+
+            if (port)
+            {
+                auto port_path = port->get_path_to_version(paths, port_version).value_or_exit(VCPKG_LINE_INFO);
+                auto maybe_scfl = Paragraphs::try_load_port(fs, port_path);
+                if (auto p = maybe_scfl.get())
+                {
+                    auto maybe_error = (*p)->check_against_feature_flags(port_path, paths.get_feature_flags());
+                    if (maybe_error) return std::move(*maybe_error.get());
+
+                    cache_it =
+                        cache.emplace(spec, SourceControlFileLocation{std::move(*p), std::move(port_path)}).first;
+                }
+                else
+                {
+                    return Strings::format("Error: when loading port `%s` from directory `%s`:\n%s\n",
+                                           spec,
+                                           fs::u8string(port_path),
+                                           maybe_scfl.error()->error);
+                }
             }
         }
 
@@ -296,179 +302,86 @@ namespace vcpkg::PortFileProvider
     {
         struct BaselineProviderImpl : IBaselineProvider, Util::ResourceBase
         {
-            BaselineProviderImpl(const VcpkgPaths& paths) : paths(paths) { }
-            BaselineProviderImpl(const VcpkgPaths& paths, StringView baseline)
-                : paths(paths), m_baseline(baseline.to_string())
-            {
-            }
-
-            const Optional<std::map<std::string, VersionT, std::less<>>>& get_baseline_cache() const
-            {
-                return baseline_cache.get_lazy([&]() -> Optional<std::map<std::string, VersionT, std::less<>>> {
-                    if (auto baseline = m_baseline.get())
-                    {
-                        auto baseline_file = get_baseline_json_path(paths, *baseline).value_or_exit(VCPKG_LINE_INFO);
-
-                        auto maybe_baselines_map =
-                            parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
-                        Checks::check_exit(VCPKG_LINE_INFO,
-                                           maybe_baselines_map.has_value(),
-                                           "Error: Couldn't parse baseline `%s` from `%s`",
-                                           "default",
-                                           fs::u8string(baseline_file));
-                        auto baselines_map = *maybe_baselines_map.get();
-                        return std::move(baselines_map);
-                    }
-                    else
-                    {
-                        // No baseline was provided, so use current repo
-                        const auto& fs = paths.get_filesystem();
-                        auto baseline_file = paths.root / fs::u8path("port_versions") / fs::u8path("baseline.json");
-                        if (fs.exists(baseline_file))
-                        {
-                            auto maybe_baselines_map =
-                                parse_baseline_file(paths.get_filesystem(), "default", baseline_file);
-                            Checks::check_exit(VCPKG_LINE_INFO,
-                                               maybe_baselines_map.has_value(),
-                                               "Error: Couldn't parse baseline `%s` from `%s`",
-                                               "default",
-                                               fs::u8string(baseline_file));
-                            auto baselines_map = *maybe_baselines_map.get();
-                            return std::move(baselines_map);
-                        }
-                        else
-                        {
-                            // No baseline file in current repo -- use current port versions.
-                            m_portfile_provider =
-                                std::make_unique<PathsPortFileProvider>(paths, std::vector<std::string>{});
-                            return nullopt;
-                        }
-                    }
-                });
-            }
+            BaselineProviderImpl(const VcpkgPaths& paths_) : paths(paths_) { }
 
             virtual Optional<VersionT> get_baseline_version(StringView port_name) const override
             {
-                const auto& cache = get_baseline_cache();
-                if (auto p_cache = cache.get())
+                auto it = m_baseline_cache.find(port_name);
+                if (it != m_baseline_cache.end())
                 {
-                    auto it = p_cache->find(port_name.to_string());
-                    if (it != p_cache->end())
-                    {
-                        return it->second;
-                    }
-                    return nullopt;
+                    return it->second;
                 }
                 else
                 {
-                    auto maybe_scfl = m_portfile_provider->get_control_file(port_name.to_string());
-                    if (auto p_scfl = maybe_scfl.get())
-                    {
-                        auto cpgh = p_scfl->source_control_file->core_paragraph.get();
-                        return VersionT{cpgh->version, cpgh->port_version};
-                    }
-                    else
-                    {
-                        return nullopt;
-                    }
+                    auto version = paths.get_configuration().registry_set.baseline_for_port(paths, port_name);
+                    m_baseline_cache.emplace(port_name.to_string(), version);
+                    return version;
                 }
             }
 
         private:
-            const VcpkgPaths& paths;
-            const Optional<std::string> m_baseline;
-            Lazy<Optional<std::map<std::string, VersionT, std::less<>>>> baseline_cache;
-            mutable std::unique_ptr<PathsPortFileProvider> m_portfile_provider;
+            const VcpkgPaths& paths; // TODO: remove this data member
+            mutable std::map<std::string, Optional<VersionT>, std::less<>> m_baseline_cache;
         };
 
         struct VersionedPortfileProviderImpl : IVersionedPortfileProvider, Util::ResourceBase
         {
-            VersionedPortfileProviderImpl(const VcpkgPaths& paths) : paths(paths) { }
+            VersionedPortfileProviderImpl(const VcpkgPaths& paths_) : paths(paths_) { }
 
-            virtual const std::vector<VersionSpec>& get_port_versions(StringView port_name) const override
+            virtual View<VersionT> get_port_versions(StringView port_name) const override
             {
-                auto cache_it = versions_cache.find(port_name.to_string());
-                if (cache_it != versions_cache.end())
+                auto entry_it = m_entry_cache.find(port_name.to_string());
+                if (entry_it != m_entry_cache.end())
                 {
-                    return cache_it->second;
+                    return entry_it->second->get_port_versions();
                 }
 
-                auto maybe_versions_file_path = get_versions_json_path(get_paths(), port_name);
-                if (auto versions_file_path = maybe_versions_file_path.get())
+                auto entry = try_load_registry_port_and_baseline(paths, port_name.to_string());
+                if (!entry.first)
                 {
-                    auto maybe_version_entries = parse_versions_file(get_filesystem(), port_name, *versions_file_path);
-                    Checks::check_exit(VCPKG_LINE_INFO,
-                                       maybe_version_entries.has_value(),
-                                       "Error: Couldn't parse versions from file: %s",
-                                       fs::u8string(*versions_file_path));
-                    auto version_entries = maybe_version_entries.value_or_exit(VCPKG_LINE_INFO);
-
-                    auto port = port_name.to_string();
-                    for (auto&& version_entry : version_entries)
-                    {
-                        VersionSpec spec(port, version_entry.version);
-                        versions_cache[port].push_back(spec);
-                        git_tree_cache.emplace(std::move(spec), std::move(version_entry.git_tree));
-                    }
-                    return versions_cache.at(port);
-                }
-                else
-                {
-                    // Fall back to current available version
-                    auto maybe_port = try_load_registry_port(paths, port_name.to_string());
-                    if (auto p = maybe_port.get())
-                    {
-                        auto maybe_error = p->source_control_file->check_against_feature_flags(
-                            p->source_location, paths.get_feature_flags());
-
-                        if (auto error = maybe_error.get())
-                        {
-                            Checks::exit_with_message(VCPKG_LINE_INFO, "Error: %s", *error);
-                        }
-
-                        VersionSpec vspec(port_name.to_string(),
-                                          VersionT(p->source_control_file->core_paragraph->version,
-                                                   p->source_control_file->core_paragraph->port_version));
-                        control_cache.emplace(vspec, std::move(*p));
-                        return versions_cache.emplace(port_name.to_string(), std::vector<VersionSpec>{vspec})
-                            .first->second;
-                    }
                     Checks::exit_with_message(
                         VCPKG_LINE_INFO, "Error: Could not find a definition for port %s", port_name);
                 }
+                auto it = m_entry_cache.emplace(port_name.to_string(), std::move(entry.first));
+                return it.first->second->get_port_versions();
             }
 
-            virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
-                const VersionSpec& version_spec) const override
+            ExpectedS<const SourceControlFileLocation&> get_control_file(const VersionSpec& version_spec) const override
             {
-                // Pre-populate versions cache.
-                get_port_versions(version_spec.port_name);
-
-                auto cache_it = control_cache.find(version_spec);
-                if (cache_it != control_cache.end())
+                auto cache_it = m_control_cache.find(version_spec);
+                if (cache_it != m_control_cache.end())
                 {
                     return cache_it->second;
                 }
 
-                auto git_tree_cache_it = git_tree_cache.find(version_spec);
-                if (git_tree_cache_it == git_tree_cache.end())
+                auto entry_it = m_entry_cache.find(version_spec.port_name);
+                if (entry_it == m_entry_cache.end())
                 {
-                    return Strings::concat("Error: No git object SHA for entry ",
-                                           version_spec.port_name,
-                                           " at version ",
-                                           version_spec.version,
-                                           ".");
+                    auto reg_for_port =
+                        paths.get_configuration().registry_set.registry_for_port(version_spec.port_name);
+
+                    if (!reg_for_port)
+                    {
+                        return Strings::format("Error: no registry set up for port %s", version_spec.port_name);
+                    }
+
+                    auto entry = reg_for_port->get_port_entry(paths, version_spec.port_name);
+                    entry_it = m_entry_cache.emplace(version_spec.port_name, std::move(entry)).first;
                 }
 
-                const std::string git_tree = git_tree_cache_it->second;
-                auto port_directory = get_paths().git_checkout_port(get_filesystem(), version_spec.port_name, git_tree);
+                auto maybe_path = entry_it->second->get_path_to_version(paths, version_spec.version);
+                if (!maybe_path.has_value())
+                {
+                    return std::move(maybe_path).error();
+                }
+                auto& port_directory = *maybe_path.get();
 
-                auto maybe_control_file = Paragraphs::try_load_port(get_filesystem(), port_directory);
+                auto maybe_control_file = Paragraphs::try_load_port(paths.get_filesystem(), port_directory);
                 if (auto scf = maybe_control_file.get())
                 {
                     if (scf->get()->core_paragraph->name == version_spec.port_name)
                     {
-                        return control_cache
+                        return m_control_cache
                             .emplace(version_spec,
                                      SourceControlFileLocation{std::move(*scf), std::move(port_directory)})
                             .first->second;
@@ -484,14 +397,10 @@ namespace vcpkg::PortFileProvider
                     "Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory));
             }
 
-            const VcpkgPaths& get_paths() const { return paths; }
-            Files::Filesystem& get_filesystem() const { return paths.get_filesystem(); }
-
         private:
-            const VcpkgPaths& paths;
-            mutable std::map<std::string, std::vector<VersionSpec>> versions_cache;
-            mutable std::unordered_map<VersionSpec, std::string, VersionSpecHasher> git_tree_cache;
-            mutable std::unordered_map<VersionSpec, SourceControlFileLocation, VersionSpecHasher> control_cache;
+            const VcpkgPaths& paths; // TODO: remove this data member
+            mutable std::unordered_map<VersionSpec, SourceControlFileLocation, VersionSpecHasher> m_control_cache;
+            mutable std::map<std::string, std::unique_ptr<RegistryEntry>, std::less<>> m_entry_cache;
         };
     }
 
@@ -500,11 +409,6 @@ namespace vcpkg::PortFileProvider
         return std::make_unique<BaselineProviderImpl>(paths);
     }
 
-    std::unique_ptr<IBaselineProvider> make_baseline_provider(const vcpkg::VcpkgPaths& paths, StringView baseline)
-    {
-        return std::make_unique<BaselineProviderImpl>(paths, baseline);
-    }
-
     std::unique_ptr<IVersionedPortfileProvider> make_versioned_portfile_provider(const vcpkg::VcpkgPaths& paths)
     {
         return std::make_unique<VersionedPortfileProviderImpl>(paths);

toolsrc/src/vcpkg/vcpkgcmdarguments.cpp

@@ -277,6 +277,7 @@ namespace vcpkg
                     {PACKAGES_ROOT_DIR_ARG, &VcpkgCmdArguments::packages_root_dir},
                     {SCRIPTS_ROOT_DIR_ARG, &VcpkgCmdArguments::scripts_root_dir},
                     {BUILTIN_PORTS_ROOT_DIR_ARG, &VcpkgCmdArguments::builtin_ports_root_dir},
+                    {BUILTIN_PORT_VERSIONS_DIR_ARG, &VcpkgCmdArguments::builtin_port_versions_dir},
                 };
 
             constexpr static std::pair<StringView, std::vector<std::string> VcpkgCmdArguments::*>

toolsrc/src/vcpkg/vcpkgpaths.cpp

@@ -10,7 +10,6 @@
 #include <vcpkg/build.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/configuration.h>
-#include <vcpkg/configurationdeserializer.h>
 #include <vcpkg/globalstate.h>
 #include <vcpkg/metrics.h>
 #include <vcpkg/packagespec.h>
@@ -89,9 +88,9 @@ namespace vcpkg
                                                    const fs::path& filepath)
     {
         Json::Reader reader;
-        ConfigurationDeserializer deserializer(args);
+        auto deserializer = make_configuration_deserializer(filepath.parent_path());
 
-        auto parsed_config_opt = reader.visit(obj, deserializer);
+        auto parsed_config_opt = reader.visit(obj, *deserializer);
         if (!reader.errors().empty())
         {
             System::print2(System::Color::error, "Errors occurred while parsing ", fs::u8string(filepath), "\n");
@@ -103,6 +102,8 @@ namespace vcpkg
             Checks::exit_fail(VCPKG_LINE_INFO);
         }
 
+        parsed_config_opt.get()->validate_feature_flags(args.feature_flag_settings());
+
         return std::move(parsed_config_opt).value_or_exit(VCPKG_LINE_INFO);
     }
 
@@ -342,6 +343,8 @@ If you wish to silence this error and use classic mode, you can:
         scripts = process_input_directory(filesystem, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO);
         builtin_ports =
             process_output_directory(filesystem, root, args.builtin_ports_root_dir.get(), "ports", VCPKG_LINE_INFO);
+        builtin_port_versions = process_output_directory(
+            filesystem, root, args.builtin_port_versions_dir.get(), "port_versions", VCPKG_LINE_INFO);
         prefab = root / fs::u8path("prefab");
 
         if (args.default_visual_studio_path)
@@ -493,11 +496,13 @@ If you wish to silence this error and use classic mode, you can:
         fs.remove_all(dot_git_dir, VCPKG_LINE_INFO);
 
         // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp}
-        // git clone --no-checkout --local {vcpkg_root} {dot_git_dir}
+        // git clone --no-checkout --local --no-hardlinks {vcpkg_root} {dot_git_dir}
+        // note that `--no-hardlinks` is added because otherwise, git fails to clone in some cases
         System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree)
                                                        .string_arg("clone")
                                                        .string_arg("--no-checkout")
                                                        .string_arg("--local")
+                                                       .string_arg("--no-hardlinks")
                                                        .path_arg(local_repo)
                                                        .path_arg(dot_git_dir);
         const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
@@ -578,6 +583,7 @@ If you wish to silence this error and use classic mode, you can:
                                                            .string_arg("clone")
                                                            .string_arg("--no-checkout")
                                                            .string_arg("--local")
+                                                           .string_arg("--no-hardlinks")
                                                            .path_arg(local_repo)
                                                            .path_arg(dot_git_dir);
             const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract());
@@ -662,8 +668,8 @@ If you wish to silence this error and use classic mode, you can:
          * Since we are checking a git tree object, all files will be checked out to the root of `work-tree`.
          * Because of that, it makes sense to use the git hash as the name for the directory.
          */
-        const fs::path local_repo = this->root;
-        const fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name);
+        const fs::path& local_repo = this->root;
+        fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name);
 
         if (!fs.exists(destination / "CONTROL") && !fs.exists(destination / "vcpkg.json"))
         {

toolsrc/src/vcpkg/versiondeserializers.cpp

@@ -146,75 +146,6 @@ namespace vcpkg
 
 namespace
 {
-    struct VersionDbEntryDeserializer final : Json::IDeserializer<VersionDbEntry>
-    {
-        static constexpr StringLiteral GIT_TREE = "git-tree";
-
-        StringView type_name() const override { return "a version database entry"; }
-        View<StringView> valid_fields() const override
-        {
-            static const StringView u[] = {GIT_TREE};
-            static const auto t = vcpkg::Util::Vectors::concat<StringView>(schemed_deserializer_fields(), u);
-            return t;
-        }
-
-        Optional<VersionDbEntry> visit_object(Json::Reader& r, const Json::Object& obj) override
-        {
-            VersionDbEntry ret;
-
-            auto schemed_version = visit_required_schemed_deserializer(type_name(), r, obj);
-            ret.scheme = schemed_version.scheme;
-            ret.version = std::move(schemed_version.versiont);
-
-            static Json::StringDeserializer git_tree_deserializer("a git object SHA");
-
-            r.required_object_field(type_name(), obj, GIT_TREE, ret.git_tree, git_tree_deserializer);
-
-            return std::move(ret);
-        }
-
-        static VersionDbEntryDeserializer instance;
-    };
-
-    struct VersionDbEntryArrayDeserializer final : Json::IDeserializer<std::vector<VersionDbEntry>>
-    {
-        virtual StringView type_name() const override { return "an array of versions"; }
-
-        virtual Optional<std::vector<VersionDbEntry>> visit_array(Json::Reader& r, const Json::Array& arr) override
-        {
-            return r.array_elements(arr, VersionDbEntryDeserializer::instance);
-        }
-
-        static VersionDbEntryArrayDeserializer instance;
-    };
-
-    VersionDbEntryDeserializer VersionDbEntryDeserializer::instance;
-    VersionDbEntryArrayDeserializer VersionDbEntryArrayDeserializer::instance;
-
-    struct BaselineDeserializer final : Json::IDeserializer<std::map<std::string, VersionT, std::less<>>>
-    {
-        StringView type_name() const override { return "a baseline object"; }
-
-        Optional<type> visit_object(Json::Reader& r, const Json::Object& obj) override
-        {
-            std::map<std::string, VersionT, std::less<>> result;
-
-            for (auto&& pr : obj)
-            {
-                const auto& version_value = pr.second;
-                VersionT version;
-                r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance());
-
-                result.emplace(pr.first.to_string(), std::move(version));
-            }
-
-            return std::move(result);
-        }
-
-        static BaselineDeserializer instance;
-    };
-    BaselineDeserializer BaselineDeserializer::instance;
-
     struct VersionTDeserializer final : Json::IDeserializer<VersionT>
     {
         StringView type_name() const override { return "a version object"; }
@@ -245,71 +176,4 @@ namespace
 namespace vcpkg
 {
     Json::IDeserializer<VersionT>& get_versiont_deserializer_instance() { return VersionTDeserializer::instance; }
-
-    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
-                                                                                StringView baseline_name,
-                                                                                const fs::path& baseline_file_path)
-    {
-        if (!fs.exists(baseline_file_path))
-        {
-            return Strings::format("Couldn't find `%s`", fs::u8string(baseline_file_path));
-        }
-
-        auto value = Json::parse_file(VCPKG_LINE_INFO, fs, baseline_file_path);
-        if (!value.first.is_object())
-        {
-            return Strings::format("Error: `%s` does not have a top-level object.", fs::u8string(baseline_file_path));
-        }
-
-        const auto& obj = value.first.object();
-        auto baseline_value = obj.get(baseline_name);
-        if (!baseline_value)
-        {
-            return Strings::format(
-                "Error: `%s` does not contain the baseline \"%s\"", fs::u8string(baseline_file_path), baseline_name);
-        }
-
-        Json::Reader r;
-        std::map<std::string, VersionT, std::less<>> result;
-        r.visit_in_key(*baseline_value, baseline_name, result, BaselineDeserializer::instance);
-        if (!r.errors().empty())
-        {
-            return Strings::format(
-                "Error: failed to parse `%s`:\n%s", fs::u8string(baseline_file_path), Strings::join("\n", r.errors()));
-        }
-        return result;
-    }
-
-    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
-                                                               StringView port_name,
-                                                               const fs::path& versions_file_path)
-    {
-        (void)port_name;
-        if (!fs.exists(versions_file_path))
-        {
-            return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path));
-        }
-
-        auto versions_json = Json::parse_file(VCPKG_LINE_INFO, fs, versions_file_path);
-        if (!versions_json.first.is_object())
-        {
-            return Strings::format("Error: `%s` does not have a top level object.", fs::u8string(versions_file_path));
-        }
-
-        const auto& versions_object = versions_json.first.object();
-        auto maybe_versions_array = versions_object.get("versions");
-        if (!maybe_versions_array || !maybe_versions_array->is_array())
-        {
-            return Strings::format("Error: `%s` does not contain a versions array.", fs::u8string(versions_file_path));
-        }
-
-        std::vector<VersionDbEntry> db_entries;
-        // Avoid warning treated as error.
-        if (maybe_versions_array != nullptr)
-        {
-            Json::Reader r;
-            r.visit_in_key(*maybe_versions_array, "versions", db_entries, VersionDbEntryArrayDeserializer::instance);
-        }
-        return db_entries;
-    }
 }

toolsrc/src/vcpkg/versions.cpp

@@ -244,4 +244,4 @@ namespace vcpkg::Versions
 
         Checks::unreachable(VCPKG_LINE_INFO);
     }
-}
+}