From c898283a41af23135c048c50836f2bc2eccea819 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Mon, 21 Dec 2020 15:40:21 -0800 Subject: [PATCH] [vcpkg registries] support versions (#15114) * [vcpkg registries] support versions This PR merges the Registries changes and the versioning changes, so that one can use both at the same time. There is one major difference between this PR and the RFC (#13590), which is that instead of version files looking like: ```json [ ... ] ``` version files look like: ``` { "versions": [ ... ] } ``` this is to support interop between this PR and existing demos and the like; fixing this, along with perhaps renaming `port_versions` to `port-versions` should be done after this is merged, should be a trivial change. --- .../end-to-end-tests-dir/create.ps1 | 4 +- .../integrate-install.ps1 | 14 +- .../end-to-end-tests-dir/registries.ps1 | 13 + .../end-to-end-tests-dir/spaces.ps1 | 4 +- .../end-to-end-tests-prelude.ps1 | 21 +- scripts/azure-pipelines/end-to-end-tests.ps1 | 2 +- .../vcpkg-uses-test-cmake/portfile.cmake | 0 .../vcpkg-uses-test-cmake/vcpkg.json | 0 .../portfile.cmake | 0 .../vcpkg.json | 0 scripts/e2e_ports/port_versions/baseline.json | 3 + .../v-/vcpkg-internal-e2e-test-port.json | 8 + .../portfile.cmake | 1 + .../vcpkg-internal-e2e-test-port/vcpkg.json | 4 + toolsrc/include/vcpkg/base/files.h | 6 + toolsrc/include/vcpkg/base/jsonreader.h | 2 +- toolsrc/include/vcpkg/configuration.h | 9 + .../include/vcpkg/configurationdeserializer.h | 66 - toolsrc/include/vcpkg/fwd/registries.h | 3 +- toolsrc/include/vcpkg/portfileprovider.h | 13 +- toolsrc/include/vcpkg/registries.h | 46 +- toolsrc/include/vcpkg/sourceparagraph.h | 4 +- toolsrc/include/vcpkg/vcpkgcmdarguments.h | 2 + toolsrc/include/vcpkg/vcpkgpaths.h | 3 +- toolsrc/include/vcpkg/versiondeserializers.h | 15 - toolsrc/src/vcpkg-test/dependencies.cpp | 5 +- toolsrc/src/vcpkg-test/registries.cpp | 132 ++ toolsrc/src/vcpkg/base/files.cpp | 8 +- toolsrc/src/vcpkg/configuration.cpp | 91 +- toolsrc/src/vcpkg/install.cpp | 18 +- toolsrc/src/vcpkg/paragraphs.cpp | 28 +- toolsrc/src/vcpkg/portfileprovider.cpp | 346 ++---- toolsrc/src/vcpkg/registries.cpp | 1080 +++++++++++++---- toolsrc/src/vcpkg/vcpkgcmdarguments.cpp | 1 + toolsrc/src/vcpkg/vcpkgpaths.cpp | 18 +- toolsrc/src/vcpkg/versiondeserializers.cpp | 136 --- toolsrc/src/vcpkg/versions.cpp | 2 +- 37 files changed, 1290 insertions(+), 818 deletions(-) create mode 100644 scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1 rename scripts/e2e_ports/{ => overlays}/vcpkg-uses-test-cmake/portfile.cmake (100%) rename scripts/e2e_ports/{ => overlays}/vcpkg-uses-test-cmake/vcpkg.json (100%) rename scripts/e2e_ports/{ => overlays}/vcpkg-uses-vcpkg-common-functions/portfile.cmake (100%) rename scripts/e2e_ports/{ => overlays}/vcpkg-uses-vcpkg-common-functions/vcpkg.json (100%) create mode 100644 scripts/e2e_ports/port_versions/baseline.json create mode 100644 scripts/e2e_ports/port_versions/v-/vcpkg-internal-e2e-test-port.json create mode 100644 scripts/e2e_ports/vcpkg-internal-e2e-test-port/portfile.cmake create mode 100644 scripts/e2e_ports/vcpkg-internal-e2e-test-port/vcpkg.json delete mode 100644 toolsrc/include/vcpkg/configurationdeserializer.h create mode 100644 toolsrc/src/vcpkg-test/registries.cpp diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/create.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/create.ps1 index 53686fe509..9d59da5396 100644 --- a/scripts/azure-pipelines/end-to-end-tests-dir/create.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests-dir/create.ps1 @@ -1,8 +1,8 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 # Test vcpkg create -$CurrentTest = "create zlib" -Write-Host $CurrentTest +$Script:CurrentTest = "create zlib" +Write-Host $Script:CurrentTest ./vcpkg --x-builtin-ports-root=$TestingRoot/ports create zlib https://github.com/madler/zlib/archive/v1.2.11.tar.gz zlib-1.2.11.tar.gz Throw-IfFailed diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/integrate-install.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/integrate-install.ps1 index 3876977514..38362ba8cb 100644 --- a/scripts/azure-pipelines/end-to-end-tests-dir/integrate-install.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests-dir/integrate-install.ps1 @@ -2,24 +2,24 @@ if (-not $IsLinux -and -not $IsMacOS) { . $PSScriptRoot/../end-to-end-tests-prelude.ps1 # Test msbuild props and targets - $CurrentTest = "zlib:x86-windows-static msbuild scripts\testing\integrate-install\..." - Write-Host $CurrentTest + $Script:CurrentTest = "zlib:x86-windows-static msbuild scripts\testing\integrate-install\..." + Write-Host $Script:CurrentTest ./vcpkg $commonArgs install zlib:x86-windows-static --x-binarysource=clear Throw-IfFailed foreach ($project in @("VcpkgTriplet", "VcpkgTriplet2", "VcpkgUseStatic", "VcpkgUseStatic2")) { - $CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj" + $Script:CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj" ./vcpkg $commonArgs env "msbuild scripts\testing\integrate-install\$project.vcxproj /p:VcpkgRoot=$TestingRoot /p:IntDir=$TestingRoot\int\ /p:OutDir=$TestingRoot\out\ " Throw-IfFailed Remove-Item -Recurse -Force $TestingRoot\int Remove-Item -Recurse -Force $TestingRoot\out } - $CurrentTest = "zlib:x86-windows msbuild scripts\testing\integrate-install\..." - Write-Host $CurrentTest + $Script:CurrentTest = "zlib:x86-windows msbuild scripts\testing\integrate-install\..." + Write-Host $Script:CurrentTest ./vcpkg $commonArgs install zlib:x86-windows --x-binarysource=clear Throw-IfFailed foreach ($project in @("Project1", "NoProps")) { - $CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj" - Write-Host $CurrentTest + $Script:CurrentTest = "msbuild scripts\testing\integrate-install\$project.vcxproj" + Write-Host $Script:CurrentTest ./vcpkg $commonArgs env "msbuild scripts\testing\integrate-install\$project.vcxproj /p:VcpkgRoot=$TestingRoot /p:IntDir=$TestingRoot\int\ /p:OutDir=$TestingRoot\out\ " Throw-IfFailed Remove-Item -Recurse -Force $TestingRoot\int diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1 new file mode 100644 index 0000000000..b985a61f56 --- /dev/null +++ b/scripts/azure-pipelines/end-to-end-tests-dir/registries.ps1 @@ -0,0 +1,13 @@ +. "$PSScriptRoot/../end-to-end-tests-prelude.ps1" + + +$commonArgs += @("--x-builtin-port-versions-dir=$PSScriptRoot/../../e2e_ports/port_versions") + +Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port' +Throw-IfNotFailed + +Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port' +Throw-IfFailed + +Run-Vcpkg install @commonArgs --feature-flags=registries 'zlib' +Throw-IfFailed diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/spaces.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/spaces.ps1 index 49796c1b06..b7aaf6462c 100644 --- a/scripts/azure-pipelines/end-to-end-tests-dir/spaces.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests-dir/spaces.ps1 @@ -1,8 +1,8 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 ##### Test spaces in the path -$CurrentTest = "zlib with spaces in path" -Write-Host $CurrentTest +$Script:CurrentTest = "zlib with spaces in path" +Write-Host $Script:CurrentTest ./vcpkg install zlib "--triplet" $Triplet ` "--no-binarycaching" ` "--x-buildtrees-root=$TestingRoot/build Trees" ` diff --git a/scripts/azure-pipelines/end-to-end-tests-prelude.ps1 b/scripts/azure-pipelines/end-to-end-tests-prelude.ps1 index 057df9fa66..0c675e9bf2 100644 --- a/scripts/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -12,9 +12,9 @@ $commonArgs = @( "--x-buildtrees-root=$buildtreesRoot", "--x-install-root=$installRoot", "--x-packages-root=$packagesRoot", - "--overlay-ports=scripts/e2e_ports" + "--overlay-ports=scripts/e2e_ports/overlays" ) -$CurrentTest = 'unassigned' +$Script:CurrentTest = 'unassigned' function Refresh-TestRoot { Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue @@ -28,7 +28,7 @@ function Require-FileExists { [string]$File ) if (-Not (Test-Path $File)) { - throw "'$CurrentTest' failed to create file '$File'" + throw "'$Script:CurrentTest' failed to create file '$File'" } } @@ -38,26 +38,29 @@ function Require-FileNotExists { [string]$File ) if (Test-Path $File) { - throw "'$CurrentTest' should not have created file '$File'" + throw "'$Script:CurrentTest' should not have created file '$File'" } } function Throw-IfFailed { if ($LASTEXITCODE -ne 0) { - throw "'$CurrentTest' had a step with a nonzero exit code" + throw "'$Script:CurrentTest' had a step with a nonzero exit code" } } function Throw-IfNotFailed { if ($LASTEXITCODE -eq 0) { - throw "'$CurrentTest' had a step with an unexpectedly zero exit code" + throw "'$Script:CurrentTest' had a step with an unexpectedly zero exit code" } } function Run-Vcpkg { - param([string[]]$TestArgs) - $CurrentTest = "./vcpkg $($testArgs -join ' ')" - Write-Host $CurrentTest + Param( + [Parameter(ValueFromRemainingArguments)] + [string[]]$TestArgs + ) + $Script:CurrentTest = "./vcpkg $($testArgs -join ' ')" + Write-Host $Script:CurrentTest ./vcpkg @testArgs } diff --git a/scripts/azure-pipelines/end-to-end-tests.ps1 b/scripts/azure-pipelines/end-to-end-tests.ps1 index f6b5c2e6bf..029120d546 100644 --- a/scripts/azure-pipelines/end-to-end-tests.ps1 +++ b/scripts/azure-pipelines/end-to-end-tests.ps1 @@ -34,7 +34,7 @@ $ErrorActionPreference = "Stop" $AllTests = Get-ChildItem $PSScriptRoot/end-to-end-tests-dir/*.ps1 if ($Filter -ne $Null) { - $AllTests = $AllTests | ? { $_ -match $Filter } + $AllTests = $AllTests | ? { $_.Name -match $Filter } } $n = 1 $m = $AllTests.Count diff --git a/scripts/e2e_ports/vcpkg-uses-test-cmake/portfile.cmake b/scripts/e2e_ports/overlays/vcpkg-uses-test-cmake/portfile.cmake similarity index 100% rename from scripts/e2e_ports/vcpkg-uses-test-cmake/portfile.cmake rename to scripts/e2e_ports/overlays/vcpkg-uses-test-cmake/portfile.cmake diff --git a/scripts/e2e_ports/vcpkg-uses-test-cmake/vcpkg.json b/scripts/e2e_ports/overlays/vcpkg-uses-test-cmake/vcpkg.json similarity index 100% rename from scripts/e2e_ports/vcpkg-uses-test-cmake/vcpkg.json rename to scripts/e2e_ports/overlays/vcpkg-uses-test-cmake/vcpkg.json diff --git a/scripts/e2e_ports/vcpkg-uses-vcpkg-common-functions/portfile.cmake b/scripts/e2e_ports/overlays/vcpkg-uses-vcpkg-common-functions/portfile.cmake similarity index 100% rename from scripts/e2e_ports/vcpkg-uses-vcpkg-common-functions/portfile.cmake rename to scripts/e2e_ports/overlays/vcpkg-uses-vcpkg-common-functions/portfile.cmake diff --git a/scripts/e2e_ports/vcpkg-uses-vcpkg-common-functions/vcpkg.json b/scripts/e2e_ports/overlays/vcpkg-uses-vcpkg-common-functions/vcpkg.json similarity index 100% rename from scripts/e2e_ports/vcpkg-uses-vcpkg-common-functions/vcpkg.json rename to scripts/e2e_ports/overlays/vcpkg-uses-vcpkg-common-functions/vcpkg.json diff --git a/scripts/e2e_ports/port_versions/baseline.json b/scripts/e2e_ports/port_versions/baseline.json new file mode 100644 index 0000000000..953e8752d5 --- /dev/null +++ b/scripts/e2e_ports/port_versions/baseline.json @@ -0,0 +1,3 @@ +{ + "vcpkg-internal-e2e-test-port": { "version-string": "1.0.0" } +} diff --git a/scripts/e2e_ports/port_versions/v-/vcpkg-internal-e2e-test-port.json b/scripts/e2e_ports/port_versions/v-/vcpkg-internal-e2e-test-port.json new file mode 100644 index 0000000000..ce7698ebb8 --- /dev/null +++ b/scripts/e2e_ports/port_versions/v-/vcpkg-internal-e2e-test-port.json @@ -0,0 +1,8 @@ +{ + "versions": [ + { + "version-string": "1.0.0", + "git-tree": "1dc3e42a3c0cafe2884d379af4399273238b986e" + } + ] +} diff --git a/scripts/e2e_ports/vcpkg-internal-e2e-test-port/portfile.cmake b/scripts/e2e_ports/vcpkg-internal-e2e-test-port/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/scripts/e2e_ports/vcpkg-internal-e2e-test-port/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/scripts/e2e_ports/vcpkg-internal-e2e-test-port/vcpkg.json b/scripts/e2e_ports/vcpkg-internal-e2e-test-port/vcpkg.json new file mode 100644 index 0000000000..a25da6d236 --- /dev/null +++ b/scripts/e2e_ports/vcpkg-internal-e2e-test-port/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "vcpkg-internal-e2e-test-port", + "version-string": "1.0.0" +} diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index a6cc06c199..69ec41fe31 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -239,6 +239,12 @@ namespace vcpkg::Files virtual void current_path(const fs::path& path, std::error_code&) = 0; void current_path(const fs::path& path, LineInfo li); + // if the path does not exist, then (try_|)take_exclusive_file_lock attempts to create the file + // (but not any path members above the file itself) + // in other words, if `/a/b` is a directory, and you're attempting to lock `/a/b/c`, + // then these lock functions create `/a/b/c` if it doesn't exist; + // however, if `/a/b` doesn't exist, then the functions will fail. + // waits forever for the file lock virtual fs::SystemHandle take_exclusive_file_lock(const fs::path& path, std::error_code&) = 0; // waits, at most, 1.5 seconds, for the file lock diff --git a/toolsrc/include/vcpkg/base/jsonreader.h b/toolsrc/include/vcpkg/base/jsonreader.h index bea60e7ce5..da086fa028 100644 --- a/toolsrc/include/vcpkg/base/jsonreader.h +++ b/toolsrc/include/vcpkg/base/jsonreader.h @@ -75,13 +75,13 @@ namespace vcpkg::Json }; std::vector m_path; + public: // checks that an object doesn't contain any fields which both: // * don't start with a `$` // * are not in `valid_fields` // if known_fields.empty(), then it's treated as if all field names are valid void check_for_unexpected_fields(const Object& obj, View valid_fields, StringView type_name); - public: template void required_object_field( StringView type, const Object& obj, StringView key, Type& place, IDeserializer& visitor) diff --git a/toolsrc/include/vcpkg/configuration.h b/toolsrc/include/vcpkg/configuration.h index 4cba88fe5e..6d6a9b1f44 100644 --- a/toolsrc/include/vcpkg/configuration.h +++ b/toolsrc/include/vcpkg/configuration.h @@ -1,6 +1,10 @@ #pragma once #include +#include + +#include +#include #include @@ -12,5 +16,10 @@ namespace vcpkg // `registries` and `default_registry`. The fall back logic is // taken care of in RegistrySet. RegistrySet registry_set; + + void validate_feature_flags(const FeatureFlagSettings& flags); }; + + std::unique_ptr> make_configuration_deserializer( + const fs::path& config_directory); } diff --git a/toolsrc/include/vcpkg/configurationdeserializer.h b/toolsrc/include/vcpkg/configurationdeserializer.h deleted file mode 100644 index d6b2393567..0000000000 --- a/toolsrc/include/vcpkg/configurationdeserializer.h +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include -#include -#include -#include - -#include -#include - -namespace vcpkg -{ - struct RegistryImplDeserializer : Json::IDeserializer> - { - constexpr static StringLiteral KIND = "kind"; - constexpr static StringLiteral PATH = "path"; - - constexpr static StringLiteral KIND_BUILTIN = "builtin"; - constexpr static StringLiteral KIND_FILESYSTEM = "filesystem"; - - virtual StringView type_name() const override; - virtual View valid_fields() const override; - - virtual Optional> visit_null(Json::Reader&) override; - virtual Optional> visit_object(Json::Reader&, const Json::Object&) override; - - static RegistryImplDeserializer instance; - }; - - struct RegistryDeserializer final : Json::IDeserializer - { - constexpr static StringLiteral PACKAGES = "packages"; - - virtual StringView type_name() const override; - virtual View valid_fields() const override; - - virtual Optional visit_object(Json::Reader&, const Json::Object&) override; - }; - - struct ConfigurationDeserializer final : Json::IDeserializer - { - virtual StringView type_name() const override { return "a configuration object"; } - - constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry"; - constexpr static StringLiteral REGISTRIES = "registries"; - virtual View valid_fields() const override - { - constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES}; - return t; - } - - virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) override; - - ConfigurationDeserializer(const VcpkgCmdArguments& args); - - private: - bool print_json; - - bool registries_enabled; - }; -} diff --git a/toolsrc/include/vcpkg/fwd/registries.h b/toolsrc/include/vcpkg/fwd/registries.h index 7352c429d1..73783cc6bb 100644 --- a/toolsrc/include/vcpkg/fwd/registries.h +++ b/toolsrc/include/vcpkg/fwd/registries.h @@ -2,7 +2,8 @@ namespace vcpkg { - struct RegistryImpl; + struct RegistryEntry; + struct RegistryImplementation; struct Registry; struct RegistrySet; } diff --git a/toolsrc/include/vcpkg/portfileprovider.h b/toolsrc/include/vcpkg/portfileprovider.h index b93c58e8c4..fde28b0dfe 100644 --- a/toolsrc/include/vcpkg/portfileprovider.h +++ b/toolsrc/include/vcpkg/portfileprovider.h @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -41,21 +42,19 @@ namespace vcpkg::PortFileProvider struct IVersionedPortfileProvider { - virtual const std::vector& get_port_versions(StringView port_name) const = 0; + virtual View get_port_versions(StringView port_name) const = 0; virtual ~IVersionedPortfileProvider() = default; virtual ExpectedS get_control_file( - const vcpkg::Versions::VersionSpec& version_spec) const = 0; + const Versions::VersionSpec& version_spec) const = 0; }; struct IBaselineProvider { - virtual ~IBaselineProvider() = default; - virtual Optional get_baseline_version(StringView port_name) const = 0; + virtual ~IBaselineProvider() = default; }; - std::unique_ptr make_baseline_provider(const vcpkg::VcpkgPaths& paths); - std::unique_ptr make_baseline_provider(const vcpkg::VcpkgPaths& paths, StringView baseline); - std::unique_ptr make_versioned_portfile_provider(const vcpkg::VcpkgPaths& paths); + std::unique_ptr make_baseline_provider(const VcpkgPaths&); + std::unique_ptr make_versioned_portfile_provider(const VcpkgPaths&); } diff --git a/toolsrc/include/vcpkg/registries.h b/toolsrc/include/vcpkg/registries.h index 1f8c0f393b..006bc789d1 100644 --- a/toolsrc/include/vcpkg/registries.h +++ b/toolsrc/include/vcpkg/registries.h @@ -1,8 +1,11 @@ #pragma once +#include +#include #include #include +#include #include #include @@ -17,40 +20,45 @@ namespace vcpkg { struct RegistryEntry { - // returns fs::path() if version doesn't exist - virtual fs::path get_port_directory(const VcpkgPaths& paths, const VersionT& version) const = 0; + virtual View get_port_versions() const = 0; + + virtual ExpectedS get_path_to_version(const VcpkgPaths& paths, const VersionT& version) const = 0; virtual ~RegistryEntry() = default; }; - struct RegistryImpl + struct RegistryImplementation { // returns nullptr if the port doesn't exist virtual std::unique_ptr get_port_entry(const VcpkgPaths& paths, StringView port_name) const = 0; + // appends the names of the ports to the out parameter + // may result in duplicated port names; make sure to Util::sort_unique_erase at the end virtual void get_all_port_names(std::vector& port_names, const VcpkgPaths& paths) const = 0; virtual Optional get_baseline_version(const VcpkgPaths& paths, StringView port_name) const = 0; - virtual ~RegistryImpl() = default; + virtual ~RegistryImplementation() = default; }; struct Registry { // requires: static_cast(implementation) - Registry(std::vector&& packages, std::unique_ptr&& implementation); + Registry(std::vector&& packages, std::unique_ptr&& implementation); Registry(std::vector&&, std::nullptr_t) = delete; // always ordered lexicographically View packages() const { return packages_; } - const RegistryImpl& implementation() const { return *implementation_; } + const RegistryImplementation& implementation() const { return *implementation_; } - static std::unique_ptr builtin_registry(); + static std::unique_ptr builtin_registry(std::string&& baseline = {}); + + friend RegistrySet; // for experimental_set_builtin_registry_baseline private: std::vector packages_; - std::unique_ptr implementation_; + std::unique_ptr implementation_; }; // this type implements the registry fall back logic from the registries RFC: @@ -65,20 +73,34 @@ namespace vcpkg // finds the correct registry for the port name // Returns the null pointer if there is no registry set up for that name - const RegistryImpl* registry_for_port(StringView port_name) const; + const RegistryImplementation* registry_for_port(StringView port_name) const; + Optional baseline_for_port(const VcpkgPaths& paths, StringView port_name) const; View registries() const { return registries_; } - const RegistryImpl* default_registry() const { return default_registry_.get(); } + const RegistryImplementation* default_registry() const { return default_registry_.get(); } // TODO: figure out how to get this to return an error (or maybe it should be a warning?) void add_registry(Registry&& r); - void set_default_registry(std::unique_ptr&& r); + void set_default_registry(std::unique_ptr&& r); void set_default_registry(std::nullptr_t r); + // this exists in order to allow versioning and registries to be developed and tested separately + void experimental_set_builtin_registry_baseline(StringView baseline) const; + + // returns whether the registry set has any modifications to the default + // (i.e., whether `default_registry` was set, or `registries` had any entries) + // for checking against the registry feature flag. + bool has_modifications() const; + private: - std::unique_ptr default_registry_; + std::unique_ptr default_registry_; std::vector registries_; }; + std::unique_ptr>> + get_registry_implementation_deserializer(const fs::path& configuration_directory); + + std::unique_ptr>> get_registry_array_deserializer( + const fs::path& configuration_directory); } diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index ee07826bba..50417b8adc 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -116,8 +116,8 @@ namespace vcpkg Json::Object serialize_debug_manifest(const SourceControlFile& scf); /// - /// Full metadata of a package: core and other features. As well as the location the SourceControlFile was - /// loaded from. + /// Full metadata of a package: core and other features, + /// as well as the port directory the SourceControlFile was loaded from /// struct SourceControlFileLocation { diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index f18f4843cb..2677e6811b 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -134,6 +134,8 @@ namespace vcpkg std::unique_ptr scripts_root_dir; constexpr static StringLiteral BUILTIN_PORTS_ROOT_DIR_ARG = "x-builtin-ports-root"; std::unique_ptr builtin_ports_root_dir; + constexpr static StringLiteral BUILTIN_PORT_VERSIONS_DIR_ARG = "x-builtin-port-versions-dir"; + std::unique_ptr builtin_port_versions_dir; constexpr static StringLiteral DEFAULT_VISUAL_STUDIO_PATH_ENV = "VCPKG_VISUAL_STUDIO_PATH"; std::unique_ptr default_visual_studio_path; diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index e71f28b0a6..1f562c1517 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -92,6 +92,7 @@ namespace vcpkg fs::path scripts; fs::path prefab; fs::path builtin_ports; + fs::path builtin_port_versions; fs::path tools; fs::path buildsystems; @@ -116,7 +117,7 @@ namespace vcpkg const fs::path& get_tool_exe(const std::string& tool) const; const std::string& get_tool_version(const std::string& tool) const; - // Git manipulation + // Git manipulation in the vcpkg directory fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const; fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const; ExpectedS git_show(const std::string& treeish, const fs::path& dot_git_dir) const; diff --git a/toolsrc/include/vcpkg/versiondeserializers.h b/toolsrc/include/vcpkg/versiondeserializers.h index 2efe340d74..02696d39b9 100644 --- a/toolsrc/include/vcpkg/versiondeserializers.h +++ b/toolsrc/include/vcpkg/versiondeserializers.h @@ -10,13 +10,6 @@ namespace vcpkg { - struct VersionDbEntry - { - VersionT version; - Versions::Scheme scheme = Versions::Scheme::String; - std::string git_tree; - }; - Json::IDeserializer& get_versiont_deserializer_instance(); std::unique_ptr> make_version_deserializer(StringLiteral type_name); @@ -44,12 +37,4 @@ namespace vcpkg const std::string& version, int port_version, bool always_emit_port_version = false); - - ExpectedS>> parse_baseline_file(Files::Filesystem& fs, - StringView baseline_name, - const fs::path& baseline_file_path); - - ExpectedS> parse_versions_file(Files::Filesystem& fs, - StringView port_name, - const fs::path& versions_file_path); } diff --git a/toolsrc/src/vcpkg-test/dependencies.cpp b/toolsrc/src/vcpkg-test/dependencies.cpp index d1834b13df..07f9d0852c 100644 --- a/toolsrc/src/vcpkg-test/dependencies.cpp +++ b/toolsrc/src/vcpkg-test/dependencies.cpp @@ -54,10 +54,7 @@ struct MockVersionedPortfileProvider : PortFileProvider::IVersionedPortfileProvi return it2->second; } - virtual const std::vector& get_port_versions(StringView) const override - { - Checks::unreachable(VCPKG_LINE_INFO); - } + virtual View get_port_versions(StringView) const override { Checks::unreachable(VCPKG_LINE_INFO); } SourceControlFileLocation& emplace(std::string&& name, Versions::Version&& version, diff --git a/toolsrc/src/vcpkg-test/registries.cpp b/toolsrc/src/vcpkg-test/registries.cpp new file mode 100644 index 0000000000..1c839409d9 --- /dev/null +++ b/toolsrc/src/vcpkg-test/registries.cpp @@ -0,0 +1,132 @@ +#include + +#include + +#include + +using namespace vcpkg; + +namespace +{ + struct TestRegistryImplementation final : RegistryImplementation + { + std::unique_ptr get_port_entry(const VcpkgPaths&, StringView) const override { return nullptr; } + + void get_all_port_names(std::vector&, const VcpkgPaths&) const override { } + + Optional get_baseline_version(const VcpkgPaths&, StringView) const override { return nullopt; } + + int number; + + TestRegistryImplementation(int n) : number(n) { } + }; + + Registry make_registry(int n, std::vector&& port_names) + { + return {std::move(port_names), std::make_unique(n)}; + } + + int get_tri_num(const RegistryImplementation& r) + { + if (auto tri = dynamic_cast(&r)) + { + return tri->number; + } + else + { + return -1; + } + } + + // test functions which parse string literals, so no concerns about failure + Json::Value parse_json(StringView sv) { return Json::parse(sv).value_or_exit(VCPKG_LINE_INFO).first; } +} + +TEST_CASE ("registry_set_selects_registry", "[registries]") +{ + RegistrySet set; + set.set_default_registry(std::make_unique(0)); + + set.add_registry(make_registry(1, {"p1", "q1", "r1"})); + set.add_registry(make_registry(2, {"p2", "q2", "r2"})); + + auto reg = set.registry_for_port("p1"); + REQUIRE(reg); + CHECK(get_tri_num(*reg) == 1); + reg = set.registry_for_port("r2"); + REQUIRE(reg); + CHECK(get_tri_num(*reg) == 2); + reg = set.registry_for_port("a"); + REQUIRE(reg); + CHECK(get_tri_num(*reg) == 0); + + set.set_default_registry(nullptr); + + reg = set.registry_for_port("q1"); + REQUIRE(reg); + CHECK(get_tri_num(*reg) == 1); + reg = set.registry_for_port("p2"); + REQUIRE(reg); + CHECK(get_tri_num(*reg) == 2); + reg = set.registry_for_port("a"); + CHECK_FALSE(reg); +} + +TEST_CASE ("registry_parsing", "[registries]") +{ + Json::Reader r; + auto registry_impl_des = get_registry_implementation_deserializer({}); + + auto test_json = parse_json(R"json( +{ + "kind": "builtin" +} + )json"); + auto registry_impl = r.visit(test_json, *registry_impl_des); + REQUIRE(registry_impl); + CHECK(*registry_impl.get()); + CHECK(r.errors().empty()); + + test_json = parse_json(R"json( +{ + "kind": "builtin", + "baseline": "hi" +} + )json"); + registry_impl = r.visit(test_json, *registry_impl_des); + REQUIRE(registry_impl); + CHECK(*registry_impl.get()); + CHECK(r.errors().empty()); + + test_json = parse_json(R"json( +{ + "kind": "builtin", + "path": "a/b" +} + )json"); + registry_impl = r.visit(test_json, *registry_impl_des); + CHECK_FALSE(r.errors().empty()); + r.errors().clear(); + + test_json = parse_json(R"json( +{ + "kind": "filesystem", + "path": "a/b/c" +} + )json"); + registry_impl = r.visit(test_json, *registry_impl_des); + REQUIRE(registry_impl); + CHECK(*registry_impl.get()); + CHECK(r.errors().empty()); + + test_json = parse_json(R"json( +{ + "kind": "filesystem", + "path": "/a/b/c" +} + )json"); + registry_impl = r.visit(test_json, *registry_impl_des); + REQUIRE(registry_impl); + CHECK(*registry_impl.get()); + CHECK(r.errors().empty()); +} diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index b0717f2d6c..ce68741d5e 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -1216,7 +1216,7 @@ namespace vcpkg::Files { } -#if defined(WIN32) +#if defined(_WIN32) void assign_busy_error(std::error_code& ec) { ec.assign(ERROR_BUSY, std::system_category()); } bool operator()(std::error_code& ec) @@ -1242,7 +1242,7 @@ namespace vcpkg::Files res.system_handle = reinterpret_cast(handle); return true; } -#else // ^^^ WIN32 / !WIN32 vvv +#else // ^^^ _WIN32 / !_WIN32 vvv int fd = -1; void assign_busy_error(std::error_code& ec) { ec.assign(EBUSY, std::generic_category()); } @@ -1252,7 +1252,7 @@ namespace vcpkg::Files ec.clear(); if (fd == -1) { - fd = ::open(native.c_str(), 0); + fd = ::open(native.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd < 0) { ec.assign(errno, std::generic_category()); @@ -1335,7 +1335,7 @@ namespace vcpkg::Files virtual void unlock_file_lock(fs::SystemHandle handle, std::error_code& ec) override { -#if defined(WIN32) +#if defined(_WIN32) if (CloseHandle(reinterpret_cast(handle.system_handle)) == 0) { ec.assign(GetLastError(), std::system_category()); diff --git a/toolsrc/src/vcpkg/configuration.cpp b/toolsrc/src/vcpkg/configuration.cpp index f062af51d3..3fa02876ff 100644 --- a/toolsrc/src/vcpkg/configuration.cpp +++ b/toolsrc/src/vcpkg/configuration.cpp @@ -2,66 +2,83 @@ #include #include -#include #include -namespace vcpkg +namespace { + using namespace vcpkg; + + struct ConfigurationDeserializer final : Json::IDeserializer + { + virtual StringView type_name() const override { return "a configuration object"; } + + constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry"; + constexpr static StringLiteral REGISTRIES = "registries"; + virtual View valid_fields() const override + { + constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES}; + return t; + } + + virtual Optional visit_object(Json::Reader& r, const Json::Object& obj) override; + + ConfigurationDeserializer(const fs::path& configuration_directory); + + private: + fs::path configuration_directory; + }; + + constexpr StringLiteral ConfigurationDeserializer::DEFAULT_REGISTRY; + constexpr StringLiteral ConfigurationDeserializer::REGISTRIES; + Optional ConfigurationDeserializer::visit_object(Json::Reader& r, const Json::Object& obj) { RegistrySet registries; - bool registries_feature_flags_warning = false; + auto impl_des = get_registry_implementation_deserializer(configuration_directory); + std::unique_ptr default_registry; + if (r.optional_object_field(obj, DEFAULT_REGISTRY, default_registry, *impl_des)) { - std::unique_ptr default_registry; - if (r.optional_object_field(obj, DEFAULT_REGISTRY, default_registry, RegistryImplDeserializer::instance)) - { - if (!registries_enabled) - { - registries_feature_flags_warning = true; - } - else - { - registries.set_default_registry(std::move(default_registry)); - } - } + registries.set_default_registry(std::move(default_registry)); } - static Json::ArrayDeserializer array_of_registries{"an array of registries"}; - + auto reg_des = get_registry_array_deserializer(configuration_directory); std::vector regs; - r.optional_object_field(obj, REGISTRIES, regs, array_of_registries); - - if (!regs.empty() && !registries_enabled) - { - registries_feature_flags_warning = true; - regs.clear(); - } + r.optional_object_field(obj, REGISTRIES, regs, *reg_des); for (Registry& reg : regs) { registries.add_registry(std::move(reg)); } - if (registries_feature_flags_warning && !print_json) + return Configuration{std::move(registries)}; + } + + ConfigurationDeserializer::ConfigurationDeserializer(const fs::path& configuration_directory) + : configuration_directory(configuration_directory) + { + } + +} + +std::unique_ptr> vcpkg::make_configuration_deserializer( + const fs::path& config_directory) +{ + return std::make_unique(config_directory); +} + +namespace vcpkg +{ + void Configuration::validate_feature_flags(const FeatureFlagSettings& flags) + { + if (!flags.registries && registry_set.has_modifications()) { System::printf(System::Color::warning, "Warning: configuration specified the \"registries\" or \"default-registries\" field, but " "the %s feature flag was not enabled.\n", VcpkgCmdArguments::REGISTRIES_FEATURE); + registry_set = RegistrySet(); } - - return Configuration{std::move(registries)}; } - - constexpr StringLiteral ConfigurationDeserializer::DEFAULT_REGISTRY; - constexpr StringLiteral ConfigurationDeserializer::REGISTRIES; - - ConfigurationDeserializer::ConfigurationDeserializer(const VcpkgCmdArguments& args) - { - registries_enabled = args.registries_enabled(); - print_json = args.output_json(); - } - } diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index a07fb431c3..50f07c7326 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include namespace vcpkg::Install { @@ -841,16 +843,12 @@ namespace vcpkg::Install if (args.versions_enabled()) { auto verprovider = PortFileProvider::make_versioned_portfile_provider(paths); - auto baseprovider = [&]() -> std::unique_ptr { - if (auto p_baseline = manifest_scf.core_paragraph->extra_info.get("$x-default-baseline")) - { - return PortFileProvider::make_baseline_provider(paths, p_baseline->string().to_string()); - } - else - { - return PortFileProvider::make_baseline_provider(paths); - } - }(); + auto baseprovider = PortFileProvider::make_baseline_provider(paths); + if (auto p_baseline = manifest_scf.core_paragraph->extra_info.get("$x-default-baseline")) + { + paths.get_configuration().registry_set.experimental_set_builtin_registry_baseline( + p_baseline->string()); + } auto install_plan = Dependencies::create_versioned_install_plan(*verprovider, diff --git a/toolsrc/src/vcpkg/paragraphs.cpp b/toolsrc/src/vcpkg/paragraphs.cpp index 421f28b002..32103c76b9 100644 --- a/toolsrc/src/vcpkg/paragraphs.cpp +++ b/toolsrc/src/vcpkg/paragraphs.cpp @@ -357,14 +357,23 @@ namespace vcpkg::Paragraphs return res; } - ExpectedS> pghs = get_paragraphs(fs, path_to_control); - if (auto vector_pghs = pghs.get()) + + if (fs.exists(path_to_control)) { - return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs)); + ExpectedS> pghs = get_paragraphs(fs, path_to_control); + if (auto vector_pghs = pghs.get()) + { + return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs)); + } + auto error_info = std::make_unique(); + error_info->name = fs::u8string(path.filename()); + error_info->error = pghs.error(); + return error_info; } + auto error_info = std::make_unique(); error_info->name = fs::u8string(path.filename()); - error_info->error = pghs.error(); + error_info->error = "Failed to find either a CONTROL file or vcpkg.json file."; return error_info; } @@ -423,15 +432,8 @@ namespace vcpkg::Paragraphs auto baseline_version = impl->get_baseline_version(paths, port_name); if (port_entry && baseline_version) { - auto port_path = port_entry->get_port_directory(paths, *baseline_version.get()); - if (port_path.empty()) - { - Debug::print("Registry for port `", - port_name, - "` is incorrect - baseline port version `", - baseline_version.get()->to_string(), - "` not found."); - } + auto port_path = + port_entry->get_path_to_version(paths, *baseline_version.get()).value_or_exit(VCPKG_LINE_INFO); auto maybe_spgh = try_load_port(fs, port_path); if (const auto spgh = maybe_spgh.get()) { diff --git a/toolsrc/src/vcpkg/portfileprovider.cpp b/toolsrc/src/vcpkg/portfileprovider.cpp index edc527c240..c956dfe9b0 100644 --- a/toolsrc/src/vcpkg/portfileprovider.cpp +++ b/toolsrc/src/vcpkg/portfileprovider.cpp @@ -17,29 +17,25 @@ using namespace Versions; namespace { - ExpectedS get_versions_json_path(const VcpkgPaths& paths, StringView port_name) - { - auto json_path = paths.root / fs::u8path("port_versions") / - fs::u8path(Strings::concat(port_name.substr(0, 1), "-")) / - fs::u8path(Strings::concat(port_name, ".json")); - if (paths.get_filesystem().exists(json_path)) - { - return std::move(json_path); - } - return {Strings::concat("Error: Versions database file does not exist: ", fs::u8string(json_path)), - expected_right_tag}; - } + using namespace vcpkg; - ExpectedS get_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha) + struct OverlayRegistryEntry final : RegistryEntry { - auto baseline_path = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha); - if (paths.get_filesystem().exists(baseline_path)) + OverlayRegistryEntry(fs::path&& p, VersionT&& v) : path(p), version(v) { } + + View get_port_versions() const override { return {&version, 1}; } + ExpectedS get_path_to_version(const VcpkgPaths&, const VersionT& v) const override { - return std::move(baseline_path); + if (v == version) + { + return path; + } + return Strings::format("Version %s not found; only %s is available.", v.to_string(), version.to_string()); } - return {Strings::concat("Error: Baseline database file does not exist: ", fs::u8string(baseline_path)), - expected_right_tag}; - } + + fs::path path; + VersionT version; + }; } namespace vcpkg::PortFileProvider @@ -95,9 +91,9 @@ namespace vcpkg::PortFileProvider } } - static Optional try_load_overlay_port(const Files::Filesystem& fs, - View overlay_ports, - const std::string& spec) + static std::unique_ptr try_load_overlay_port(const Files::Filesystem& fs, + View overlay_ports, + const std::string& spec) { for (auto&& ports_dir : overlay_ports) { @@ -105,11 +101,12 @@ namespace vcpkg::PortFileProvider if (Paragraphs::is_port_directory(fs, ports_dir)) { auto maybe_scf = Paragraphs::try_load_port(fs, ports_dir); - if (auto scf = maybe_scf.get()) + if (auto scfp = maybe_scf.get()) { - if (scf->get()->core_paragraph->name == spec) + auto& scf = *scfp; + if (scf->core_paragraph->name == spec) { - return SourceControlFileLocation{std::move(*scf), ports_dir}; + return std::make_unique(fs::path(ports_dir), scf->to_versiont()); } } else @@ -126,17 +123,18 @@ namespace vcpkg::PortFileProvider if (Paragraphs::is_port_directory(fs, ports_spec)) { auto found_scf = Paragraphs::try_load_port(fs, ports_spec); - if (auto scf = found_scf.get()) + if (auto scfp = found_scf.get()) { - if (scf->get()->core_paragraph->name == spec) + auto& scf = *scfp; + if (scf->core_paragraph->name == spec) { - return SourceControlFileLocation{std::move(*scf), std::move(ports_spec)}; + return std::make_unique(std::move(ports_spec), scf->to_versiont()); } Checks::exit_with_message(VCPKG_LINE_INFO, "Error: Failed to load port from %s: names did not match: '%s' != '%s'", fs::u8string(ports_spec), spec, - scf->get()->core_paragraph->name); + scf->core_paragraph->name); } else { @@ -146,61 +144,38 @@ namespace vcpkg::PortFileProvider } } } - return nullopt; + return nullptr; } - static Optional try_load_registry_port(const VcpkgPaths& paths, const std::string& spec) + static std::pair, Optional> try_load_registry_port_and_baseline( + const VcpkgPaths& paths, const std::string& spec) { - const auto& fs = paths.get_filesystem(); if (auto registry = paths.get_configuration().registry_set.registry_for_port(spec)) { - auto baseline_version = registry->get_baseline_version(paths, spec); auto entry = registry->get_port_entry(paths, spec); - if (entry && baseline_version) + auto maybe_baseline = registry->get_baseline_version(paths, spec); + if (entry) { - auto port_directory = entry->get_port_directory(paths, *baseline_version.get()); - if (port_directory.empty()) + if (!maybe_baseline) { - Checks::exit_with_message(VCPKG_LINE_INFO, - "Error: registry is incorrect. Baseline version for port `%s` is `%s`, " - "but that version is not in the registry.\n", - spec, - baseline_version.get()->to_string()); - } - auto found_scf = Paragraphs::try_load_port(fs, port_directory); - if (auto scf = found_scf.get()) - { - if (scf->get()->core_paragraph->name == spec) + if (entry->get_port_versions().size() == 1) { - return SourceControlFileLocation{std::move(*scf), std::move(port_directory)}; + maybe_baseline = entry->get_port_versions()[0]; } - Checks::exit_with_message(VCPKG_LINE_INFO, - "Error: Failed to load port from %s: names did not match: '%s' != '%s'", - fs::u8string(port_directory), - spec, - scf->get()->core_paragraph->name); - } - else - { - print_error_message(found_scf.error()); - Checks::exit_with_message( - VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(port_directory)); } + return {std::move(entry), std::move(maybe_baseline)}; } else { - Debug::print("Failed to find port `", - spec, - "` in registry:", - entry ? " entry found;" : " no entry found;", - baseline_version ? " baseline version found\n" : " no baseline version found\n"); + Debug::print("Failed to find port `", spec, "` in registry: no entry found.\n"); } } else { Debug::print("Failed to find registry for port: `", spec, "`.\n"); } - return nullopt; + + return {nullptr, nullopt}; } ExpectedS PathsPortFileProvider::get_control_file(const std::string& spec) const @@ -209,18 +184,49 @@ namespace vcpkg::PortFileProvider if (cache_it == cache.end()) { const auto& fs = paths.get_filesystem(); - auto maybe_port = try_load_overlay_port(fs, overlay_ports, spec); - if (!maybe_port) - { - maybe_port = try_load_registry_port(paths, spec); - } - if (auto p = maybe_port.get()) - { - auto maybe_error = - p->source_control_file->check_against_feature_flags(p->source_location, paths.get_feature_flags()); - if (maybe_error) return std::move(*maybe_error.get()); - cache_it = cache.emplace(spec, std::move(*p)).first; + std::unique_ptr port; + VersionT port_version; + + auto maybe_overlay_port = try_load_overlay_port(fs, overlay_ports, spec); + if (maybe_overlay_port) + { + port_version = maybe_overlay_port->version; + port = std::move(maybe_overlay_port); + } + else + { + auto maybe_registry_port = try_load_registry_port_and_baseline(paths, spec); + port = std::move(maybe_registry_port.first); + if (auto version = maybe_registry_port.second.get()) + { + port_version = std::move(*version); + } + else if (port) + { + return std::string("No baseline version available."); + } + } + + if (port) + { + auto port_path = port->get_path_to_version(paths, port_version).value_or_exit(VCPKG_LINE_INFO); + auto maybe_scfl = Paragraphs::try_load_port(fs, port_path); + if (auto p = maybe_scfl.get()) + { + auto maybe_error = (*p)->check_against_feature_flags(port_path, paths.get_feature_flags()); + if (maybe_error) return std::move(*maybe_error.get()); + + cache_it = + cache.emplace(spec, SourceControlFileLocation{std::move(*p), std::move(port_path)}).first; + } + else + { + return Strings::format("Error: when loading port `%s` from directory `%s`:\n%s\n", + spec, + fs::u8string(port_path), + maybe_scfl.error()->error); + } } } @@ -296,179 +302,86 @@ namespace vcpkg::PortFileProvider { struct BaselineProviderImpl : IBaselineProvider, Util::ResourceBase { - BaselineProviderImpl(const VcpkgPaths& paths) : paths(paths) { } - BaselineProviderImpl(const VcpkgPaths& paths, StringView baseline) - : paths(paths), m_baseline(baseline.to_string()) - { - } - - const Optional>>& get_baseline_cache() const - { - return baseline_cache.get_lazy([&]() -> Optional>> { - if (auto baseline = m_baseline.get()) - { - auto baseline_file = get_baseline_json_path(paths, *baseline).value_or_exit(VCPKG_LINE_INFO); - - auto maybe_baselines_map = - parse_baseline_file(paths.get_filesystem(), "default", baseline_file); - Checks::check_exit(VCPKG_LINE_INFO, - maybe_baselines_map.has_value(), - "Error: Couldn't parse baseline `%s` from `%s`", - "default", - fs::u8string(baseline_file)); - auto baselines_map = *maybe_baselines_map.get(); - return std::move(baselines_map); - } - else - { - // No baseline was provided, so use current repo - const auto& fs = paths.get_filesystem(); - auto baseline_file = paths.root / fs::u8path("port_versions") / fs::u8path("baseline.json"); - if (fs.exists(baseline_file)) - { - auto maybe_baselines_map = - parse_baseline_file(paths.get_filesystem(), "default", baseline_file); - Checks::check_exit(VCPKG_LINE_INFO, - maybe_baselines_map.has_value(), - "Error: Couldn't parse baseline `%s` from `%s`", - "default", - fs::u8string(baseline_file)); - auto baselines_map = *maybe_baselines_map.get(); - return std::move(baselines_map); - } - else - { - // No baseline file in current repo -- use current port versions. - m_portfile_provider = - std::make_unique(paths, std::vector{}); - return nullopt; - } - } - }); - } + BaselineProviderImpl(const VcpkgPaths& paths_) : paths(paths_) { } virtual Optional get_baseline_version(StringView port_name) const override { - const auto& cache = get_baseline_cache(); - if (auto p_cache = cache.get()) + auto it = m_baseline_cache.find(port_name); + if (it != m_baseline_cache.end()) { - auto it = p_cache->find(port_name.to_string()); - if (it != p_cache->end()) - { - return it->second; - } - return nullopt; + return it->second; } else { - auto maybe_scfl = m_portfile_provider->get_control_file(port_name.to_string()); - if (auto p_scfl = maybe_scfl.get()) - { - auto cpgh = p_scfl->source_control_file->core_paragraph.get(); - return VersionT{cpgh->version, cpgh->port_version}; - } - else - { - return nullopt; - } + auto version = paths.get_configuration().registry_set.baseline_for_port(paths, port_name); + m_baseline_cache.emplace(port_name.to_string(), version); + return version; } } private: - const VcpkgPaths& paths; - const Optional m_baseline; - Lazy>>> baseline_cache; - mutable std::unique_ptr m_portfile_provider; + const VcpkgPaths& paths; // TODO: remove this data member + mutable std::map, std::less<>> m_baseline_cache; }; struct VersionedPortfileProviderImpl : IVersionedPortfileProvider, Util::ResourceBase { - VersionedPortfileProviderImpl(const VcpkgPaths& paths) : paths(paths) { } + VersionedPortfileProviderImpl(const VcpkgPaths& paths_) : paths(paths_) { } - virtual const std::vector& get_port_versions(StringView port_name) const override + virtual View get_port_versions(StringView port_name) const override { - auto cache_it = versions_cache.find(port_name.to_string()); - if (cache_it != versions_cache.end()) + auto entry_it = m_entry_cache.find(port_name.to_string()); + if (entry_it != m_entry_cache.end()) { - return cache_it->second; + return entry_it->second->get_port_versions(); } - auto maybe_versions_file_path = get_versions_json_path(get_paths(), port_name); - if (auto versions_file_path = maybe_versions_file_path.get()) + auto entry = try_load_registry_port_and_baseline(paths, port_name.to_string()); + if (!entry.first) { - auto maybe_version_entries = parse_versions_file(get_filesystem(), port_name, *versions_file_path); - Checks::check_exit(VCPKG_LINE_INFO, - maybe_version_entries.has_value(), - "Error: Couldn't parse versions from file: %s", - fs::u8string(*versions_file_path)); - auto version_entries = maybe_version_entries.value_or_exit(VCPKG_LINE_INFO); - - auto port = port_name.to_string(); - for (auto&& version_entry : version_entries) - { - VersionSpec spec(port, version_entry.version); - versions_cache[port].push_back(spec); - git_tree_cache.emplace(std::move(spec), std::move(version_entry.git_tree)); - } - return versions_cache.at(port); - } - else - { - // Fall back to current available version - auto maybe_port = try_load_registry_port(paths, port_name.to_string()); - if (auto p = maybe_port.get()) - { - auto maybe_error = p->source_control_file->check_against_feature_flags( - p->source_location, paths.get_feature_flags()); - - if (auto error = maybe_error.get()) - { - Checks::exit_with_message(VCPKG_LINE_INFO, "Error: %s", *error); - } - - VersionSpec vspec(port_name.to_string(), - VersionT(p->source_control_file->core_paragraph->version, - p->source_control_file->core_paragraph->port_version)); - control_cache.emplace(vspec, std::move(*p)); - return versions_cache.emplace(port_name.to_string(), std::vector{vspec}) - .first->second; - } Checks::exit_with_message( VCPKG_LINE_INFO, "Error: Could not find a definition for port %s", port_name); } + auto it = m_entry_cache.emplace(port_name.to_string(), std::move(entry.first)); + return it.first->second->get_port_versions(); } - virtual ExpectedS get_control_file( - const VersionSpec& version_spec) const override + ExpectedS get_control_file(const VersionSpec& version_spec) const override { - // Pre-populate versions cache. - get_port_versions(version_spec.port_name); - - auto cache_it = control_cache.find(version_spec); - if (cache_it != control_cache.end()) + auto cache_it = m_control_cache.find(version_spec); + if (cache_it != m_control_cache.end()) { return cache_it->second; } - auto git_tree_cache_it = git_tree_cache.find(version_spec); - if (git_tree_cache_it == git_tree_cache.end()) + auto entry_it = m_entry_cache.find(version_spec.port_name); + if (entry_it == m_entry_cache.end()) { - return Strings::concat("Error: No git object SHA for entry ", - version_spec.port_name, - " at version ", - version_spec.version, - "."); + auto reg_for_port = + paths.get_configuration().registry_set.registry_for_port(version_spec.port_name); + + if (!reg_for_port) + { + return Strings::format("Error: no registry set up for port %s", version_spec.port_name); + } + + auto entry = reg_for_port->get_port_entry(paths, version_spec.port_name); + entry_it = m_entry_cache.emplace(version_spec.port_name, std::move(entry)).first; } - const std::string git_tree = git_tree_cache_it->second; - auto port_directory = get_paths().git_checkout_port(get_filesystem(), version_spec.port_name, git_tree); + auto maybe_path = entry_it->second->get_path_to_version(paths, version_spec.version); + if (!maybe_path.has_value()) + { + return std::move(maybe_path).error(); + } + auto& port_directory = *maybe_path.get(); - auto maybe_control_file = Paragraphs::try_load_port(get_filesystem(), port_directory); + auto maybe_control_file = Paragraphs::try_load_port(paths.get_filesystem(), port_directory); if (auto scf = maybe_control_file.get()) { if (scf->get()->core_paragraph->name == version_spec.port_name) { - return control_cache + return m_control_cache .emplace(version_spec, SourceControlFileLocation{std::move(*scf), std::move(port_directory)}) .first->second; @@ -484,14 +397,10 @@ namespace vcpkg::PortFileProvider "Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory)); } - const VcpkgPaths& get_paths() const { return paths; } - Files::Filesystem& get_filesystem() const { return paths.get_filesystem(); } - private: - const VcpkgPaths& paths; - mutable std::map> versions_cache; - mutable std::unordered_map git_tree_cache; - mutable std::unordered_map control_cache; + const VcpkgPaths& paths; // TODO: remove this data member + mutable std::unordered_map m_control_cache; + mutable std::map, std::less<>> m_entry_cache; }; } @@ -500,11 +409,6 @@ namespace vcpkg::PortFileProvider return std::make_unique(paths); } - std::unique_ptr make_baseline_provider(const vcpkg::VcpkgPaths& paths, StringView baseline) - { - return std::make_unique(paths, baseline); - } - std::unique_ptr make_versioned_portfile_provider(const vcpkg::VcpkgPaths& paths) { return std::make_unique(paths); diff --git a/toolsrc/src/vcpkg/registries.cpp b/toolsrc/src/vcpkg/registries.cpp index 51c8849c97..22ac26fd9d 100644 --- a/toolsrc/src/vcpkg/registries.cpp +++ b/toolsrc/src/vcpkg/registries.cpp @@ -3,8 +3,9 @@ #include #include -#include +#include #include +#include #include #include #include @@ -16,326 +17,365 @@ namespace { using namespace vcpkg; - struct BuiltinEntry final : RegistryEntry + using Baseline = std::map>; + + static fs::path port_versions_dir = fs::u8path("port_versions"); + + // this class is an implementation detail of `BuiltinRegistryEntry`; + // when `BuiltinRegistryEntry` is using a port versions file for a port, + // it uses this as it's underlying type; + // when `BuiltinRegistryEntry` is using a port tree, it uses the scfl + struct GitRegistryEntry { - fs::path port_directory; + explicit GitRegistryEntry(std::string&& port_name) : port_name(port_name) { } - BuiltinEntry(fs::path&& p) : port_directory(std::move(p)) { } + std::string port_name; - fs::path get_port_directory(const VcpkgPaths&, const VersionT&) const override { return port_directory; } + // these two map port versions to git trees + // these shall have the same size, and git_trees[i] shall be the git tree for port_versions[i] + std::vector port_versions; + std::vector git_trees; }; - struct BuiltinRegistry final : RegistryImpl + struct BuiltinRegistryEntry final : RegistryEntry { - std::unique_ptr get_port_entry(const VcpkgPaths& paths, StringView port_name) const override + explicit BuiltinRegistryEntry(std::unique_ptr&& entry) + : git_entry(std::move(entry)), scfl(nullptr) { - auto p = paths.builtin_ports_directory() / fs::u8path(port_name); - if (paths.get_filesystem().exists(p)) + } + explicit BuiltinRegistryEntry(std::unique_ptr&& scfl_) + : git_entry(nullptr), scfl(std::move(scfl_)), scfl_version(scfl->to_versiont()) + { + } + + View get_port_versions() const override + { + if (git_entry) { - return std::make_unique(std::move(p)); + return git_entry->port_versions; } else { - return nullptr; + return {&scfl_version, 1}; } } + ExpectedS get_path_to_version(const VcpkgPaths&, const VersionT& version) const override; - void get_all_port_names(std::vector& names, const VcpkgPaths& paths) const override - { - const auto& fs = paths.get_filesystem(); - auto port_dirs = fs.get_files_non_recursive(paths.builtin_ports_directory()); - Util::sort(port_dirs); + // exactly one of these two shall be null - Util::erase_remove_if(port_dirs, - [&](auto&& port_dir_entry) { return port_dir_entry.filename() == ".DS_Store"; }); - - std::transform(port_dirs.begin(), port_dirs.end(), std::back_inserter(names), [](const fs::path& p) { - return fs::u8string(p.filename()); - }); - } - - Optional get_baseline_version(const VcpkgPaths&, StringView) const override { return VersionT{}; } + // if we find a versions.json, this shall be non-null and BuiltinRegistryEntry uses git_entry's implementation + std::unique_ptr git_entry; + // otherwise, if we don't find a versions.json, + // we fall back to just using the version in the ports directory, and this is the non-null one + std::unique_ptr scfl; + VersionT scfl_version; // this exists so that we can return a pointer to it }; - struct FilesystemEntry final : RegistryEntry + struct FilesystemRegistryEntry final : RegistryEntry { - std::map versions; + explicit FilesystemRegistryEntry(std::string&& port_name) : port_name(port_name) { } - fs::path get_port_directory(const VcpkgPaths&, const VersionT& version) const override - { - auto it = versions.find(version); - if (it != versions.end()) - { - return it->second; - } - return {}; - } + View get_port_versions() const override { return port_versions; } + + ExpectedS get_path_to_version(const VcpkgPaths& paths, const VersionT& version) const override; + + std::string port_name; + + // these two map port versions to paths + // these shall have the same size, and paths[i] shall be the path for port_versions[i] + std::vector port_versions; + std::vector version_paths; }; - struct FilesystemVersionEntryDeserializer final : Json::IDeserializer> + struct BuiltinRegistry final : RegistryImplementation { - StringView type_name() const override { return "a version entry object"; } - View valid_fields() const override - { - static const StringView t[] = {"version-string", "port-version", "registry-path"}; - return t; - } + BuiltinRegistry(std::string&& baseline) : m_baseline_identifier(std::move(baseline)) { } - Optional> visit_object(Json::Reader& r, const Json::Object& obj) override - { - fs::path registry_path; + std::unique_ptr get_port_entry(const VcpkgPaths& paths, StringView port_name) const override; - auto version = get_versiont_deserializer_instance().visit_object(r, obj); + void get_all_port_names(std::vector&, const VcpkgPaths&) const override; - r.required_object_field( - "version entry", obj, "registry-path", registry_path, Json::PathDeserializer::instance); - // registry_path should look like `/blah/foo` - if (registry_path.has_root_name() || !registry_path.has_root_directory()) - { - r.add_generic_error(type_name(), "must be an absolute path without a drive name"); - registry_path.clear(); - } + Optional get_baseline_version(const VcpkgPaths& paths, StringView port_name) const override; - return std::pair{std::move(version).value_or(VersionT{}), std::move(registry_path)}; - } + ~BuiltinRegistry() = default; - static FilesystemVersionEntryDeserializer instance; + std::string m_baseline_identifier; + DelayedInit m_baseline; }; - FilesystemVersionEntryDeserializer FilesystemVersionEntryDeserializer::instance; - struct FilesystemEntryDeserializer final : Json::IDeserializer + struct FilesystemRegistry final : RegistryImplementation { - StringView type_name() const override { return "a registry entry object"; } - - Optional visit_array(Json::Reader& r, const Json::Array& arr) override + FilesystemRegistry(fs::path&& path, std::string&& baseline) + : m_path(std::move(path)), m_baseline_identifier(baseline) { - FilesystemEntry res; + } - std::pair buffer; - for (std::size_t idx = 0; idx < arr.size(); ++idx) + std::unique_ptr get_port_entry(const VcpkgPaths&, StringView) const override; + + void get_all_port_names(std::vector&, const VcpkgPaths&) const override; + + Optional get_baseline_version(const VcpkgPaths&, StringView) const override; + + private: + fs::path m_path; + std::string m_baseline_identifier; + DelayedInit m_baseline; + }; + + ExpectedS get_git_baseline_json_path(const VcpkgPaths& paths, StringView baseline_commit_sha) + { + auto baseline_path = paths.git_checkout_baseline(paths.get_filesystem(), baseline_commit_sha); + if (paths.get_filesystem().exists(baseline_path)) + { + return std::move(baseline_path); + } + return {Strings::concat("Error: Baseline database file does not exist: ", fs::u8string(baseline_path)), + expected_right_tag}; + } + + struct VersionDbEntry + { + VersionT version; + Versions::Scheme scheme = Versions::Scheme::String; + + // only one of these may be non-empty + std::string git_tree; + fs::path path; + }; + + // VersionDbType::Git => VersionDbEntry.git_tree is filled + // VersionDbType::Filesystem => VersionDbEntry.path is filled + enum class VersionDbType + { + Git, + Filesystem, + }; + + fs::path relative_path_to_versions(StringView port_name); + ExpectedS> load_versions_file(Files::Filesystem& fs, + VersionDbType vdb, + const fs::path& port_versions, + StringView port_name, + const fs::path& registry_root = {}); + + // returns nullopt if the baseline is valid, but doesn't contain the specified baseline, + // or (equivalently) if the baseline does not exist. + ExpectedS> parse_baseline_versions(StringView contents, StringView baseline); + ExpectedS> load_baseline_versions(const VcpkgPaths& paths, + const fs::path& path_to_baseline, + StringView identifier = {}); + + void load_all_port_names_from_port_versions(std::vector& out, + const VcpkgPaths& paths, + const fs::path& port_versions_path) + { + for (auto super_directory : fs::directory_iterator(port_versions_path)) + { + if (!fs::is_directory(paths.get_filesystem().status(VCPKG_LINE_INFO, super_directory))) continue; + + for (auto file : fs::directory_iterator(super_directory)) { - r.visit_at_index( - arr[idx], static_cast(idx), buffer, FilesystemVersionEntryDeserializer::instance); + auto filename = fs::u8string(file.path().filename()); + if (!Strings::ends_with(filename, ".json")) continue; - auto it = res.versions.lower_bound(buffer.first); - if (it == res.versions.end() || it->first != buffer.first) + auto port_name = filename.substr(0, filename.size() - 5); + if (!Json::PackageNameDeserializer::is_package_name(port_name)) { - buffer.second = registry_root / fs::lexically_normal(buffer.second).relative_path(); - res.versions.insert(it, std::move(buffer)); - } - else if (buffer.first != VersionT{}) - { - r.add_generic_error( - type_name(), "Gave multiple definitions for version: ", buffer.first.to_string()); + Checks::exit_with_message( + VCPKG_LINE_INFO, "Error: found invalid port version file name: `%s`.", fs::u8string(file)); } + out.push_back(std::move(port_name)); } + } + } + // { RegistryImplementation + + // { BuiltinRegistry::RegistryImplementation + std::unique_ptr BuiltinRegistry::get_port_entry(const VcpkgPaths& paths, StringView port_name) const + { + auto versions_path = paths.builtin_port_versions / relative_path_to_versions(port_name); + if (paths.get_feature_flags().registries && paths.get_filesystem().exists(versions_path)) + { + auto maybe_version_entries = + load_versions_file(paths.get_filesystem(), VersionDbType::Git, paths.builtin_port_versions, port_name); + Checks::check_exit( + VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: %s", maybe_version_entries.error()); + auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO); + + auto res = + std::make_unique(std::make_unique(port_name.to_string())); + auto gre = res->git_entry.get(); + for (auto&& version_entry : version_entries) + { + gre->port_versions.push_back(version_entry.version); + gre->git_trees.push_back(version_entry.git_tree); + } return res; } - FilesystemEntryDeserializer(const fs::path& p) : registry_root(p) { } + // Fall back to current available version + auto port_directory = paths.builtin_ports_directory() / fs::u8path(port_name); + if (paths.get_filesystem().exists(port_directory)) + { + auto found_scf = Paragraphs::try_load_port(paths.get_filesystem(), port_directory); + if (auto scfp = found_scf.get()) + { + auto& scf = *scfp; + auto maybe_error = scf->check_against_feature_flags(port_directory, paths.get_feature_flags()); + if (maybe_error) + { + Checks::exit_with_message(VCPKG_LINE_INFO, "Parsing manifest failed: %s", *maybe_error.get()); + } - const fs::path& registry_root; - }; + if (scf->core_paragraph->name == port_name) + { + return std::make_unique( + std::make_unique(std::move(scf), std::move(port_directory))); + } + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: Failed to load port from %s: names did not match: '%s' != '%s'", + fs::u8string(port_directory), + port_name, + scf->core_paragraph->name); + } + } - struct FilesystemRegistry final : RegistryImpl + return nullptr; + } + + Baseline parse_builtin_baseline(const VcpkgPaths& paths, StringView baseline_identifier) { - std::unique_ptr get_port_entry(const VcpkgPaths& paths, StringView port_name) const override + auto path_to_baseline = paths.builtin_port_versions / fs::u8path("baseline.json"); + auto res_baseline = load_baseline_versions(paths, path_to_baseline, baseline_identifier); + + if (!res_baseline.has_value()) { - const auto& fs = paths.get_filesystem(); - auto entry_path = this->path_to_port_entry(paths, port_name); - if (!fs.exists(entry_path)) - { - Debug::print( - "Failed to find entry for port `", port_name, "` in file: ", fs::u8string(entry_path), "\n"); - return nullptr; - } - std::error_code ec; - auto json_document = Json::parse_file(fs, entry_path, ec); - - if (auto p = json_document.get()) - { - Json::Reader r; - auto real_path = paths.config_root_dir / path; - FilesystemEntryDeserializer deserializer{real_path}; - auto entry = r.visit(p->first, deserializer); - auto pentry = entry.get(); - if (pentry && r.errors().empty()) - { - return std::make_unique(std::move(*pentry)); - } - else - { - vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, - "Failed to parse the port entry for port `%s` at `%s`.\n%s", - port_name, - fs::u8string(entry_path), - Strings::join("\n", r.errors())); - } - } - else - { - Debug::print("Failed to parse json document: ", json_document.error()->format(), "\n"); - } - - return nullptr; + Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error()); + } + auto opt_baseline = res_baseline.get(); + if (auto p = opt_baseline->get()) + { + return std::move(*p); } - void get_all_port_names(std::vector& port_names, const VcpkgPaths& paths) const override + if (baseline_identifier.size() == 0) { - std::error_code ec; - for (const auto& super_dir : fs::directory_iterator(path)) - { - if (!fs::is_directory(paths.get_filesystem().status(super_dir, ec))) - { - continue; - } - - auto super_dir_filename = fs::u8string(super_dir.path().filename()); - if (!Strings::ends_with(super_dir_filename, "-")) - { - continue; - } - - super_dir_filename.pop_back(); - for (const auto& database_entry : fs::directory_iterator(super_dir)) - { - auto database_entry_filename = database_entry.path().filename(); - auto database_entry_filename_str = fs::u8string(database_entry_filename); - - if (!Strings::starts_with(database_entry_filename_str, super_dir_filename) || - !Strings::ends_with(database_entry_filename_str, ".json")) - { - Debug::print("Unexpected file in database (this is not an error): ", - fs::u8string(database_entry.path()), - "\n"); - continue; - } - - port_names.push_back(fs::u8string(database_entry_filename.replace_extension())); - } - } + return {}; } - Optional get_baseline_version(const VcpkgPaths& paths, StringView port_name) const override + if (baseline_identifier == "default") { - if (!paths.get_feature_flags().versions) - { - Checks::check_exit(VCPKG_LINE_INFO, - "This invocation failed because the `versions` feature flag is not enabled."); - } + Checks::exit_with_message(VCPKG_LINE_INFO, + "Couldn't find explicitly specified baseline `\"default\"` in the baseline file.", + baseline_identifier); + } - const auto& baseline_cache = baseline.get([this, &paths] { return load_baseline_versions(paths); }); - auto it = baseline_cache.find(port_name); - if (it != baseline_cache.end()) + // attempt to check out the baseline: + auto maybe_path = get_git_baseline_json_path(paths, baseline_identifier); + if (!maybe_path.has_value()) + { + Checks::exit_with_message(VCPKG_LINE_INFO, + "Couldn't find explicitly specified baseline `\"%s\"` in the baseline file, " + "and there was no baseline at that commit or the commit didn't exist.\n%s", + baseline_identifier, + maybe_path.error()); + } + + res_baseline = load_baseline_versions(paths, *maybe_path.get()); + if (!res_baseline.has_value()) + { + Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error()); + } + opt_baseline = res_baseline.get(); + if (auto p = opt_baseline->get()) + { + return std::move(*p); + } + + Checks::exit_with_message(VCPKG_LINE_INFO, + "Couldn't find explicitly specified baseline `\"%s\"` in the baseline " + "file, and the `\"default\"` baseline does not exist at that commit.", + baseline_identifier); + } + Optional BuiltinRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const + { + if (paths.get_feature_flags().registries) + { + const auto& baseline = m_baseline.get( + [this, &paths]() -> Baseline { return parse_builtin_baseline(paths, m_baseline_identifier); }); + + auto it = baseline.find(port_name); + if (it != baseline.end()) { return it->second; } - else - { - return nullopt; - } } - FilesystemRegistry(fs::path&& path_) : path(path_) { } - - private: - fs::path path_to_registry_database(const VcpkgPaths& paths) const + // fall back to using the ports directory version + auto maybe_scf = + Paragraphs::try_load_port(paths.get_filesystem(), paths.builtin_ports_directory() / fs::u8path(port_name)); + if (auto pscf = maybe_scf.get()) { - fs::path path_to_db = paths.config_root_dir / path; - path_to_db /= fs::u8path( - {'\xF0', '\x9F', '\x98', '\x87'}); // utf-8 for https://emojipedia.org/smiling-face-with-halo/ - return path_to_db; + auto& scf = *pscf; + return scf->to_versiont(); } - - fs::path path_to_port_entry(const VcpkgPaths& paths, StringView port_name) const - { - Checks::check_exit(VCPKG_LINE_INFO, port_name.size() != 0); - fs::path path_to_entry = path_to_registry_database(paths); - path_to_entry /= fs::u8path({port_name.byte_at_index(0), '-'}); - path_to_entry /= fs::u8path(port_name); - path_to_entry += fs::u8path(".json"); - - return path_to_entry; - } - - std::map> load_baseline_versions(const VcpkgPaths& paths) const - { - auto baseline_file = path_to_registry_database(paths) / fs::u8path("baseline.json"); - - auto value = Json::parse_file(VCPKG_LINE_INFO, paths.get_filesystem(), baseline_file); - if (!value.first.is_object()) - { - Checks::exit_with_message(VCPKG_LINE_INFO, "Error: `baseline.json` does not have a top-level object."); - } - - auto maybe_baseline_versions = parse_baseline_file(paths.get_filesystem(), "default", baseline_file); - if (auto baseline_versions = maybe_baseline_versions.get()) - { - return std::move(*baseline_versions); - } - else - { - Checks::exit_with_message(VCPKG_LINE_INFO, - "Error: failed to parse `%s`:\n%s", - fs::u8string(baseline_file), - maybe_baseline_versions.error()); - } - } - - fs::path path; - DelayedInit>> baseline; - }; -} - -namespace vcpkg -{ - std::unique_ptr Registry::builtin_registry() { return std::make_unique(); } - - Registry::Registry(std::vector&& packages, std::unique_ptr&& impl) - : packages_(std::move(packages)), implementation_(std::move(impl)) - { - Checks::check_exit(VCPKG_LINE_INFO, implementation_ != nullptr); + Debug::print("Failed to load port `", port_name, "` from the ports tree: ", maybe_scf.error()->error, "\n"); + return nullopt; } - RegistryImplDeserializer RegistryImplDeserializer::instance; - - StringView RegistryImplDeserializer::type_name() const { return "a registry"; } - - constexpr StringLiteral RegistryImplDeserializer::KIND; - constexpr StringLiteral RegistryImplDeserializer::PATH; - constexpr StringLiteral RegistryImplDeserializer::KIND_BUILTIN; - constexpr StringLiteral RegistryImplDeserializer::KIND_FILESYSTEM; - - View RegistryImplDeserializer::valid_fields() const + void BuiltinRegistry::get_all_port_names(std::vector& out, const VcpkgPaths& paths) const { - static const StringView t[] = {KIND, PATH}; - return t; - } - - Optional> RegistryImplDeserializer::visit_null(Json::Reader&) { return nullptr; } - - Optional> RegistryImplDeserializer::visit_object(Json::Reader& r, - const Json::Object& obj) - { - static Json::StringDeserializer kind_deserializer{"a registry implementation kind"}; - std::string kind; - r.required_object_field(type_name(), obj, KIND, kind, kind_deserializer); - - if (kind == KIND_BUILTIN) + if (paths.get_feature_flags().registries && paths.get_filesystem().exists(paths.builtin_port_versions)) { - if (obj.contains(PATH)) - { - r.add_extra_field_error("a builtin registry", PATH); - } - return static_cast>(std::make_unique()); + load_all_port_names_from_port_versions(out, paths, paths.builtin_port_versions); } - else if (kind == KIND_FILESYSTEM) - { - fs::path path; - r.required_object_field("a filesystem registry", obj, PATH, path, Json::PathDeserializer::instance); - return static_cast>(std::make_unique(std::move(path))); + for (auto port_directory : fs::directory_iterator(paths.builtin_ports_directory())) + { + if (!fs::is_directory(paths.get_filesystem().status(VCPKG_LINE_INFO, port_directory))) continue; + auto filename = fs::u8string(port_directory.path().filename()); + if (filename == ".DS_Store") continue; + out.push_back(filename); + } + } + // } BuiltinRegistry::RegistryImplementation + + // { FilesystemRegistry::RegistryImplementation + Baseline parse_filesystem_baseline(const VcpkgPaths& paths, const fs::path& root, StringView baseline_identifier) + { + auto path_to_baseline = root / port_versions_dir / fs::u8path("baseline.json"); + auto res_baseline = load_baseline_versions(paths, path_to_baseline, baseline_identifier); + if (auto opt_baseline = res_baseline.get()) + { + if (auto p = opt_baseline->get()) + { + return std::move(*p); + } + + if (baseline_identifier.size() == 0) + { + return {}; + } + + Checks::exit_with_message( + VCPKG_LINE_INFO, + "Error: could not find explicitly specified baseline `\"%s\"` in baseline file `%s`.", + baseline_identifier, + fs::u8string(path_to_baseline)); + } + + Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error()); + } + Optional FilesystemRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const + { + const auto& baseline = m_baseline.get( + [this, &paths]() -> Baseline { return parse_filesystem_baseline(paths, m_path, m_baseline_identifier); }); + + auto it = baseline.find(port_name); + if (it != baseline.end()) + { + return it->second; } else { @@ -343,14 +383,329 @@ namespace vcpkg } } - StringView RegistryDeserializer::type_name() const { return "a registry"; } + std::unique_ptr FilesystemRegistry::get_port_entry(const VcpkgPaths& paths, + StringView port_name) const + { + auto maybe_version_entries = load_versions_file( + paths.get_filesystem(), VersionDbType::Filesystem, m_path / port_versions_dir, port_name, m_path); + Checks::check_exit( + VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: %s", maybe_version_entries.error()); + auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO); + auto res = std::make_unique(port_name.to_string()); + for (auto&& version_entry : version_entries) + { + res->port_versions.push_back(std::move(version_entry.version)); + res->version_paths.push_back(std::move(version_entry.path)); + } + return res; + } + + void FilesystemRegistry::get_all_port_names(std::vector& out, const VcpkgPaths& paths) const + { + load_all_port_names_from_port_versions(out, paths, m_path / port_versions_dir); + } + // } FilesystemRegistry::RegistryImplementation + + // } RegistryImplementation + + // { RegistryEntry + + // { BuiltinRegistryEntry::RegistryEntry + ExpectedS BuiltinRegistryEntry::get_path_to_version(const VcpkgPaths& paths, + const VersionT& version) const + { + if (git_entry) + { + auto it = std::find(git_entry->port_versions.begin(), git_entry->port_versions.end(), version); + if (it == git_entry->port_versions.end()) + { + return Strings::concat( + "Error: No version entry for ", git_entry->port_name, " at version ", version, "."); + } + + const auto& git_tree = git_entry->git_trees[it - git_entry->port_versions.begin()]; + return paths.git_checkout_port(paths.get_filesystem(), git_entry->port_name, git_tree); + } + + if (scfl_version == version) + { + return scfl->source_location; + } + + auto& name = scfl->source_control_file->core_paragraph->name; + return Strings::format( + "Error: no version entry for %s at version %s.\n" + "We are currently using the version in the ports tree, since no %s.json was found in port_versions.", + name, + scfl->to_versiont().to_string(), + name); + } + // } BuiltinRegistryEntry::RegistryEntry + + // { FilesystemRegistryEntry::RegistryEntry + ExpectedS FilesystemRegistryEntry::get_path_to_version(const VcpkgPaths&, const VersionT& version) const + { + auto it = std::find(port_versions.begin(), port_versions.end(), version); + if (it == port_versions.end()) + { + return Strings::concat("Error: No version entry for ", port_name, " at version ", version, "."); + } + return version_paths[it - port_versions.begin()]; + } + // } FilesystemRegistryEntry::RegistryEntry + + // } RegistryEntry +} + +// deserializers +namespace +{ + using namespace vcpkg; + + struct BaselineDeserializer final : Json::IDeserializer>> + { + StringView type_name() const override { return "a baseline object"; } + + Optional visit_object(Json::Reader& r, const Json::Object& obj) override + { + std::map> result; + + for (auto pr : obj) + { + const auto& version_value = pr.second; + VersionT version; + r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance()); + + result.emplace(pr.first.to_string(), std::move(version)); + } + + return std::move(result); + } + + static BaselineDeserializer instance; + }; + BaselineDeserializer BaselineDeserializer::instance; + + struct VersionDbEntryDeserializer final : Json::IDeserializer + { + static constexpr StringLiteral GIT_TREE = "git-tree"; + static constexpr StringLiteral PATH = "path"; + + StringView type_name() const override { return "a version database entry"; } + View valid_fields() const override + { + static const StringView u_git[] = {GIT_TREE}; + static const StringView u_path[] = {PATH}; + static const auto t_git = vcpkg::Util::Vectors::concat(schemed_deserializer_fields(), u_git); + static const auto t_path = vcpkg::Util::Vectors::concat(schemed_deserializer_fields(), u_path); + + return type == VersionDbType::Git ? t_git : t_path; + } + + Optional visit_object(Json::Reader& r, const Json::Object& obj) override + { + VersionDbEntry ret; + + auto schemed_version = visit_required_schemed_deserializer(type_name(), r, obj); + ret.scheme = schemed_version.scheme; + ret.version = std::move(schemed_version.versiont); + + static Json::StringDeserializer git_tree_deserializer("a git object SHA"); + static Json::StringDeserializer path_deserializer("a registry path"); + + switch (type) + { + case VersionDbType::Git: + { + r.required_object_field(type_name(), obj, GIT_TREE, ret.git_tree, git_tree_deserializer); + break; + } + case VersionDbType::Filesystem: + { + std::string path_res; + r.required_object_field(type_name(), obj, PATH, path_res, path_deserializer); + fs::path p = fs::u8path(path_res); + if (p.is_absolute()) + { + r.add_generic_error("a registry path", + "A registry path may not be absolute, and must start with a `$` to mean " + "the registry root; e.g., `$/foo/bar`."); + return ret; + } + else if (p.empty()) + { + r.add_generic_error("a registry path", "A registry path must not be empty."); + return ret; + } + + auto it = p.begin(); + if (*it != "$") + { + r.add_generic_error( + "a registry path", + "A registry path must start with `$` to mean the registry root; e.g., `$/foo/bar`"); + } + + ret.path = registry_root; + ++it; + std::for_each(it, p.end(), [&r, &ret](const fs::path& p) { + if (p == "..") + { + r.add_generic_error("a registry path", "A registry path must not contain `..`."); + } + else + { + ret.path /= p; + } + }); + + break; + } + } + + return ret; + } + + VersionDbEntryDeserializer(VersionDbType type, const fs::path& root) : type(type), registry_root(root) { } + + VersionDbType type; + fs::path registry_root; + }; + + struct VersionDbEntryArrayDeserializer final : Json::IDeserializer> + { + virtual StringView type_name() const override { return "an array of versions"; } + + virtual Optional> visit_array(Json::Reader& r, const Json::Array& arr) override + { + return r.array_elements(arr, underlying); + } + + VersionDbEntryArrayDeserializer(VersionDbType type, const fs::path& root) : underlying{type, root} { } + + VersionDbEntryDeserializer underlying; + }; + + struct RegistryImplDeserializer : Json::IDeserializer> + { + constexpr static StringLiteral KIND = "kind"; + constexpr static StringLiteral BASELINE = "baseline"; + constexpr static StringLiteral PATH = "path"; + + constexpr static StringLiteral KIND_BUILTIN = "builtin"; + constexpr static StringLiteral KIND_FILESYSTEM = "filesystem"; + + virtual StringView type_name() const override { return "a registry"; } + virtual View valid_fields() const override; + + virtual Optional> visit_null(Json::Reader&) override; + virtual Optional> visit_object(Json::Reader&, + const Json::Object&) override; + + RegistryImplDeserializer(const fs::path& configuration_directory) + : config_directory(configuration_directory) { } + + fs::path config_directory; + }; + constexpr StringLiteral RegistryImplDeserializer::KIND; + constexpr StringLiteral RegistryImplDeserializer::BASELINE; + constexpr StringLiteral RegistryImplDeserializer::PATH; + constexpr StringLiteral RegistryImplDeserializer::KIND_BUILTIN; + constexpr StringLiteral RegistryImplDeserializer::KIND_FILESYSTEM; + + struct RegistryDeserializer final : Json::IDeserializer + { + constexpr static StringLiteral PACKAGES = "packages"; + + virtual StringView type_name() const override { return "a registry"; } + virtual View valid_fields() const override; + + virtual Optional visit_object(Json::Reader&, const Json::Object&) override; + + explicit RegistryDeserializer(const fs::path& configuration_directory) : impl_des(configuration_directory) { } + + RegistryImplDeserializer impl_des; + }; constexpr StringLiteral RegistryDeserializer::PACKAGES; + View RegistryImplDeserializer::valid_fields() const + { + static const StringView t[] = {KIND, BASELINE, PATH}; + return t; + } + View valid_builtin_fields() + { + static const StringView t[] = { + RegistryImplDeserializer::KIND, + RegistryImplDeserializer::BASELINE, + RegistryDeserializer::PACKAGES, + }; + return t; + } + View valid_filesystem_fields() + { + static const StringView t[] = { + RegistryImplDeserializer::KIND, + RegistryImplDeserializer::BASELINE, + RegistryImplDeserializer::PATH, + RegistryDeserializer::PACKAGES, + }; + return t; + } + + Optional> RegistryImplDeserializer::visit_null(Json::Reader&) + { + return nullptr; + } + + Optional> RegistryImplDeserializer::visit_object(Json::Reader& r, + const Json::Object& obj) + { + static Json::StringDeserializer kind_deserializer{"a registry implementation kind"}; + static Json::StringDeserializer baseline_deserializer{"a baseline"}; + std::string kind; + std::string baseline; + + r.required_object_field(type_name(), obj, KIND, kind, kind_deserializer); + r.optional_object_field(obj, BASELINE, baseline, baseline_deserializer); + + std::unique_ptr res; + + if (kind == KIND_BUILTIN) + { + r.check_for_unexpected_fields(obj, valid_builtin_fields(), "a builtin registry"); + res = std::make_unique(std::move(baseline)); + } + else if (kind == KIND_FILESYSTEM) + { + r.check_for_unexpected_fields(obj, valid_filesystem_fields(), "a filesystem registry"); + + fs::path path; + r.required_object_field("a filesystem registry", obj, PATH, path, Json::PathDeserializer::instance); + + res = std::make_unique(config_directory / path, std::move(baseline)); + } + else + { + StringLiteral valid_kinds[] = {KIND_BUILTIN, KIND_FILESYSTEM}; + r.add_generic_error(type_name(), + "Field \"kind\" did not have an expected value (expected one of: \"", + Strings::join("\", \"", valid_kinds), + "\", found \"", + kind, + "\")."); + return nullopt; + } + + return std::move(res); + } + View RegistryDeserializer::valid_fields() const { static const StringView t[] = { RegistryImplDeserializer::KIND, + RegistryImplDeserializer::BASELINE, RegistryImplDeserializer::PATH, PACKAGES, }; @@ -359,7 +714,7 @@ namespace vcpkg Optional RegistryDeserializer::visit_object(Json::Reader& r, const Json::Object& obj) { - auto impl = RegistryImplDeserializer::instance.visit_object(r, obj); + auto impl = impl_des.visit_object(r, obj); if (!impl.has_value()) { @@ -375,9 +730,159 @@ namespace vcpkg return Registry{std::move(packages), std::move(impl).value_or_exit(VCPKG_LINE_INFO)}; } + fs::path relative_path_to_versions(StringView port_name) + { + auto port_filename = fs::u8path(port_name.to_string() + ".json"); + return fs::u8path({port_name.byte_at_index(0), '-'}) / port_filename; + } + + ExpectedS> load_versions_file(Files::Filesystem& fs, + VersionDbType type, + const fs::path& port_versions, + StringView port_name, + const fs::path& registry_root) + { + Checks::check_exit(VCPKG_LINE_INFO, + !(type == VersionDbType::Filesystem && registry_root.empty()), + "Bug in vcpkg; type should never = Filesystem when registry_root is empty."); + + auto versions_file_path = port_versions / relative_path_to_versions(port_name); + + if (!fs.exists(versions_file_path)) + { + return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path)); + } + + auto maybe_contents = fs.read_contents(versions_file_path); + if (!maybe_contents.has_value()) + { + return Strings::format("Failed to load the versions database file %s: %s", + fs::u8string(versions_file_path), + maybe_contents.error().message()); + } + + auto maybe_versions_json = Json::parse(*maybe_contents.get()); + if (!maybe_versions_json.has_value()) + { + return Strings::format( + "Error: failed to parse versions file for `%s`: %s", port_name, maybe_versions_json.error()->format()); + } + if (!maybe_versions_json.get()->first.is_object()) + { + return Strings::format("Error: versions file for `%s` does not have a top level object.", port_name); + } + + const auto& versions_object = maybe_versions_json.get()->first.object(); + auto maybe_versions_array = versions_object.get("versions"); + if (!maybe_versions_array || !maybe_versions_array->is_array()) + { + return Strings::format("Error: versions file for `%s` does not contain a versions array.", port_name); + } + + std::vector db_entries; + VersionDbEntryArrayDeserializer deserializer{type, registry_root}; + // Avoid warning treated as error. + if (maybe_versions_array != nullptr) + { + Json::Reader r; + r.visit_in_key(*maybe_versions_array, "versions", db_entries, deserializer); + if (!r.errors().empty()) + { + return Strings::format( + "Error: failed to parse versions file for `%s`:\n%s", port_name, Strings::join("\n", r.errors())); + } + } + return db_entries; + } + + ExpectedS> parse_baseline_versions(StringView contents, StringView baseline) + { + auto maybe_value = Json::parse(contents); + if (!maybe_value.has_value()) + { + return Strings::format("Error: failed to parse baseline file: %s", maybe_value.error()->format()); + } + + auto& value = *maybe_value.get(); + + if (!value.first.is_object()) + { + return std::string("Error: baseline does not have a top-level object."); + } + + auto real_baseline = baseline.size() == 0 ? "default" : baseline; + + const auto& obj = value.first.object(); + auto baseline_value = obj.get(real_baseline); + if (!baseline_value) + { + return {nullopt, expected_left_tag}; + } + + Json::Reader r; + std::map> result; + r.visit_in_key(*baseline_value, real_baseline, result, BaselineDeserializer::instance); + if (r.errors().empty()) + { + return {std::move(result), expected_left_tag}; + } + else + { + Checks::exit_with_message( + VCPKG_LINE_INFO, "Error: failed to parse baseline:\n%s", Strings::join("\n", r.errors())); + } + } + + ExpectedS> load_baseline_versions(const VcpkgPaths& paths, + const fs::path& path_to_baseline, + StringView baseline) + { + auto maybe_contents = paths.get_filesystem().read_contents(path_to_baseline); + if (auto contents = maybe_contents.get()) + { + return parse_baseline_versions(*contents, baseline); + } + else if (maybe_contents.error() == std::errc::no_such_file_or_directory) + { + return {nullopt, expected_left_tag}; + } + else + { + return Strings::format("Error: failed to read file `%s`: %s", + fs::u8string(path_to_baseline), + maybe_contents.error().message()); + } + } +} + +namespace vcpkg +{ + std::unique_ptr>> + get_registry_implementation_deserializer(const fs::path& configuration_directory) + { + return std::make_unique(configuration_directory); + } + std::unique_ptr>> get_registry_array_deserializer( + const fs::path& configuration_directory) + { + return std::make_unique>( + "an array of registries", RegistryDeserializer(configuration_directory)); + } + + std::unique_ptr Registry::builtin_registry(std::string&& baseline) + { + return std::make_unique(std::move(baseline)); + } + + Registry::Registry(std::vector&& packages, std::unique_ptr&& impl) + : packages_(std::move(packages)), implementation_(std::move(impl)) + { + Checks::check_exit(VCPKG_LINE_INFO, implementation_ != nullptr); + } + RegistrySet::RegistrySet() : default_registry_(Registry::builtin_registry()), registries_() { } - const RegistryImpl* RegistrySet::registry_for_port(StringView name) const + const RegistryImplementation* RegistrySet::registry_for_port(StringView name) const { for (const auto& registry : registries()) { @@ -390,8 +895,63 @@ namespace vcpkg return default_registry(); } + Optional RegistrySet::baseline_for_port(const VcpkgPaths& paths, StringView port_name) const + { + auto impl = registry_for_port(port_name); + if (!impl) return nullopt; + return impl->get_baseline_version(paths, port_name); + } + void RegistrySet::add_registry(Registry&& r) { registries_.push_back(std::move(r)); } - void RegistrySet::set_default_registry(std::unique_ptr&& r) { default_registry_ = std::move(r); } + void RegistrySet::set_default_registry(std::unique_ptr&& r) + { + default_registry_ = std::move(r); + } void RegistrySet::set_default_registry(std::nullptr_t) { default_registry_.reset(); } + + void RegistrySet::experimental_set_builtin_registry_baseline(StringView baseline) const + { + // to check if we should warn + bool default_registry_is_builtin = false; + if (auto builtin_registry = dynamic_cast(default_registry_.get())) + { + default_registry_is_builtin = true; + builtin_registry->m_baseline_identifier.assign(baseline.begin(), baseline.end()); + } + + if (!default_registry_is_builtin || registries_.size() != 0) + { + System::print2(System::Color::warning, + "Warning: when using the registries feature, one should not use `\"$x-default-baseline\"` " + "to set the baseline.\n", + " Instead, use the \"baseline\" field of the registry."); + } + + for (auto& reg : registries_) + { + if (auto builtin_registry = dynamic_cast(reg.implementation_.get())) + { + builtin_registry->m_baseline_identifier.assign(baseline.begin(), baseline.end()); + } + } + } + + bool RegistrySet::has_modifications() const + { + if (!registries_.empty()) + { + return true; + } + if (auto builtin_reg = dynamic_cast(default_registry_.get())) + { + if (builtin_reg->m_baseline_identifier.empty()) + { + return false; + } + return true; + } + // default_registry_ is not a BuiltinRegistry + return true; + } } diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index 5b25b32d8a..0310a01ae3 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -277,6 +277,7 @@ namespace vcpkg {PACKAGES_ROOT_DIR_ARG, &VcpkgCmdArguments::packages_root_dir}, {SCRIPTS_ROOT_DIR_ARG, &VcpkgCmdArguments::scripts_root_dir}, {BUILTIN_PORTS_ROOT_DIR_ARG, &VcpkgCmdArguments::builtin_ports_root_dir}, + {BUILTIN_PORT_VERSIONS_DIR_ARG, &VcpkgCmdArguments::builtin_port_versions_dir}, }; constexpr static std::pair VcpkgCmdArguments::*> diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index 5c7e6a77e8..a412587617 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -89,9 +88,9 @@ namespace vcpkg const fs::path& filepath) { Json::Reader reader; - ConfigurationDeserializer deserializer(args); + auto deserializer = make_configuration_deserializer(filepath.parent_path()); - auto parsed_config_opt = reader.visit(obj, deserializer); + auto parsed_config_opt = reader.visit(obj, *deserializer); if (!reader.errors().empty()) { System::print2(System::Color::error, "Errors occurred while parsing ", fs::u8string(filepath), "\n"); @@ -103,6 +102,8 @@ namespace vcpkg Checks::exit_fail(VCPKG_LINE_INFO); } + parsed_config_opt.get()->validate_feature_flags(args.feature_flag_settings()); + return std::move(parsed_config_opt).value_or_exit(VCPKG_LINE_INFO); } @@ -342,6 +343,8 @@ If you wish to silence this error and use classic mode, you can: scripts = process_input_directory(filesystem, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO); builtin_ports = process_output_directory(filesystem, root, args.builtin_ports_root_dir.get(), "ports", VCPKG_LINE_INFO); + builtin_port_versions = process_output_directory( + filesystem, root, args.builtin_port_versions_dir.get(), "port_versions", VCPKG_LINE_INFO); prefab = root / fs::u8path("prefab"); if (args.default_visual_studio_path) @@ -493,11 +496,13 @@ If you wish to silence this error and use classic mode, you can: fs.remove_all(dot_git_dir, VCPKG_LINE_INFO); // All git commands are run with: --git-dir={dot_git_dir} --work-tree={work_tree_temp} - // git clone --no-checkout --local {vcpkg_root} {dot_git_dir} + // git clone --no-checkout --local --no-hardlinks {vcpkg_root} {dot_git_dir} + // note that `--no-hardlinks` is added because otherwise, git fails to clone in some cases System::CmdLineBuilder clone_cmd_builder = git_cmd_builder(paths, dot_git_dir, work_tree) .string_arg("clone") .string_arg("--no-checkout") .string_arg("--local") + .string_arg("--no-hardlinks") .path_arg(local_repo) .path_arg(dot_git_dir); const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract()); @@ -578,6 +583,7 @@ If you wish to silence this error and use classic mode, you can: .string_arg("clone") .string_arg("--no-checkout") .string_arg("--local") + .string_arg("--no-hardlinks") .path_arg(local_repo) .path_arg(dot_git_dir); const auto clone_output = System::cmd_execute_and_capture_output(clone_cmd_builder.extract()); @@ -662,8 +668,8 @@ If you wish to silence this error and use classic mode, you can: * Since we are checking a git tree object, all files will be checked out to the root of `work-tree`. * Because of that, it makes sense to use the git hash as the name for the directory. */ - const fs::path local_repo = this->root; - const fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name); + const fs::path& local_repo = this->root; + fs::path destination = this->versions_output / fs::u8path(git_tree) / fs::u8path(port_name); if (!fs.exists(destination / "CONTROL") && !fs.exists(destination / "vcpkg.json")) { diff --git a/toolsrc/src/vcpkg/versiondeserializers.cpp b/toolsrc/src/vcpkg/versiondeserializers.cpp index ee64630e32..d791e3e145 100644 --- a/toolsrc/src/vcpkg/versiondeserializers.cpp +++ b/toolsrc/src/vcpkg/versiondeserializers.cpp @@ -146,75 +146,6 @@ namespace vcpkg namespace { - struct VersionDbEntryDeserializer final : Json::IDeserializer - { - static constexpr StringLiteral GIT_TREE = "git-tree"; - - StringView type_name() const override { return "a version database entry"; } - View valid_fields() const override - { - static const StringView u[] = {GIT_TREE}; - static const auto t = vcpkg::Util::Vectors::concat(schemed_deserializer_fields(), u); - return t; - } - - Optional visit_object(Json::Reader& r, const Json::Object& obj) override - { - VersionDbEntry ret; - - auto schemed_version = visit_required_schemed_deserializer(type_name(), r, obj); - ret.scheme = schemed_version.scheme; - ret.version = std::move(schemed_version.versiont); - - static Json::StringDeserializer git_tree_deserializer("a git object SHA"); - - r.required_object_field(type_name(), obj, GIT_TREE, ret.git_tree, git_tree_deserializer); - - return std::move(ret); - } - - static VersionDbEntryDeserializer instance; - }; - - struct VersionDbEntryArrayDeserializer final : Json::IDeserializer> - { - virtual StringView type_name() const override { return "an array of versions"; } - - virtual Optional> visit_array(Json::Reader& r, const Json::Array& arr) override - { - return r.array_elements(arr, VersionDbEntryDeserializer::instance); - } - - static VersionDbEntryArrayDeserializer instance; - }; - - VersionDbEntryDeserializer VersionDbEntryDeserializer::instance; - VersionDbEntryArrayDeserializer VersionDbEntryArrayDeserializer::instance; - - struct BaselineDeserializer final : Json::IDeserializer>> - { - StringView type_name() const override { return "a baseline object"; } - - Optional visit_object(Json::Reader& r, const Json::Object& obj) override - { - std::map> result; - - for (auto&& pr : obj) - { - const auto& version_value = pr.second; - VersionT version; - r.visit_in_key(version_value, pr.first, version, get_versiont_deserializer_instance()); - - result.emplace(pr.first.to_string(), std::move(version)); - } - - return std::move(result); - } - - static BaselineDeserializer instance; - }; - BaselineDeserializer BaselineDeserializer::instance; - struct VersionTDeserializer final : Json::IDeserializer { StringView type_name() const override { return "a version object"; } @@ -245,71 +176,4 @@ namespace namespace vcpkg { Json::IDeserializer& get_versiont_deserializer_instance() { return VersionTDeserializer::instance; } - - ExpectedS>> parse_baseline_file(Files::Filesystem& fs, - StringView baseline_name, - const fs::path& baseline_file_path) - { - if (!fs.exists(baseline_file_path)) - { - return Strings::format("Couldn't find `%s`", fs::u8string(baseline_file_path)); - } - - auto value = Json::parse_file(VCPKG_LINE_INFO, fs, baseline_file_path); - if (!value.first.is_object()) - { - return Strings::format("Error: `%s` does not have a top-level object.", fs::u8string(baseline_file_path)); - } - - const auto& obj = value.first.object(); - auto baseline_value = obj.get(baseline_name); - if (!baseline_value) - { - return Strings::format( - "Error: `%s` does not contain the baseline \"%s\"", fs::u8string(baseline_file_path), baseline_name); - } - - Json::Reader r; - std::map> result; - r.visit_in_key(*baseline_value, baseline_name, result, BaselineDeserializer::instance); - if (!r.errors().empty()) - { - return Strings::format( - "Error: failed to parse `%s`:\n%s", fs::u8string(baseline_file_path), Strings::join("\n", r.errors())); - } - return result; - } - - ExpectedS> parse_versions_file(Files::Filesystem& fs, - StringView port_name, - const fs::path& versions_file_path) - { - (void)port_name; - if (!fs.exists(versions_file_path)) - { - return Strings::format("Couldn't find the versions database file: %s", fs::u8string(versions_file_path)); - } - - auto versions_json = Json::parse_file(VCPKG_LINE_INFO, fs, versions_file_path); - if (!versions_json.first.is_object()) - { - return Strings::format("Error: `%s` does not have a top level object.", fs::u8string(versions_file_path)); - } - - const auto& versions_object = versions_json.first.object(); - auto maybe_versions_array = versions_object.get("versions"); - if (!maybe_versions_array || !maybe_versions_array->is_array()) - { - return Strings::format("Error: `%s` does not contain a versions array.", fs::u8string(versions_file_path)); - } - - std::vector db_entries; - // Avoid warning treated as error. - if (maybe_versions_array != nullptr) - { - Json::Reader r; - r.visit_in_key(*maybe_versions_array, "versions", db_entries, VersionDbEntryArrayDeserializer::instance); - } - return db_entries; - } } diff --git a/toolsrc/src/vcpkg/versions.cpp b/toolsrc/src/vcpkg/versions.cpp index 5ea2a8182e..239c0e7285 100644 --- a/toolsrc/src/vcpkg/versions.cpp +++ b/toolsrc/src/vcpkg/versions.cpp @@ -244,4 +244,4 @@ namespace vcpkg::Versions Checks::unreachable(VCPKG_LINE_INFO); } -} \ No newline at end of file +}