From 2a42024b53ebb512fb5dd63c523338bf26c8489c Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Thu, 7 Jan 2021 18:04:11 -0800 Subject: [PATCH] [vcpkg] Add commands to maintain and verify versions db integrity (#14999) * [vcpkg] Add x-ci-verify-versions command * Code cleanup * Remove port version splitting from x-history * Fix wrong message on success * Parallelize versions file generator * Use cpu_count()/2 to avoid crashes * Check db SHA against local SHA * Check baseline version with x-ci-verify-versions and make baseline generator much faster * Implement x-add-version to update version db files * Better checks for x-add-info and make x-ci-verify-versions silent on success * Use find() instead of [] on maps * Create version file if does not exist * Allow redirection of ports/ and port_versions/ * add test ports * WIP end-to-end tests * Change pats in e2e tests * Fix e2e args * e2e once more * Pass ersions feature flag to e2e * Exit with code 1 if there are errors * Files to test for failure cases * Update test files * Add test for x-add-version * fix redirected ports in last test * Add CI check (use dummy data) * Add feature-flags=versions * Ignore subdirectories inside ports * Add --verify-git-trees switch * [vcpkg] Fix build breaks * [x-ci-verify-versions] PR comments * [x-add-version] PR comments * Fix merge conflicts * Modify tests and pipeline * Baselines should only have version-string * Refactor x-add-version * [vcpkg] Fix help message * [vcpkg] Fix minor warnings * `x-add-version --all` doesn't stop on first failure and reduced default verbosity * [vcpkg] Fix default-baseline * Load file instead of using paths provider * Format * Remove ci test * Add fish port for testing * Update version files * Update fish port to cause SHA discrepancy * Test for discrepancy between local SHA and declared SHA * Missing = operator * Check for error message since x-add-version exits with code 0 * Make x-add-version fail with non-zero exit code if not run with --all Co-authored-by: Robert Schumacher Co-authored-by: Robert Schumacher --- .../end-to-end-tests-dir/versions.ps1 | 75 ++++ .../default-baseline-1/vcpkg.json | 8 + .../port_versions/z-/zlib.json | 14 + .../default-baseline-2/vcpkg.json | 8 + .../version-files/port_versions/baseline.json | 20 + .../version-files/port_versions/c-/cat.json | 9 + .../version-files/port_versions/d-/dog.json | 9 + .../version-files/port_versions/d-/duck.json | 9 + .../version-files/port_versions/m-/mouse.json | 9 + .../port_versions_incomplete/baseline.json | 20 + .../port_versions_incomplete/c-/cat.json | 9 + .../port_versions_incomplete/d-/dog.json | 9 + .../port_versions_incomplete/f-/fish.json | 9 + .../port_versions_incomplete/m-/mouse.json | 9 + .../version-files/ports/cat/portfile.cmake | 1 + .../version-files/ports/cat/vcpkg.json | 4 + .../version-files/ports/dog/portfile.cmake | 1 + .../version-files/ports/dog/vcpkg.json | 4 + .../version-files/ports/duck/portfile.cmake | 1 + .../version-files/ports/duck/vcpkg.json | 4 + .../version-files/ports/mouse/portfile.cmake | 1 + .../version-files/ports/mouse/vcpkg.json | 4 + .../ports_incomplete/cat/portfile.cmake | 1 + .../ports_incomplete/cat/vcpkg.json | 4 + .../ports_incomplete/dog/portfile.cmake | 1 + .../ports_incomplete/dog/vcpkg.json | 5 + .../ports_incomplete/duck/portfile.cmake | 1 + .../ports_incomplete/duck/vcpkg.json | 4 + .../ports_incomplete/ferret/portfile.cmake | 1 + .../ports_incomplete/ferret/vcpkg.json | 4 + .../ports_incomplete/fish/portfile.cmake | 1 + .../ports_incomplete/fish/vcpkg.json | 5 + .../ports_incomplete/mouse/portfile.cmake | 1 + .../ports_incomplete/mouse/vcpkg.json | 4 + toolsrc/include/vcpkg/commands.add-version.h | 13 + .../include/vcpkg/commands.civerifyversions.h | 13 + toolsrc/include/vcpkg/registries.h | 7 + toolsrc/include/vcpkg/vcpkgpaths.h | 2 + toolsrc/src/vcpkg-test/arguments.cpp | 4 + toolsrc/src/vcpkg-test/commands.cpp | 2 + toolsrc/src/vcpkg/commands.add-version.cpp | 383 ++++++++++++++++++ .../src/vcpkg/commands.civerifyversions.cpp | 328 +++++++++++++++ toolsrc/src/vcpkg/commands.cpp | 6 + toolsrc/src/vcpkg/commands.porthistory.cpp | 6 - toolsrc/src/vcpkg/portfileprovider.cpp | 8 +- toolsrc/src/vcpkg/registries.cpp | 86 ++-- toolsrc/src/vcpkg/vcpkgcmdarguments.cpp | 5 + toolsrc/src/vcpkg/vcpkgpaths.cpp | 47 ++- toolsrc/windows-bootstrap/vcpkg.vcxproj | 4 + 49 files changed, 1146 insertions(+), 37 deletions(-) create mode 100644 scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1 create mode 100644 scripts/testing/version-files/default-baseline-1/vcpkg.json create mode 100644 scripts/testing/version-files/default-baseline-2/port_versions/z-/zlib.json create mode 100644 scripts/testing/version-files/default-baseline-2/vcpkg.json create mode 100644 scripts/testing/version-files/port_versions/baseline.json create mode 100644 scripts/testing/version-files/port_versions/c-/cat.json create mode 100644 scripts/testing/version-files/port_versions/d-/dog.json create mode 100644 scripts/testing/version-files/port_versions/d-/duck.json create mode 100644 scripts/testing/version-files/port_versions/m-/mouse.json create mode 100644 scripts/testing/version-files/port_versions_incomplete/baseline.json create mode 100644 scripts/testing/version-files/port_versions_incomplete/c-/cat.json create mode 100644 scripts/testing/version-files/port_versions_incomplete/d-/dog.json create mode 100644 scripts/testing/version-files/port_versions_incomplete/f-/fish.json create mode 100644 scripts/testing/version-files/port_versions_incomplete/m-/mouse.json create mode 100644 scripts/testing/version-files/ports/cat/portfile.cmake create mode 100644 scripts/testing/version-files/ports/cat/vcpkg.json create mode 100644 scripts/testing/version-files/ports/dog/portfile.cmake create mode 100644 scripts/testing/version-files/ports/dog/vcpkg.json create mode 100644 scripts/testing/version-files/ports/duck/portfile.cmake create mode 100644 scripts/testing/version-files/ports/duck/vcpkg.json create mode 100644 scripts/testing/version-files/ports/mouse/portfile.cmake create mode 100644 scripts/testing/version-files/ports/mouse/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/cat/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/cat/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/dog/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/dog/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/duck/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/duck/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/ferret/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/ferret/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/fish/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/fish/vcpkg.json create mode 100644 scripts/testing/version-files/ports_incomplete/mouse/portfile.cmake create mode 100644 scripts/testing/version-files/ports_incomplete/mouse/vcpkg.json create mode 100644 toolsrc/include/vcpkg/commands.add-version.h create mode 100644 toolsrc/include/vcpkg/commands.civerifyversions.h create mode 100644 toolsrc/src/vcpkg/commands.add-version.cpp create mode 100644 toolsrc/src/vcpkg/commands.civerifyversions.cpp diff --git a/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1 b/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1 new file mode 100644 index 0000000000..5d61b2101d --- /dev/null +++ b/scripts/azure-pipelines/end-to-end-tests-dir/versions.ps1 @@ -0,0 +1,75 @@ +. $PSScriptRoot/../end-to-end-tests-prelude.ps1 + +# Test verify versions +mkdir $VersionFilesRoot +Copy-Item -Recurse "scripts/testing/version-files/port_versions_incomplete" $VersionFilesRoot +$portsRedirectArgsOK = @( + "--feature-flags=versions", + "--x-builtin-ports-root=scripts/testing/version-files/ports", + "--x-builtin-port-versions-dir=scripts/testing/version-files/port_versions" +) +$portsRedirectArgsIncomplete = @( + "--feature-flags=versions", + "--x-builtin-ports-root=scripts/testing/version-files/ports_incomplete", + "--x-builtin-port-versions-dir=$VersionFilesRoot/port_versions_incomplete" +) +$CurrentTest = "x-verify-ci-versions (All files OK)" +Write-Host $CurrentTest +./vcpkg $portsRedirectArgsOK x-ci-verify-versions --verbose +Throw-IfFailed + +$CurrentTest = "x-verify-ci-versions (Incomplete)" +./vcpkg $portsRedirectArgsIncomplete x-ci-verify-versions --verbose +Throw-IfNotFailed + +$CurrentTest = "x-add-version cat" +# Do not fail if there's nothing to update +./vcpkg $portsRedirectArgsIncomplete x-add-version cat +Throw-IfFailed + +$CurrentTest = "x-add-version dog" +# Local version is not in baseline and versions file +./vcpkg $portsRedirectArgsIncomplete x-add-version dog +Throw-IfFailed + +$CurrentTest = "x-add-version duck" +# Missing versions file +./vcpkg $portsRedirectArgsIncomplete x-add-version duck +Throw-IfFailed + +$CurrentTest = "x-add-version ferret" +# Missing versions file and missing baseline entry +./vcpkg $portsRedirectArgsIncomplete x-add-version ferret +Throw-IfFailed + +$CurrentTest = "x-add-version fish (must fail)" +# Discrepancy between local SHA and SHA in fish.json. Requires --overwrite-version. +$out = ./vcpkg $portsRedirectArgsIncomplete x-add-version fish +Throw-IfNotFailed +$CurrentTest = "x-add-version fish --overwrite-version" +./vcpkg $portsRedirectArgsIncomplete x-add-version fish --overwrite-version +Throw-IfFailed + +$CurrentTest = "x-add-version mouse" +# Missing baseline entry +./vcpkg $portsRedirectArgsIncomplete x-add-version mouse +Throw-IfFailed +# Validate changes +./vcpkg $portsRedirectArgsIncomplete x-ci-verify-versions --verbose +Throw-IfFailed + +$CurrentTest = "default baseline" +$out = ./vcpkg $commonArgs "--feature-flags=versions" install --x-manifest-root=scripts/testing/version-files/default-baseline-1 +Throw-IfNotFailed +# if ($out -notmatch "Error: while checking out baseline" -or $out -notmatch " does not exist in ") +# { +# $out +# throw "Expected to fail due to missing baseline" +# } + +git fetch https://github.com/vicroms/test-registries +$CurrentTest = "default baseline" +./vcpkg $commonArgs "--feature-flags=versions" install ` + "--x-manifest-root=scripts/testing/version-files/default-baseline-2" ` + "--x-builtin-port-versions-dir=scripts/testing/version-files/default-baseline-2/port_versions" +Throw-IfFailed diff --git a/scripts/testing/version-files/default-baseline-1/vcpkg.json b/scripts/testing/version-files/default-baseline-1/vcpkg.json new file mode 100644 index 0000000000..f6d902393d --- /dev/null +++ b/scripts/testing/version-files/default-baseline-1/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "default-baseline-test", + "version-string": "0", + "$x-default-baseline": "fca18ba3572f8aebe3b8158c359db62a7e26134e", + "dependencies": [ + "zlib" + ] +} \ No newline at end of file diff --git a/scripts/testing/version-files/default-baseline-2/port_versions/z-/zlib.json b/scripts/testing/version-files/default-baseline-2/port_versions/z-/zlib.json new file mode 100644 index 0000000000..f5ee7cb9dc --- /dev/null +++ b/scripts/testing/version-files/default-baseline-2/port_versions/z-/zlib.json @@ -0,0 +1,14 @@ +{ + "versions": [ + { + "git-tree": "7bb2b2f3783303a4dd41163553fe4cc103dc9262", + "version-string": "1.2.11", + "port-version": 9 + }, + { + "git-tree": "4927735fa9baca564ebddf6e6880de344b20d7a8", + "version-string": "1.2.11", + "port-version": 8 + } + ] +} \ No newline at end of file diff --git a/scripts/testing/version-files/default-baseline-2/vcpkg.json b/scripts/testing/version-files/default-baseline-2/vcpkg.json new file mode 100644 index 0000000000..1e41002ff9 --- /dev/null +++ b/scripts/testing/version-files/default-baseline-2/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "default-baseline-test", + "version-string": "0", + "$x-default-baseline": "cbd5a68012471f820b7cf28d618199b4a4d89c58", + "dependencies": [ + "zlib" + ] +} \ No newline at end of file diff --git a/scripts/testing/version-files/port_versions/baseline.json b/scripts/testing/version-files/port_versions/baseline.json new file mode 100644 index 0000000000..bfe49f9846 --- /dev/null +++ b/scripts/testing/version-files/port_versions/baseline.json @@ -0,0 +1,20 @@ +{ + "default": { + "cat": { + "version-string": "1.0", + "port-version": 0 + }, + "dog": { + "version-string": "2001-01-01", + "port-version": 0 + }, + "duck": { + "version-string": "mallard", + "port-version": 0 + }, + "mouse": { + "version-string": "1.0.0", + "port-version": 0 + } + } +} diff --git a/scripts/testing/version-files/port_versions/c-/cat.json b/scripts/testing/version-files/port_versions/c-/cat.json new file mode 100644 index 0000000000..e39cd6d4e8 --- /dev/null +++ b/scripts/testing/version-files/port_versions/c-/cat.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "5dd257451526d5b9e560f5f35d7029ba40d88587", + "version": "1.0", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions/d-/dog.json b/scripts/testing/version-files/port_versions/d-/dog.json new file mode 100644 index 0000000000..49c086c389 --- /dev/null +++ b/scripts/testing/version-files/port_versions/d-/dog.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "e170a2ed0da7ba5d434c4a0a98ffd7a3159e3200", + "version-date": "2001-01-01", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions/d-/duck.json b/scripts/testing/version-files/port_versions/d-/duck.json new file mode 100644 index 0000000000..4f4b209e22 --- /dev/null +++ b/scripts/testing/version-files/port_versions/d-/duck.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "0a52a9d722c75b3bfe47d5f5db6c9eb1a64af156", + "version-string": "mallard", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions/m-/mouse.json b/scripts/testing/version-files/port_versions/m-/mouse.json new file mode 100644 index 0000000000..139c8450fa --- /dev/null +++ b/scripts/testing/version-files/port_versions/m-/mouse.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "55ed624191e0a1905bd97af29fdf6a1d7f4e6d7c", + "version-semver": "1.0.0", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions_incomplete/baseline.json b/scripts/testing/version-files/port_versions_incomplete/baseline.json new file mode 100644 index 0000000000..aadd634fc9 --- /dev/null +++ b/scripts/testing/version-files/port_versions_incomplete/baseline.json @@ -0,0 +1,20 @@ +{ + "default": { + "cat": { + "version-string": "1.0", + "port-version": 0 + }, + "dog": { + "version-string": "2001-01-01", + "port-version": 0 + }, + "duck": { + "version-string": "mallard", + "port-version": 0 + }, + "fish": { + "version-string": "1.0.0", + "port-version": 0 + } + } +} diff --git a/scripts/testing/version-files/port_versions_incomplete/c-/cat.json b/scripts/testing/version-files/port_versions_incomplete/c-/cat.json new file mode 100644 index 0000000000..e39cd6d4e8 --- /dev/null +++ b/scripts/testing/version-files/port_versions_incomplete/c-/cat.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "5dd257451526d5b9e560f5f35d7029ba40d88587", + "version": "1.0", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions_incomplete/d-/dog.json b/scripts/testing/version-files/port_versions_incomplete/d-/dog.json new file mode 100644 index 0000000000..49c086c389 --- /dev/null +++ b/scripts/testing/version-files/port_versions_incomplete/d-/dog.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "e170a2ed0da7ba5d434c4a0a98ffd7a3159e3200", + "version-date": "2001-01-01", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/port_versions_incomplete/f-/fish.json b/scripts/testing/version-files/port_versions_incomplete/f-/fish.json new file mode 100644 index 0000000000..dc457aabde --- /dev/null +++ b/scripts/testing/version-files/port_versions_incomplete/f-/fish.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "cf3be634f203c1b4152b65ec7700d5695a1fca5c", + "version-string": "1.0.0", + "port-version": 0 + } + ] +} \ No newline at end of file diff --git a/scripts/testing/version-files/port_versions_incomplete/m-/mouse.json b/scripts/testing/version-files/port_versions_incomplete/m-/mouse.json new file mode 100644 index 0000000000..139c8450fa --- /dev/null +++ b/scripts/testing/version-files/port_versions_incomplete/m-/mouse.json @@ -0,0 +1,9 @@ +{ + "versions": [ + { + "git-tree": "55ed624191e0a1905bd97af29fdf6a1d7f4e6d7c", + "version-semver": "1.0.0", + "port-version": 0 + } + ] +} diff --git a/scripts/testing/version-files/ports/cat/portfile.cmake b/scripts/testing/version-files/ports/cat/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports/cat/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports/cat/vcpkg.json b/scripts/testing/version-files/ports/cat/vcpkg.json new file mode 100644 index 0000000000..b87e4a2dac --- /dev/null +++ b/scripts/testing/version-files/ports/cat/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "cat", + "version": "1.0" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports/dog/portfile.cmake b/scripts/testing/version-files/ports/dog/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports/dog/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports/dog/vcpkg.json b/scripts/testing/version-files/ports/dog/vcpkg.json new file mode 100644 index 0000000000..028447b3e3 --- /dev/null +++ b/scripts/testing/version-files/ports/dog/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "dog", + "version-date": "2001-01-01" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports/duck/portfile.cmake b/scripts/testing/version-files/ports/duck/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports/duck/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports/duck/vcpkg.json b/scripts/testing/version-files/ports/duck/vcpkg.json new file mode 100644 index 0000000000..d780d01bfc --- /dev/null +++ b/scripts/testing/version-files/ports/duck/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "duck", + "version-string": "mallard" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports/mouse/portfile.cmake b/scripts/testing/version-files/ports/mouse/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports/mouse/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports/mouse/vcpkg.json b/scripts/testing/version-files/ports/mouse/vcpkg.json new file mode 100644 index 0000000000..fd6cab451f --- /dev/null +++ b/scripts/testing/version-files/ports/mouse/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "mouse", + "version-semver": "1.0.0" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/cat/portfile.cmake b/scripts/testing/version-files/ports_incomplete/cat/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/cat/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/cat/vcpkg.json b/scripts/testing/version-files/ports_incomplete/cat/vcpkg.json new file mode 100644 index 0000000000..b87e4a2dac --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/cat/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "cat", + "version": "1.0" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/dog/portfile.cmake b/scripts/testing/version-files/ports_incomplete/dog/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/dog/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/dog/vcpkg.json b/scripts/testing/version-files/ports_incomplete/dog/vcpkg.json new file mode 100644 index 0000000000..65465b9408 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/dog/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "dog", + "version-date": "2001-01-01", + "port-version": 1 +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/duck/portfile.cmake b/scripts/testing/version-files/ports_incomplete/duck/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/duck/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/duck/vcpkg.json b/scripts/testing/version-files/ports_incomplete/duck/vcpkg.json new file mode 100644 index 0000000000..d780d01bfc --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/duck/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "duck", + "version-string": "mallard" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/ferret/portfile.cmake b/scripts/testing/version-files/ports_incomplete/ferret/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/ferret/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/ferret/vcpkg.json b/scripts/testing/version-files/ports_incomplete/ferret/vcpkg.json new file mode 100644 index 0000000000..1b4b5c2c1b --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/ferret/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "ferret", + "version": "1" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/fish/portfile.cmake b/scripts/testing/version-files/ports_incomplete/fish/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/fish/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/fish/vcpkg.json b/scripts/testing/version-files/ports_incomplete/fish/vcpkg.json new file mode 100644 index 0000000000..f67c10a5a0 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/fish/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "fish", + "version-string": "1.0.0", + "description": "This description causes an intentional discrepancy between the local SHA and the SHA in fish.json for version 1.0.0" +} \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/mouse/portfile.cmake b/scripts/testing/version-files/ports_incomplete/mouse/portfile.cmake new file mode 100644 index 0000000000..0015715fb6 --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/mouse/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/scripts/testing/version-files/ports_incomplete/mouse/vcpkg.json b/scripts/testing/version-files/ports_incomplete/mouse/vcpkg.json new file mode 100644 index 0000000000..fd6cab451f --- /dev/null +++ b/scripts/testing/version-files/ports_incomplete/mouse/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "mouse", + "version-semver": "1.0.0" +} \ No newline at end of file diff --git a/toolsrc/include/vcpkg/commands.add-version.h b/toolsrc/include/vcpkg/commands.add-version.h new file mode 100644 index 0000000000..8b9e0336e9 --- /dev/null +++ b/toolsrc/include/vcpkg/commands.add-version.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace vcpkg::Commands::AddVersion +{ + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths); + + struct AddVersionCommand : PathsCommand + { + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) const override; + }; +} \ No newline at end of file diff --git a/toolsrc/include/vcpkg/commands.civerifyversions.h b/toolsrc/include/vcpkg/commands.civerifyversions.h new file mode 100644 index 0000000000..17d8f56139 --- /dev/null +++ b/toolsrc/include/vcpkg/commands.civerifyversions.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace vcpkg::Commands::CIVerifyVersions +{ + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths); + + struct CIVerifyVersionsCommand : PathsCommand + { + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) const override; + }; +} \ No newline at end of file diff --git a/toolsrc/include/vcpkg/registries.h b/toolsrc/include/vcpkg/registries.h index 006bc789d1..c36c0179dc 100644 --- a/toolsrc/include/vcpkg/registries.h +++ b/toolsrc/include/vcpkg/registries.h @@ -9,8 +9,10 @@ #include #include +#include #include +#include #include #include #include @@ -103,4 +105,9 @@ namespace vcpkg std::unique_ptr>> get_registry_array_deserializer( const fs::path& configuration_directory); + + ExpectedS>> get_builtin_versions(const VcpkgPaths& paths, + StringView port_name); + + ExpectedS>> get_builtin_baseline(const VcpkgPaths& paths); } diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index 1f562c1517..a175d51e21 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -122,6 +122,8 @@ namespace vcpkg fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const; ExpectedS git_show(const std::string& treeish, const fs::path& dot_git_dir) const; + ExpectedS>> git_get_local_port_treeish_map() const; + Optional get_manifest() const; Optional get_manifest_path() const; const Configuration& get_configuration() const; diff --git a/toolsrc/src/vcpkg-test/arguments.cpp b/toolsrc/src/vcpkg-test/arguments.cpp index f5cbf7b15e..27fa74b8df 100644 --- a/toolsrc/src/vcpkg-test/arguments.cpp +++ b/toolsrc/src/vcpkg-test/arguments.cpp @@ -15,6 +15,7 @@ TEST_CASE ("VcpkgCmdArguments from lowercase argument sequence", "[arguments]") "C:\\vcpkg", "--x-scripts-root=C:\\scripts", "--x-builtin-ports-root=C:\\ports", + "--x-builtin-port-versions-dir=C:\\port_versions", "--debug", "--sendmetrics", "--printmetrics", @@ -27,6 +28,7 @@ TEST_CASE ("VcpkgCmdArguments from lowercase argument sequence", "[arguments]") REQUIRE(*v.vcpkg_root_dir == "C:\\vcpkg"); REQUIRE(*v.scripts_root_dir == "C:\\scripts"); REQUIRE(*v.builtin_ports_root_dir == "C:\\ports"); + REQUIRE(*v.builtin_port_versions_dir == "C:\\port_versions"); REQUIRE(v.debug); REQUIRE(*v.debug.get()); REQUIRE(v.send_metrics); @@ -49,6 +51,7 @@ TEST_CASE ("VcpkgCmdArguments from uppercase argument sequence", "[arguments]") "C:\\vcpkg", "--X-SCRIPTS-ROOT=C:\\scripts", "--X-BUILTIN-PORTS-ROOT=C:\\ports", + "--X-BUILTIN-PORT-VERSIONS-DIR=C:\\port_versions", "--DEBUG", "--SENDMETRICS", "--PRINTMETRICS", @@ -61,6 +64,7 @@ TEST_CASE ("VcpkgCmdArguments from uppercase argument sequence", "[arguments]") REQUIRE(*v.vcpkg_root_dir == "C:\\vcpkg"); REQUIRE(*v.scripts_root_dir == "C:\\scripts"); REQUIRE(*v.builtin_ports_root_dir == "C:\\ports"); + REQUIRE(*v.builtin_port_versions_dir == "C:\\port_versions"); REQUIRE(v.debug); REQUIRE(*v.debug.get()); REQUIRE(v.send_metrics); diff --git a/toolsrc/src/vcpkg-test/commands.cpp b/toolsrc/src/vcpkg-test/commands.cpp index 38346291ce..e417505ad1 100644 --- a/toolsrc/src/vcpkg-test/commands.cpp +++ b/toolsrc/src/vcpkg-test/commands.cpp @@ -60,6 +60,8 @@ TEST_CASE ("get_available_paths_commands works", "[commands]") "x-history", "x-package-info", "x-vsinstances", + "x-ci-verify-versions", + "x-add-version", }); } diff --git a/toolsrc/src/vcpkg/commands.add-version.cpp b/toolsrc/src/vcpkg/commands.add-version.cpp new file mode 100644 index 0000000000..c650fa9dd2 --- /dev/null +++ b/toolsrc/src/vcpkg/commands.add-version.cpp @@ -0,0 +1,383 @@ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vcpkg; + +namespace +{ + using VersionGitTree = std::pair; + + void insert_version_to_json_object(Json::Object& obj, const VersionT& version) + { + obj.insert("version-string", Json::Value::string(version.text())); + obj.insert("port-version", Json::Value::integer(version.port_version())); + } + + static Json::Object serialize_baseline(const std::map>& baseline) + { + Json::Object port_entries_obj; + for (auto&& kv_pair : baseline) + { + Json::Object baseline_version_obj; + insert_version_to_json_object(baseline_version_obj, kv_pair.second); + port_entries_obj.insert(kv_pair.first, baseline_version_obj); + } + + Json::Object baseline_obj; + baseline_obj.insert("default", port_entries_obj); + return baseline_obj; + } + + static Json::Object serialize_versions(const std::vector& versions) + { + Json::Array versions_array; + for (auto&& version : versions) + { + Json::Object version_obj; + version_obj.insert("git-tree", Json::Value::string(version.second)); + insert_version_to_json_object(version_obj, version.first); + versions_array.push_back(std::move(version_obj)); + } + + Json::Object output_object; + output_object.insert("versions", versions_array); + return output_object; + } + + static void write_baseline_file(Files::Filesystem& fs, + const std::map>& baseline_map, + const fs::path& output_path) + { + auto backup_path = fs::u8path(Strings::concat(fs::u8string(output_path), ".backup")); + if (fs.exists(output_path)) + { + fs.rename(output_path, backup_path, VCPKG_LINE_INFO); + fs.remove(output_path, VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.write_contents(output_path, Json::stringify(serialize_baseline(baseline_map), {}), ec); + if (ec) + { + System::printf( + System::Color::error, "Error: Couldn't write baseline file to %s.", fs::u8string(output_path)); + if (fs.exists(backup_path)) + { + fs.rename(backup_path, output_path, VCPKG_LINE_INFO); + } + Checks::exit_fail(VCPKG_LINE_INFO); + } + if (fs.exists(backup_path)) + { + fs.remove(backup_path, VCPKG_LINE_INFO); + } + } + + static void write_versions_file(Files::Filesystem& fs, + const std::vector& versions, + const fs::path& output_path) + { + auto backup_path = fs::u8path(Strings::concat(fs::u8string(output_path), ".backup")); + if (fs.exists(output_path)) + { + fs.rename(output_path, backup_path, VCPKG_LINE_INFO); + fs.remove(output_path, VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.create_directories(output_path.parent_path(), VCPKG_LINE_INFO); + fs.write_contents(output_path, Json::stringify(serialize_versions(versions), {}), ec); + if (ec) + { + System::printf( + System::Color::error, "Error: Couldn't write versions file to %s.", fs::u8string(output_path)); + if (fs.exists(backup_path)) + { + fs.rename(backup_path, output_path, VCPKG_LINE_INFO); + } + Checks::exit_fail(VCPKG_LINE_INFO); + } + if (fs.exists(backup_path)) + { + fs.remove(backup_path, VCPKG_LINE_INFO); + } + } + + static void update_baseline_version(const VcpkgPaths& paths, + const std::string& port_name, + const VersionT& version, + const fs::path& baseline_path, + bool print_success) + { + bool is_new_file = false; + auto& fs = paths.get_filesystem(); + auto baseline_map = [&]() -> std::map> { + if (!fs.exists(VCPKG_LINE_INFO, baseline_path)) + { + is_new_file = true; + std::map> ret; + return ret; + } + auto maybe_baseline_map = vcpkg::get_builtin_baseline(paths); + return maybe_baseline_map.value_or_exit(VCPKG_LINE_INFO); + }(); + + auto it = baseline_map.find(port_name); + if (it != baseline_map.end()) + { + auto& baseline_version = it->second; + if (baseline_version == version) + { + if (print_success) + { + System::printf(System::Color::success, + "Version `%s` is already in `%s`\n", + version, + fs::u8string(baseline_path)); + } + return; + } + baseline_version = version; + } + else + { + baseline_map.emplace(port_name, version); + } + + write_baseline_file(fs, baseline_map, baseline_path); + if (print_success) + { + System::printf(System::Color::success, + "Added version `%s` to `%s`%s.\n", + version.to_string(), + fs::u8string(baseline_path), + is_new_file ? " (new file)" : ""); + } + return; + } + + static void update_version_db_file(const VcpkgPaths& paths, + const std::string& port_name, + const VersionT& version, + const std::string& git_tree, + const fs::path& version_db_file_path, + bool overwrite_version, + bool print_success, + bool keep_going) + { + auto& fs = paths.get_filesystem(); + if (!fs.exists(VCPKG_LINE_INFO, version_db_file_path)) + { + std::vector new_entry{{version, git_tree}}; + write_versions_file(fs, new_entry, version_db_file_path); + if (print_success) + { + System::printf(System::Color::success, + "Added version `%s` to `%s` (new file).\n", + version.to_string(), + fs::u8string(version_db_file_path)); + } + return; + } + + auto maybe_versions = get_builtin_versions(paths, port_name); + if (auto versions = maybe_versions.get()) + { + const auto& versions_end = versions->end(); + + auto found_same_sha = std::find_if( + versions->begin(), versions_end, [&](auto&& entry) -> bool { return entry.second == git_tree; }); + if (found_same_sha != versions_end) + { + if (found_same_sha->first == version) + { + if (print_success) + { + System::printf(System::Color::success, + "Version `%s` is already in `%s`\n", + version.to_string(), + fs::u8string(version_db_file_path)); + } + return; + } + System::printf(System::Color::warning, + "Warning: Local port files SHA is the same as version `%s` in `%s`.\n" + "-- SHA: %s\n" + "-- Did you remember to commit your changes?\n" + "***No files were updated.***\n", + found_same_sha->first.to_string(), + fs::u8string(version_db_file_path), + git_tree); + if (keep_going) return; + Checks::exit_fail(VCPKG_LINE_INFO); + } + + auto it = std::find_if( + versions->begin(), versions_end, [&](auto&& entry) -> bool { return entry.first == version; }); + + if (it != versions_end) + { + if (!overwrite_version) + { + System::printf(System::Color::error, + "Error: Local changes detected for %s but no changes to version or port version.\n" + "-- Version: %s\n" + "-- Old SHA: %s\n" + "-- New SHA: %s\n" + "-- Did you remember to update the version or port version?\n" + "-- Pass `--overwrite-version` to bypass this check.\n" + "***No files were updated.***\n", + port_name, + version.to_string(), + it->second, + git_tree); + if (keep_going) return; + Checks::exit_fail(VCPKG_LINE_INFO); + } + + it->first = version; + it->second = git_tree; + } + else + { + versions->insert(versions->begin(), std::make_pair(version, git_tree)); + } + + write_versions_file(fs, *versions, version_db_file_path); + if (print_success) + { + System::printf(System::Color::success, + "Added version `%s` to `%s`.\n", + version.to_string(), + fs::u8string(version_db_file_path)); + } + return; + } + + System::printf(System::Color::error, + "Error: Unable to parse versions file %s.\n%s\n", + fs::u8string(version_db_file_path), + maybe_versions.error()); + Checks::exit_fail(VCPKG_LINE_INFO); + } +} + +namespace vcpkg::Commands::AddVersion +{ + static constexpr StringLiteral OPTION_ALL = "all"; + static constexpr StringLiteral OPTION_OVERWRITE_VERSION = "overwrite-version"; + static constexpr StringLiteral OPTION_VERBOSE = "verbose"; + + const CommandSwitch COMMAND_SWITCHES[] = { + {OPTION_ALL, "Process versions for all ports."}, + {OPTION_OVERWRITE_VERSION, "Overwrite `git-tree` of an existing version."}, + {OPTION_VERBOSE, "Print success messages instead of just errors."}, + }; + + const CommandStructure COMMAND_STRUCTURE{ + create_example_string(R"###(x-add-version )###"), + 0, + 1, + {{COMMAND_SWITCHES}, {}, {}}, + nullptr, + }; + + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) + { + auto parsed_args = args.parse_arguments(COMMAND_STRUCTURE); + const bool add_all = Util::Sets::contains(parsed_args.switches, OPTION_ALL); + const bool overwrite_version = Util::Sets::contains(parsed_args.switches, OPTION_OVERWRITE_VERSION); + const bool verbose = Util::Sets::contains(parsed_args.switches, OPTION_VERBOSE); + + auto& fs = paths.get_filesystem(); + auto baseline_path = paths.builtin_port_versions / fs::u8path("baseline.json"); + if (!fs.exists(VCPKG_LINE_INFO, baseline_path)) + { + System::printf( + System::Color::error, "Error: Couldn't find required file `%s`\n.", fs::u8string(baseline_path)); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + std::vector port_names; + if (!args.command_arguments.empty()) + { + if (add_all) + { + System::printf(System::Color::warning, + "Warning: Ignoring option `--%s` since a port name argument was provided.\n", + OPTION_ALL); + } + port_names.emplace_back(args.command_arguments[0]); + } + else + { + if (!add_all) + { + System::printf(System::Color::error, + "Error: Use option `--%s` to update version files for all ports at once.\n", + OPTION_ALL); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + for (auto&& port_dir : fs::directory_iterator(paths.builtin_ports_directory())) + { + port_names.emplace_back(fs::u8string(port_dir.path().stem())); + } + } + + // Get tree-ish from local repository state. + auto maybe_git_tree_map = paths.git_get_local_port_treeish_map(); + auto git_tree_map = maybe_git_tree_map.value_or_exit(VCPKG_LINE_INFO); + + for (auto&& port_name : port_names) + { + // Get version information of the local port + auto maybe_scf = Paragraphs::try_load_port(fs, paths.builtin_ports_directory() / fs::u8path(port_name)); + if (!maybe_scf.has_value()) + { + if (add_all) continue; + System::printf(System::Color::error, "Error: Couldn't load port `%s`.", port_name); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + const auto& scf = maybe_scf.value_or_exit(VCPKG_LINE_INFO); + const auto& versiont = scf->to_versiont(); + + auto git_tree_it = git_tree_map.find(port_name); + if (git_tree_it == git_tree_map.end()) + { + System::printf(System::Color::warning, + "Warning: No local Git SHA was found for port `%s`.\n" + "-- Did you remember to commit your changes?\n" + "***No files were updated.***\n", + port_name); + if (add_all) continue; + Checks::exit_fail(VCPKG_LINE_INFO); + } + const auto& git_tree = git_tree_it->second; + + auto port_versions_path = + paths.builtin_port_versions / Strings::concat(port_name[0], '-') / Strings::concat(port_name, ".json"); + update_version_db_file( + paths, port_name, versiont, git_tree, port_versions_path, overwrite_version, verbose, add_all); + update_baseline_version(paths, port_name, versiont, baseline_path, verbose); + } + Checks::exit_success(VCPKG_LINE_INFO); + } + + void AddVersionCommand::perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) const + { + AddVersion::perform_and_exit(args, paths); + } +} \ No newline at end of file diff --git a/toolsrc/src/vcpkg/commands.civerifyversions.cpp b/toolsrc/src/vcpkg/commands.civerifyversions.cpp new file mode 100644 index 0000000000..a0de254c53 --- /dev/null +++ b/toolsrc/src/vcpkg/commands.civerifyversions.cpp @@ -0,0 +1,328 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using namespace vcpkg; + +} + +namespace vcpkg::Commands::CIVerifyVersions +{ + static constexpr StringLiteral OPTION_EXCLUDE = "exclude"; + static constexpr StringLiteral OPTION_VERBOSE = "verbose"; + static constexpr StringLiteral OPTION_VERIFY_GIT_TREES = "verify-git-trees"; + + static constexpr CommandSwitch VERIFY_VERSIONS_SWITCHES[]{ + {OPTION_VERBOSE, "Print result for each port instead of just errors."}, + {OPTION_VERIFY_GIT_TREES, "Verify that each git tree object matches its declared version (this is very slow)"}, + }; + + static constexpr CommandSetting VERIFY_VERSIONS_SETTINGS[] = { + {OPTION_EXCLUDE, "Comma-separated list of ports to skip"}, + }; + + const CommandStructure COMMAND_STRUCTURE{ + create_example_string(R"###(x-ci-verify-versions)###"), + 0, + SIZE_MAX, + {{VERIFY_VERSIONS_SWITCHES}, {VERIFY_VERSIONS_SETTINGS}, {}}, + nullptr, + }; + + static ExpectedS verify_version_in_db(const VcpkgPaths& paths, + const std::map> baseline, + const std::string& port_name, + const fs::path& port_path, + const fs::path& versions_file_path, + const std::string& local_git_tree, + bool verify_git_trees) + { + auto maybe_versions = vcpkg::get_builtin_versions(paths, port_name); + if (!maybe_versions.has_value()) + { + return { + Strings::format( + "Error: Cannot parse `%s`.\n\t%s", fs::u8string(versions_file_path), maybe_versions.error()), + expected_right_tag, + }; + } + + const auto& versions = maybe_versions.value_or_exit(VCPKG_LINE_INFO); + if (versions.empty()) + { + return { + Strings::format("Error: File `%s` contains no versions.", fs::u8string(versions_file_path)), + expected_right_tag, + }; + } + + if (verify_git_trees) + { + for (auto&& version_entry : versions) + { + bool version_ok = false; + for (const std::string& control_file : {"CONTROL", "vcpkg.json"}) + { + auto treeish = Strings::concat(version_entry.second, ':', control_file); + auto maybe_file = paths.git_show(Strings::concat(treeish), paths.root / fs::u8path(".git")); + if (!maybe_file.has_value()) continue; + + const auto& file = maybe_file.value_or_exit(VCPKG_LINE_INFO); + auto maybe_scf = Paragraphs::try_load_port_text(file, treeish, control_file == "vcpkg.json"); + if (!maybe_scf.has_value()) + { + return { + Strings::format("Error: Unable to parse `%s` used in version `%s`.\n%s\n", + treeish, + version_entry.first.to_string(), + maybe_scf.error()->error), + expected_right_tag, + }; + } + + const auto& scf = maybe_scf.value_or_exit(VCPKG_LINE_INFO); + auto&& git_tree_version = scf.get()->to_versiont(); + if (version_entry.first != git_tree_version) + { + return { + Strings::format("Error: Version in git-tree `%s` does not match version in file " + "`%s`.\n\tgit-tree version: %s\n\t file version: %s\n", + version_entry.second, + fs::u8string(versions_file_path), + git_tree_version.to_string(), + version_entry.first.to_string()), + expected_right_tag, + }; + } + version_ok = true; + break; + } + + if (!version_ok) + { + return { + Strings::format("Error: The git-tree `%s` for version `%s` in `%s` does not contain a " + "CONTROL file or vcpkg.json file.", + version_entry.second, + version_entry.first, + fs::u8string(versions_file_path)), + expected_right_tag, + }; + } + } + } + + const auto& top_entry = versions.front(); + + auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(), port_path); + if (!maybe_scf.has_value()) + { + return { + Strings::format("Error: Cannot load port `%s`.\n\t%s", port_name, maybe_scf.error()->error), + expected_right_tag, + }; + } + + const auto found_version = maybe_scf.value_or_exit(VCPKG_LINE_INFO)->to_versiont(); + if (top_entry.first != found_version) + { + auto versions_end = versions.end(); + auto it = std::find_if( + versions.begin(), versions_end, [&](auto&& entry) { return entry.first == found_version; }); + if (it != versions_end) + { + return { + Strings::format("Error: Version `%s` found but is not the top entry in `%s`.", + found_version, + fs::u8string(versions_file_path)), + expected_right_tag, + }; + } + else + { + return { + Strings::format( + "Error: Version `%s` not found in `%s`.", found_version, fs::u8string(versions_file_path)), + expected_right_tag, + }; + } + } + + if (local_git_tree != top_entry.second) + { + return { + Strings::format("Error: Git tree-ish object for version `%s` in `%s` does not match local port files.\n" + "\tLocal SHA: %s\n" + "\t File SHA: %s", + found_version, + fs::u8string(versions_file_path), + local_git_tree, + top_entry.second), + expected_right_tag, + }; + } + + auto maybe_baseline = baseline.find(port_name); + if (maybe_baseline == baseline.end()) + { + return { + Strings::format("Error: Couldn't find baseline version for port `%s`.", port_name), + expected_right_tag, + }; + } + + auto&& baseline_version = maybe_baseline->second; + if (baseline_version != top_entry.first) + { + return { + Strings::format("Error: The baseline version for port `%s` doesn't match the latest version.\n" + "\tBaseline version: %s\n" + "\t Latest version: %s (%s)", + port_name, + baseline_version, + top_entry.first, + fs::u8string(versions_file_path)), + expected_right_tag, + }; + } + + if (local_git_tree != top_entry.second) + { + return { + Strings::format("Error: Git tree-ish object for version `%s` in `%s` does not match local port files.\n" + "\tLocal SHA: %s\n" + "\t File SHA: %s", + found_version, + fs::u8string(versions_file_path), + local_git_tree, + top_entry.second), + expected_right_tag, + }; + } + + return { + Strings::format("OK: %s\t%s -> %s\n", top_entry.second, port_name, top_entry.first), + expected_left_tag, + }; + } + + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) + { + auto parsed_args = args.parse_arguments(COMMAND_STRUCTURE); + + bool verbose = Util::Sets::contains(parsed_args.switches, OPTION_VERBOSE); + bool verify_git_trees = Util::Sets::contains(parsed_args.switches, OPTION_VERIFY_GIT_TREES); + + std::set exclusion_set; + auto settings = parsed_args.settings; + auto it_exclusions = settings.find(OPTION_EXCLUDE); + if (it_exclusions != settings.end()) + { + auto exclusions = Strings::split(it_exclusions->second, ','); + exclusion_set.insert(exclusions.begin(), exclusions.end()); + } + + auto maybe_port_git_tree_map = paths.git_get_local_port_treeish_map(); + Checks::check_exit(VCPKG_LINE_INFO, + maybe_port_git_tree_map.has_value(), + "Error: Failed to obtain git treeish objects for local ports.\n%s", + maybe_port_git_tree_map.error()); + auto port_git_tree_map = maybe_port_git_tree_map.value_or_exit(VCPKG_LINE_INFO); + + // Baseline is required. + auto baseline = get_builtin_baseline(paths).value_or_exit(VCPKG_LINE_INFO); + auto& fs = paths.get_filesystem(); + std::set errors; + for (const auto& dir : fs::directory_iterator(paths.builtin_ports_directory())) + { + const auto& port_path = dir.path(); + + auto&& port_name = fs::u8string(port_path.stem()); + if (Util::Sets::contains(exclusion_set, port_name)) + { + if (verbose) System::printf("SKIP: %s\n", port_name); + continue; + } + auto git_tree_it = port_git_tree_map.find(port_name); + if (git_tree_it == port_git_tree_map.end()) + { + System::printf(System::Color::error, "FAIL: %s\n", port_name); + errors.emplace(Strings::format("Error: Missing local git tree object for port `%s`.", port_name)); + continue; + } + auto git_tree = git_tree_it->second; + + auto control_path = port_path / fs::u8path("CONTROL"); + auto manifest_path = port_path / fs::u8path("vcpkg.json"); + auto manifest_exists = fs.exists(manifest_path); + auto control_exists = fs.exists(control_path); + + if (manifest_exists && control_exists) + { + System::printf(System::Color::error, "FAIL: %s\n", port_name); + errors.emplace( + Strings::format("Error: Both a manifest file and a CONTROL file exist in port directory: %s", + fs::u8string(port_path))); + continue; + } + + if (!manifest_exists && !control_exists) + { + System::printf(System::Color::error, "FAIL: %s\n", port_name); + errors.emplace(Strings::format("Error: No manifest file or CONTROL file exist in port directory: %s", + fs::u8string(port_path))); + continue; + } + + auto versions_file_path = + paths.builtin_port_versions / Strings::concat(port_name[0], '-') / Strings::concat(port_name, ".json"); + if (!fs.exists(versions_file_path)) + { + System::printf(System::Color::error, "FAIL: %s\n", port_name); + errors.emplace(Strings::format("Error: Missing versions file for `%s`. Expected at `%s`.", + port_name, + fs::u8string(versions_file_path))); + continue; + } + + auto maybe_ok = verify_version_in_db( + paths, baseline, port_name, port_path, versions_file_path, git_tree, verify_git_trees); + + if (!maybe_ok.has_value()) + { + System::printf(System::Color::error, "FAIL: %s\n", port_name); + errors.emplace(maybe_ok.error()); + continue; + } + + if (verbose) System::printf("%s", maybe_ok.value_or_exit(VCPKG_LINE_INFO)); + } + + if (!errors.empty()) + { + System::print2(System::Color::error, "Found the following errors:\n"); + for (auto&& error : errors) + { + System::printf(System::Color::error, "%s\n", error); + } + Checks::exit_fail(VCPKG_LINE_INFO); + } + Checks::exit_success(VCPKG_LINE_INFO); + } + + void CIVerifyVersionsCommand::perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) const + { + CIVerifyVersions::perform_and_exit(args, paths); + } +} \ No newline at end of file diff --git a/toolsrc/src/vcpkg/commands.cpp b/toolsrc/src/vcpkg/commands.cpp index 6e87c17748..19b5bdcde9 100644 --- a/toolsrc/src/vcpkg/commands.cpp +++ b/toolsrc/src/vcpkg/commands.cpp @@ -1,11 +1,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -73,6 +75,8 @@ namespace vcpkg::Commands static const PortHistory::PortHistoryCommand porthistory{}; static const X_VSInstances::VSInstancesCommand vsinstances{}; static const FormatManifest::FormatManifestCommand format_manifest{}; + static const CIVerifyVersions::CIVerifyVersionsCommand ci_verify_versions{}; + static const AddVersion::AddVersionCommand add_version{}; static std::vector> t = { {"/?", &help}, @@ -94,6 +98,8 @@ namespace vcpkg::Commands {"x-history", &porthistory}, {"x-vsinstances", &vsinstances}, {"format-manifest", &format_manifest}, + {"x-ci-verify-versions", &ci_verify_versions}, + {"x-add-version", &add_version}, }; return t; } diff --git a/toolsrc/src/vcpkg/commands.porthistory.cpp b/toolsrc/src/vcpkg/commands.porthistory.cpp index 92d782c36b..894b46b6eb 100644 --- a/toolsrc/src/vcpkg/commands.porthistory.cpp +++ b/toolsrc/src/vcpkg/commands.porthistory.cpp @@ -52,12 +52,6 @@ namespace vcpkg::Commands::PortHistory return run_git_command_inner(paths, dot_git_dir, work_dir, cmd); } - bool is_date(const std::string& version_string) - { - std::regex re("^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})$"); - return std::regex_match(version_string, re); - } - vcpkg::Optional get_version_from_text(const std::string& text, const std::string& git_tree, const std::string& commit_id, diff --git a/toolsrc/src/vcpkg/portfileprovider.cpp b/toolsrc/src/vcpkg/portfileprovider.cpp index 05c5109c44..f455c8233e 100644 --- a/toolsrc/src/vcpkg/portfileprovider.cpp +++ b/toolsrc/src/vcpkg/portfileprovider.cpp @@ -210,7 +210,13 @@ namespace vcpkg::PortFileProvider if (port) { - auto port_path = port->get_path_to_version(paths, port_version).value_or_exit(VCPKG_LINE_INFO); + auto maybe_port_path = port->get_path_to_version(paths, port_version); + if (!maybe_port_path.has_value()) + { + return std::move(maybe_port_path.error()); + } + auto port_path = std::move(maybe_port_path).value_or_exit(VCPKG_LINE_INFO); + auto maybe_scfl = Paragraphs::try_load_port(fs, port_path); if (auto p = maybe_scfl.get()) { diff --git a/toolsrc/src/vcpkg/registries.cpp b/toolsrc/src/vcpkg/registries.cpp index 22ac26fd9d..31f3078deb 100644 --- a/toolsrc/src/vcpkg/registries.cpp +++ b/toolsrc/src/vcpkg/registries.cpp @@ -195,7 +195,8 @@ namespace std::unique_ptr BuiltinRegistry::get_port_entry(const VcpkgPaths& paths, StringView port_name) const { auto versions_path = paths.builtin_port_versions / relative_path_to_versions(port_name); - if (paths.get_feature_flags().registries && paths.get_filesystem().exists(versions_path)) + if ((paths.get_feature_flags().registries || paths.get_feature_flags().versions) && + paths.get_filesystem().exists(versions_path)) { auto maybe_version_entries = load_versions_file(paths.get_filesystem(), VersionDbType::Git, paths.builtin_port_versions, port_name); @@ -244,15 +245,16 @@ namespace return nullptr; } - Baseline parse_builtin_baseline(const VcpkgPaths& paths, StringView baseline_identifier) + ExpectedS try_parse_builtin_baseline(const VcpkgPaths& paths, StringView baseline_identifier) { auto path_to_baseline = paths.builtin_port_versions / fs::u8path("baseline.json"); auto res_baseline = load_baseline_versions(paths, path_to_baseline, baseline_identifier); if (!res_baseline.has_value()) { - Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error()); + return res_baseline.error(); } + auto opt_baseline = res_baseline.get(); if (auto p = opt_baseline->get()) { @@ -261,31 +263,29 @@ namespace if (baseline_identifier.size() == 0) { - return {}; + return {{}, expected_left_tag}; } if (baseline_identifier == "default") { - Checks::exit_with_message(VCPKG_LINE_INFO, - "Couldn't find explicitly specified baseline `\"default\"` in the baseline file.", - baseline_identifier); + return Strings::format("Couldn't find explicitly specified baseline `\"default\"` in the baseline file.", + baseline_identifier); } // attempt to check out the baseline: auto maybe_path = get_git_baseline_json_path(paths, baseline_identifier); if (!maybe_path.has_value()) { - Checks::exit_with_message(VCPKG_LINE_INFO, - "Couldn't find explicitly specified baseline `\"%s\"` in the baseline file, " - "and there was no baseline at that commit or the commit didn't exist.\n%s", - baseline_identifier, - maybe_path.error()); + return Strings::format("Couldn't find explicitly specified baseline `\"%s\"` in the baseline file, " + "and there was no baseline at that commit or the commit didn't exist.\n%s", + baseline_identifier, + maybe_path.error()); } res_baseline = load_baseline_versions(paths, *maybe_path.get()); if (!res_baseline.has_value()) { - Checks::exit_with_message(VCPKG_LINE_INFO, res_baseline.error()); + return res_baseline.error(); } opt_baseline = res_baseline.get(); if (auto p = opt_baseline->get()) @@ -293,14 +293,19 @@ namespace return std::move(*p); } - Checks::exit_with_message(VCPKG_LINE_INFO, - "Couldn't find explicitly specified baseline `\"%s\"` in the baseline " - "file, and the `\"default\"` baseline does not exist at that commit.", - baseline_identifier); + return Strings::format("Couldn't find explicitly specified baseline `\"%s\"` in the baseline " + "file, and the `\"default\"` baseline does not exist at that commit.", + baseline_identifier); + } + + Baseline parse_builtin_baseline(const VcpkgPaths& paths, StringView baseline_identifier) + { + auto maybe_baseline = try_parse_builtin_baseline(paths, baseline_identifier); + return maybe_baseline.value_or_exit(VCPKG_LINE_INFO); } Optional BuiltinRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const { - if (paths.get_feature_flags().registries) + if (!m_baseline_identifier.empty()) { const auto& baseline = m_baseline.get( [this, &paths]() -> Baseline { return parse_builtin_baseline(paths, m_baseline_identifier); }); @@ -311,22 +316,25 @@ namespace return it->second; } } - - // fall back to using the ports directory version - auto maybe_scf = - Paragraphs::try_load_port(paths.get_filesystem(), paths.builtin_ports_directory() / fs::u8path(port_name)); - if (auto pscf = maybe_scf.get()) + else { - auto& scf = *pscf; - return scf->to_versiont(); + // fall back to using the ports directory version + auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(), + paths.builtin_ports_directory() / fs::u8path(port_name)); + if (auto pscf = maybe_scf.get()) + { + auto& scf = *pscf; + return scf->to_versiont(); + } + Debug::print("Failed to load port `", port_name, "` from the ports tree: ", maybe_scf.error()->error, "\n"); } - Debug::print("Failed to load port `", port_name, "` from the ports tree: ", maybe_scf.error()->error, "\n"); return nullopt; } void BuiltinRegistry::get_all_port_names(std::vector& out, const VcpkgPaths& paths) const { - if (paths.get_feature_flags().registries && paths.get_filesystem().exists(paths.builtin_port_versions)) + if ((paths.get_feature_flags().registries || paths.get_feature_flags().versions) && + paths.get_filesystem().exists(paths.builtin_port_versions)) { load_all_port_names_from_port_versions(out, paths, paths.builtin_port_versions); } @@ -436,8 +444,9 @@ namespace auto& name = scfl->source_control_file->core_paragraph->name; return Strings::format( "Error: no version entry for %s at version %s.\n" - "We are currently using the version in the ports tree, since no %s.json was found in port_versions.", + "We are currently using the version in the ports tree (%s), since no %s.json was found in /port_versions.", name, + version.to_string(), scfl->to_versiont().to_string(), name); } @@ -925,7 +934,7 @@ namespace vcpkg System::print2(System::Color::warning, "Warning: when using the registries feature, one should not use `\"$x-default-baseline\"` " "to set the baseline.\n", - " Instead, use the \"baseline\" field of the registry."); + " Instead, use the \"baseline\" field of the registry.\n"); } for (auto& reg : registries_) @@ -954,4 +963,23 @@ namespace vcpkg // default_registry_ is not a BuiltinRegistry return true; } + + ExpectedS>> get_builtin_versions(const VcpkgPaths& paths, + StringView port_name) + { + auto maybe_versions = + load_versions_file(paths.get_filesystem(), VersionDbType::Git, paths.builtin_port_versions, port_name); + if (auto pversions = maybe_versions.get()) + { + return Util::fmap( + *pversions, [](auto&& entry) -> auto { return std::make_pair(entry.version, entry.git_tree); }); + } + + return maybe_versions.error(); + } + + ExpectedS>> get_builtin_baseline(const VcpkgPaths& paths) + { + return try_parse_builtin_baseline(paths, "default"); + } } diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index 0310a01ae3..648b2065cf 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -628,6 +628,10 @@ namespace vcpkg table.format(opt(INSTALL_ROOT_DIR_ARG, "=", ""), "(Experimental) Specify the install root directory"); table.format(opt(PACKAGES_ROOT_DIR_ARG, "=", ""), "(Experimental) Specify the packages root directory"); table.format(opt(SCRIPTS_ROOT_DIR_ARG, "=", ""), "(Experimental) Specify the scripts root directory"); + table.format(opt(BUILTIN_PORTS_ROOT_DIR_ARG, "=", ""), + "(Experimental) Specify the packages root directory"); + table.format(opt(BUILTIN_PORT_VERSIONS_DIR_ARG, "=", ""), + "(Experimental) Specify the versions root directory"); table.format(opt(JSON_SWITCH, "", ""), "(Experimental) Request JSON output"); } @@ -916,6 +920,7 @@ namespace vcpkg constexpr StringLiteral VcpkgCmdArguments::PACKAGES_ROOT_DIR_ARG; constexpr StringLiteral VcpkgCmdArguments::SCRIPTS_ROOT_DIR_ARG; constexpr StringLiteral VcpkgCmdArguments::BUILTIN_PORTS_ROOT_DIR_ARG; + constexpr StringLiteral VcpkgCmdArguments::BUILTIN_PORT_VERSIONS_DIR_ARG; constexpr StringLiteral VcpkgCmdArguments::DEFAULT_VISUAL_STUDIO_PATH_ENV; diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index a412587617..27b7cfe028 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -367,7 +367,6 @@ If you wish to silence this error and use classic mode, you can: vcpkg_dir_info = vcpkg_dir / fs::u8path("info"); vcpkg_dir_updates = vcpkg_dir / fs::u8path("updates"); - // Versioning paths const auto versioning_tmp = buildtrees / fs::u8path("versioning_tmp"); const auto versioning_output = buildtrees / fs::u8path("versioning"); @@ -564,6 +563,52 @@ If you wish to silence this error and use classic mode, you can: } } + ExpectedS>> VcpkgPaths::git_get_local_port_treeish_map() const + { + const auto local_repo = this->root / fs::u8path(".git"); + const auto path_with_separator = + Strings::concat(fs::u8string(this->builtin_ports_directory()), Files::preferred_separator); + const auto git_cmd = git_cmd_builder(*this, local_repo, this->root) + .string_arg("ls-tree") + .string_arg("-d") + .string_arg("HEAD") + .string_arg("--") + .path_arg(path_with_separator) + .extract(); + + auto output = System::cmd_execute_and_capture_output(git_cmd); + if (output.exit_code != 0) + return Strings::format("Error: Couldn't get local treeish objects for ports.\n%s", output.output); + + std::map> ret; + auto lines = Strings::split(output.output, '\n'); + // The first line of the output is always the parent directory itself. + for (auto line : lines) + { + // The default output comes in the format: + // SP SP TAB + auto split_line = Strings::split(line, '\t'); + if (split_line.size() != 2) + return Strings::format( + "Error: Unexpected output from command `%s`. Couldn't split by `\\t`.\n%s", git_cmd, line); + + auto file_info_section = Strings::split(split_line[0], ' '); + if (file_info_section.size() != 3) + return Strings::format( + "Error: Unexepcted output from command `%s`. Couldn't split by ` `.\n%s", git_cmd, line); + + const auto index = split_line[1].find_last_of('/'); + if (index == std::string::npos) + { + return Strings::format( + "Error: Unexpected output from command `%s`. Couldn't split by `/`.\n%s", git_cmd, line); + } + + ret.emplace(split_line[1].substr(index + 1), file_info_section.back()); + } + return ret; + } + void VcpkgPaths::git_checkout_object(const VcpkgPaths& paths, StringView git_object, const fs::path& local_repo, diff --git a/toolsrc/windows-bootstrap/vcpkg.vcxproj b/toolsrc/windows-bootstrap/vcpkg.vcxproj index b11d5540c7..9f035bb375 100644 --- a/toolsrc/windows-bootstrap/vcpkg.vcxproj +++ b/toolsrc/windows-bootstrap/vcpkg.vcxproj @@ -213,11 +213,13 @@ + + @@ -303,11 +305,13 @@ + +