Add the ability to do full rebuilds with updated copies of vcpkg-tool. (#24611)

* Add the ability to do full rebuilds with updated copies of vcpkg-tool.

Resurrection of https://github.com/microsoft/vcpkg/pull/23753/

* Extract all scripts out of inline blocks.

* Replicate all changes to the "unstable" testing.

* Fix bootstrap :)

* Fix version files validation path.

* Malformed.

* Paths and permissions.

* Fixed file name typo

* Update scripts/azure-pipelines/bootstrap-from-source.sh

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

scripts/azure-pipelines/azure-pipelines.yml

@@ -6,53 +6,68 @@ variables:
   linux-pool: 'PrLin-2022-05-19'
   osx-pool: 'PrOsx-2022-02-04'
 
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+
 jobs:
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: x86-windows
     jobName: x86_windows
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: x64-windows
     jobName: x64_windows
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: x64-windows-static
     jobName: x64_windows_static
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: x64-windows-static-md
     jobName: x64_windows_static_md
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: x64-uwp
     jobName: x64_uwp
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: arm64-windows
     jobName: arm64_windows
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: windows/azure-pipelines.yml
   parameters:
     triplet: arm-uwp
     jobName: arm_uwp
     poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: osx/azure-pipelines.yml
   parameters:
     poolName: $(osx-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
 
 - template: linux/azure-pipelines.yml
   parameters:
     poolName: $(linux-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

scripts/azure-pipelines/bootstrap-from-source.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+
+git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool switch -d $1
+rm -rf build.x64.release
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x64.release -S vcpkg-tool
+ninja -C build.x64.release
+mv build.x64.release/vcpkg vcpkg

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -2,6 +2,14 @@
 # SPDX-License-Identifier: MIT
 #
 
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: poolName
+    type: string
+
 jobs:
 - job: x64_linux
   pool:
@@ -22,35 +30,27 @@ jobs:
     value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"
 
   steps:
-  - bash: df -h
+    # Note: /mnt is the Azure machines' temporary disk.
-    displayName: 'Report on Disk Space'
   - bash: |
       sudo mkdir /home/agent -m=777
       sudo chown `id -u` /home/agent
-      exit 0
-    displayName: 'Create /home/agent'
-    # Note: /mnt is the Azure machines' temporary disk.
-  - bash: |
-      sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }}
       sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
       sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
       exit 0
-    displayName: 'Create ${{ variables.WORKING_ROOT }} and ${{ variables.VCPKG_DOWNLOADS }}'
+    displayName: 'Create working directories'
-  - task: Bash@3
+  - bash: ./bootstrap-vcpkg.sh
     displayName: 'Bootstrap vcpkg'
-    inputs:
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
-      filePath: bootstrap-vcpkg.sh
+  - bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
   - task: PowerShell@2
-    displayName: '*** Test Modified Ports for x64-linux'
+    displayName: '*** Test Modified Ports'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
       arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
       pwsh: true
-  - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: failure logs for x64-linux'
     inputs:

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -3,6 +3,10 @@
 #
 
 parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
   - name: poolName
     type: string
 
@@ -26,20 +30,21 @@ jobs:
     value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"
 
   steps:
-  - bash: df -h
-    displayName: 'Report on Disk Space'
   - bash: |
-      sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }} || 0
       sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
       sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
       exit 0
     displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
-  - task: Bash@3
+  - bash: ./bootstrap-vcpkg.sh
     displayName: 'Bootstrap vcpkg'
-    inputs:
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
-      filePath: bootstrap-vcpkg.sh
+  - bash: |
+      brew install cmake
+      ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
   - task: PowerShell@2
-    displayName: '*** Test Modified Ports for x64-osx'
+    displayName: '*** Test Modified Ports'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
@@ -50,18 +55,13 @@ jobs:
         -WorkingRoot "${{ variables.WORKING_ROOT }}"
         -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
       pwsh: true
-  - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: failure logs for x64-osx'
     inputs:
       PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs'
       ArtifactName: 'failure logs for x64-osx${{ variables.Postfix }}'
     condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
-  - bash: |
+  - bash: python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
-      python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
     displayName: 'Build a file list for all packages'
     condition: always()
   - task: PublishBuildArtifacts@1

scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json

@@ -1,10 +1,11 @@
 {
   "$schema": "./vagrant-box-configuration.schema.json",
   "brew": [
-    "autoconf",
     "autoconf-archive",
+    "autoconf",
     "automake",
     "bison",
+    "cmake",
     "gettext",
     "gfortran",
     "gperf",
@@ -13,6 +14,7 @@
     "meson",
     "mono",
     "nasm",
+    "ninja",
     "pkg-config",
     "texinfo",
     "yasm"

scripts/azure-pipelines/windows-unstable/job.yml

@@ -8,7 +8,7 @@ jobs:
     name: $(unstable-pool)
   workspace:
     clean: resources
-  timeoutInMinutes: 1440 # 1 day
+  timeoutInMinutes: 2880 # 2 days
   variables:
   - name: WORKING_ROOT
     value: D:\
@@ -59,30 +59,18 @@ jobs:
       filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
       arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
       pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
   - script: .\bootstrap-vcpkg.bat
-    displayName: 'Build vcpkg'
+    displayName: 'Bootstrap vcpkg'
   - task: PowerShell@2
-    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
+    displayName: '*** Test Modified Ports'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
       arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -PassingIsPassing'
       pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
     inputs:
       PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs'
       ArtifactName: 'failure logs for ${{ parameters.triplet }}'
-    condition: always()
+    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -2,6 +2,20 @@
 # SPDX-License-Identifier: MIT
 #
 
+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: triplet
+    type: string
+    default: 'x86-windows'
+  - name: jobName
+    type: string
+    default: 'x86_windows'
+  - name: poolName
+    type: string
+
 jobs:
 - job: ${{ parameters.jobName }}
   pool:
@@ -26,14 +40,12 @@ jobs:
     value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"
 
   steps:
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
   - script: .\bootstrap-vcpkg.bat
     displayName: 'Bootstrap vcpkg'
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - script: .\scripts\azure-pipelines\windows\bootstrap-from-source.cmd ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
   # Note that we run docs checks before PR checks because they are likely to invalidate a whole run anyway
   - task: Powershell@2
     displayName: 'Generate Documentation'
@@ -59,7 +71,7 @@ jobs:
       PathtoPublish: '$(DiffFile)'
       ArtifactName: 'format.diff'
   - task: PowerShell@2
-    displayName: '*** Test Modified Ports for ${{ parameters.triplet }}'
+    displayName: '*** Test Modified Ports'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
@@ -69,43 +81,7 @@ jobs:
     displayName: 'Validate version files'
     condition: eq('${{ parameters.triplet }}', '${{ variables.ExtraChecksTriplet }}')
     inputs:
-      targetType: inline
+      filePath: 'scripts/azure-pipelines/windows/validate-version-files.ps1'
-      script: |
-        ./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
-        ForEach-Object -Begin {
-          $long_error = ''
-        } -Process {
-          if ($long_error -ne '' -and $_ -match '^$|^       ') {
-             # Extend multi-line message
-             $long_error = -join($long_error, "%0D%0A", $_ -replace '^       ','' `
-               -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
-          } else {
-            if ($long_error -ne '') {
-              # Flush multi-line message
-              $long_error
-              $long_error = ''
-            }
-            if ($_ -match '^Error: ') {
-              # Start multi-line message
-              $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
-                -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
-            } else {
-              # Normal line
-              $_
-            }
-          }
-        } -End {
-          if ($long_error -ne '') {
-            # Flush multi-line message
-            $long_error
-          }
-        }
-      pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
       pwsh: true
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'

scripts/azure-pipelines/windows/bootstrap-from-source.cmd

@@ -0,0 +1,7 @@
+call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
+git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool switch -d %1
+rmdir /s /q build.x86.release > nul 2> nul
+cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x86.release -S vcpkg-tool
+ninja.exe -C build.x86.release
+move build.x86.release\vcpkg.exe vcpkg.exe

scripts/azure-pipelines/windows/validate-version-files.ps1

@@ -0,0 +1,29 @@
+./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
+ForEach-Object -Begin {
+  $long_error = ''
+} -Process {
+  if ($long_error -ne '' -and $_ -match '^$|^       ') {
+     # Extend multi-line message
+     $long_error = -join($long_error, "%0D%0A", $_ -replace '^       ','' `
+       -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
+  } else {
+    if ($long_error -ne '') {
+      # Flush multi-line message
+      $long_error
+      $long_error = ''
+    }
+    if ($_ -match '^Error: ') {
+      # Start multi-line message
+      $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
+        -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
+    } else {
+      # Normal line
+      $_
+    }
+  }
+} -End {
+  if ($long_error -ne '') {
+    # Flush multi-line message
+    $long_error
+  }
+}