Add the ability to do full rebuilds with updated copies of vcpkg-tool. (#24611)

* Add the ability to do full rebuilds with updated copies of vcpkg-tool. Resurrection of https://github.com/microsoft/vcpkg/pull/23753/ * Extract all scripts out of inline blocks. * Replicate all changes to the "unstable" testing. * Fix bootstrap :) * Fix version files validation path. * Malformed. * Paths and permissions. * Fixed file name typo * Update scripts/azure-pipelines/bootstrap-from-source.sh Co-authored-by: Robert Schumacher <roschuma@microsoft.com> Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
2024-12-27 02:11:58 +08:00 · 2022-05-23 13:33:58 -07:00 · 2022-05-23 13:33:58 -07:00 · 4c82b77b45
commit 4c82b77b45
parent 413c11d3c5
9 changed files with 118 additions and 92 deletions
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@ -6,53 +6,68 @@ variables:
  linux-pool: 'PrLin-2022-05-19'
  osx-pool: 'PrOsx-2022-02-04'

+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+
 jobs:
 - template: windows/azure-pipelines.yml
  parameters:
    triplet: x86-windows
    jobName: x86_windows
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: x64-windows
    jobName: x64_windows
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: x64-windows-static
    jobName: x64_windows_static
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: x64-windows-static-md
    jobName: x64_windows_static_md
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: x64-uwp
    jobName: x64_uwp
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: arm64-windows
    jobName: arm64_windows
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: windows/azure-pipelines.yml
  parameters:
    triplet: arm-uwp
    jobName: arm_uwp
    poolName: $(windows-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: osx/azure-pipelines.yml
  parameters:
    poolName: $(osx-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}

 - template: linux/azure-pipelines.yml
  parameters:
    poolName: $(linux-pool)
+    vcpkgToolSha: ${{ parameters.vcpkgToolSha }}
--- a/scripts/azure-pipelines/bootstrap-from-source.sh
+++ b/scripts/azure-pipelines/bootstrap-from-source.sh
@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+
+git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool switch -d $1
+rm -rf build.x64.release
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x64.release -S vcpkg-tool
+ninja -C build.x64.release
+mv build.x64.release/vcpkg vcpkg
--- a/scripts/azure-pipelines/linux/azure-pipelines.yml
+++ b/scripts/azure-pipelines/linux/azure-pipelines.yml
@ -2,6 +2,14 @@
 # SPDX-License-Identifier: MIT
 #

+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: poolName
+    type: string
+
 jobs:
 - job: x64_linux
  pool:
@ -22,35 +30,27 @@ jobs:
    value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"

  steps:
-  - bash: df -h
-    displayName: 'Report on Disk Space'
+    # Note: /mnt is the Azure machines' temporary disk.
  - bash: |
      sudo mkdir /home/agent -m=777
      sudo chown `id -u` /home/agent
-      exit 0
-    displayName: 'Create /home/agent'
-    # Note: /mnt is the Azure machines' temporary disk.
-  - bash: |
-      sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }}
      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
      exit 0
-    displayName: 'Create ${{ variables.WORKING_ROOT }} and ${{ variables.VCPKG_DOWNLOADS }}'
-  - task: Bash@3
+    displayName: 'Create working directories'
+  - bash: ./bootstrap-vcpkg.sh
    displayName: 'Bootstrap vcpkg'
-    inputs:
-      filePath: bootstrap-vcpkg.sh
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
  - task: PowerShell@2
-    displayName: '*** Test Modified Ports for x64-linux'
+    displayName: '*** Test Modified Ports'
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
      pwsh: true
-  - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
  - task: PublishBuildArtifacts@1
    displayName: 'Publish Artifact: failure logs for x64-linux'
    inputs:
--- a/scripts/azure-pipelines/osx/azure-pipelines.yml
+++ b/scripts/azure-pipelines/osx/azure-pipelines.yml
@ -3,8 +3,12 @@
 #

 parameters:
- name: poolName
-  type: string
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: poolName
+    type: string

 jobs:
 - job: x64_osx
@ -26,20 +30,21 @@ jobs:
    value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite"

  steps:
-  - bash: df -h
-    displayName: 'Report on Disk Space'
  - bash: |
-      sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }} || 0
      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
      sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
      exit 0
    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
-  - task: Bash@3
+  - bash: ./bootstrap-vcpkg.sh
    displayName: 'Bootstrap vcpkg'
-    inputs:
-      filePath: bootstrap-vcpkg.sh
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - bash: |
+      brew install cmake
+      ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
  - task: PowerShell@2
-    displayName: '*** Test Modified Ports for x64-osx'
+    displayName: '*** Test Modified Ports'
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
@ -50,18 +55,13 @@ jobs:
        -WorkingRoot "${{ variables.WORKING_ROOT }}"
        -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
      pwsh: true
-  - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
  - task: PublishBuildArtifacts@1
    displayName: 'Publish Artifact: failure logs for x64-osx'
    inputs:
      PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs'
      ArtifactName: 'failure logs for x64-osx${{ variables.Postfix }}'
    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
-  - bash: |
-      python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
+  - bash: python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/
    displayName: 'Build a file list for all packages'
    condition: always()
  - task: PublishBuildArtifacts@1
--- a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json
+++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json
@ -1,10 +1,11 @@
 {
  "$schema": "./vagrant-box-configuration.schema.json",
  "brew": [
-    "autoconf",
    "autoconf-archive",
+    "autoconf",
    "automake",
    "bison",
+    "cmake",
    "gettext",
    "gfortran",
    "gperf",
@ -13,6 +14,7 @@
    "meson",
    "mono",
    "nasm",
+    "ninja",
    "pkg-config",
    "texinfo",
    "yasm"
--- a/scripts/azure-pipelines/windows-unstable/job.yml
+++ b/scripts/azure-pipelines/windows-unstable/job.yml
@ -8,7 +8,7 @@ jobs:
    name: $(unstable-pool)
  workspace:
    clean: resources
-  timeoutInMinutes: 1440 # 1 day
+  timeoutInMinutes: 2880 # 2 days
  variables:
  - name: WORKING_ROOT
    value: D:\
@ -59,30 +59,18 @@ jobs:
      filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
      arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
      pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
  - script: .\bootstrap-vcpkg.bat
-    displayName: 'Build vcpkg'
+    displayName: 'Bootstrap vcpkg'
  - task: PowerShell@2
-    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
+    displayName: '*** Test Modified Ports'
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -PassingIsPassing'
      pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
  - task: PublishBuildArtifacts@1
    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
    inputs:
      PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs'
      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
-    condition: always()
+    condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True')
--- a/scripts/azure-pipelines/windows/azure-pipelines.yml
+++ b/scripts/azure-pipelines/windows/azure-pipelines.yml
@ -2,6 +2,20 @@
 # SPDX-License-Identifier: MIT
 #

+parameters:
+  - name: vcpkgToolSha
+    displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
+    type: string
+    default: 'use default'
+  - name: triplet
+    type: string
+    default: 'x86-windows'
+  - name: jobName
+    type: string
+    default: 'x86_windows'
+  - name: poolName
+    type: string
+
 jobs:
 - job: ${{ parameters.jobName }}
  pool:
@ -26,14 +40,12 @@ jobs:
    value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)"

  steps:
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
-      pwsh: true
  - script: .\bootstrap-vcpkg.bat
    displayName: 'Bootstrap vcpkg'
+    condition: eq('use default', '${{ parameters.vcpkgToolSha }}')
+  - script: .\scripts\azure-pipelines\windows\bootstrap-from-source.cmd ${{ parameters.vcpkgToolSha }}
+    displayName: "Build vcpkg with CMake"
+    condition: ne('use default', '${{ parameters.vcpkgToolSha }}')
  # Note that we run docs checks before PR checks because they are likely to invalidate a whole run anyway
  - task: Powershell@2
    displayName: 'Generate Documentation'
@ -59,7 +71,7 @@ jobs:
      PathtoPublish: '$(DiffFile)'
      ArtifactName: 'format.diff'
  - task: PowerShell@2
-    displayName: '*** Test Modified Ports for ${{ parameters.triplet }}'
+    displayName: '*** Test Modified Ports'
    inputs:
      failOnStderr: true
      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
@ -69,43 +81,7 @@ jobs:
    displayName: 'Validate version files'
    condition: eq('${{ parameters.triplet }}', '${{ variables.ExtraChecksTriplet }}')
    inputs:
-      targetType: inline
-      script: |
-        ./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
-        ForEach-Object -Begin {
-          $long_error = ''
-        } -Process {
-          if ($long_error -ne '' -and $_ -match '^$|^       ') {
-             # Extend multi-line message
-             $long_error = -join($long_error, "%0D%0A", $_ -replace '^       ','' `
-               -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
-          } else {
-            if ($long_error -ne '') {
-              # Flush multi-line message
-              $long_error
-              $long_error = ''
-            }
-            if ($_ -match '^Error: ') {
-              # Start multi-line message
-              $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
-                -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
-            } else {
-              # Normal line
-              $_
-            }
-          }
-        } -End {
-          if ($long_error -ne '') {
-            # Flush multi-line message
-            $long_error
-          }
-        }
-      pwsh: true
-  - task: PowerShell@2
-    displayName: 'Report on Disk Space After Build'
-    condition: always()
-    inputs:
-      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
+      filePath: 'scripts/azure-pipelines/windows/validate-version-files.ps1'
      pwsh: true
  - task: PublishBuildArtifacts@1
    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
--- a/scripts/azure-pipelines/windows/bootstrap-from-source.cmd
+++ b/scripts/azure-pipelines/windows/bootstrap-from-source.cmd
@ -0,0 +1,7 @@
+call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
+git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool
+git -C vcpkg-tool switch -d %1
+rmdir /s /q build.x86.release > nul 2> nul
+cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x86.release -S vcpkg-tool
+ninja.exe -C build.x86.release
+move build.x86.release\vcpkg.exe vcpkg.exe
--- a/scripts/azure-pipelines/windows/validate-version-files.ps1
+++ b/scripts/azure-pipelines/windows/validate-version-files.ps1
@ -0,0 +1,29 @@
+./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose |
+ForEach-Object -Begin {
+  $long_error = ''
+} -Process {
+  if ($long_error -ne '' -and $_ -match '^$|^       ') {
+     # Extend multi-line message
+     $long_error = -join($long_error, "%0D%0A", $_ -replace '^       ','' `
+       -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' )
+  } else {
+    if ($long_error -ne '') {
+      # Flush multi-line message
+      $long_error
+      $long_error = ''
+    }
+    if ($_ -match '^Error: ') {
+      # Start multi-line message
+      $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' `
+        -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5'
+    } else {
+      # Normal line
+      $_
+    }
+  }
+} -End {
+  if ($long_error -ne '') {
+    # Flush multi-line message
+    $long_error
+  }
+}