diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 55407f184a..02dc9ede08 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -6,53 +6,68 @@ variables: linux-pool: 'PrLin-2022-05-19' osx-pool: 'PrOsx-2022-02-04' +parameters: + - name: vcpkgToolSha + displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' + type: string + default: 'use default' + jobs: - template: windows/azure-pipelines.yml parameters: triplet: x86-windows jobName: x86_windows poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: x64-windows jobName: x64_windows poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: x64-windows-static jobName: x64_windows_static poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: x64-windows-static-md jobName: x64_windows_static_md poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: x64-uwp jobName: x64_uwp poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: arm64-windows jobName: arm64_windows poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: windows/azure-pipelines.yml parameters: triplet: arm-uwp jobName: arm_uwp poolName: $(windows-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: osx/azure-pipelines.yml parameters: poolName: $(osx-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - template: linux/azure-pipelines.yml parameters: poolName: $(linux-pool) + vcpkgToolSha: ${{ parameters.vcpkgToolSha }} diff --git a/scripts/azure-pipelines/bootstrap-from-source.sh b/scripts/azure-pipelines/bootstrap-from-source.sh new file mode 100755 index 0000000000..b998568eb8 --- /dev/null +++ b/scripts/azure-pipelines/bootstrap-from-source.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool +git -C vcpkg-tool switch -d $1 +rm -rf build.x64.release +cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x64.release -S vcpkg-tool +ninja -C build.x64.release +mv build.x64.release/vcpkg vcpkg diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 92da1eea40..ad50322393 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -2,6 +2,14 @@ # SPDX-License-Identifier: MIT # +parameters: + - name: vcpkgToolSha + displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' + type: string + default: 'use default' + - name: poolName + type: string + jobs: - job: x64_linux pool: @@ -22,35 +30,27 @@ jobs: value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)" steps: - - bash: df -h - displayName: 'Report on Disk Space' + # Note: /mnt is the Azure machines' temporary disk. - bash: | sudo mkdir /home/agent -m=777 sudo chown `id -u` /home/agent - exit 0 - displayName: 'Create /home/agent' - # Note: /mnt is the Azure machines' temporary disk. - - bash: | - sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }} sudo mkdir ${{ variables.WORKING_ROOT }} -m=777 sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777 exit 0 - displayName: 'Create ${{ variables.WORKING_ROOT }} and ${{ variables.VCPKG_DOWNLOADS }}' - - task: Bash@3 + displayName: 'Create working directories' + - bash: ./bootstrap-vcpkg.sh displayName: 'Bootstrap vcpkg' - inputs: - filePath: bootstrap-vcpkg.sh + condition: eq('use default', '${{ parameters.vcpkgToolSha }}') + - bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }} + displayName: "Build vcpkg with CMake" + condition: ne('use default', '${{ parameters.vcpkgToolSha }}') - task: PowerShell@2 - displayName: '*** Test Modified Ports for x64-linux' + displayName: '*** Test Modified Ports' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -BinarySourceStub "$(X_VCPKG_BINARY_SOURCE_STUB)" -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)' pwsh: true - - bash: | - df -h - displayName: 'Report on Disk Space After Build' - condition: always() - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: failure logs for x64-linux' inputs: diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 2feadc42c1..a735d66df4 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -3,8 +3,12 @@ # parameters: -- name: poolName - type: string + - name: vcpkgToolSha + displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' + type: string + default: 'use default' + - name: poolName + type: string jobs: - job: x64_osx @@ -26,20 +30,21 @@ jobs: value: "x-azurl,$(root-url-ea),$(sas-ea),readwrite" steps: - - bash: df -h - displayName: 'Report on Disk Space' - bash: | - sudo rm -rf ${{ variables.VCPKG_DOWNLOADS }} || 0 sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0 sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0 exit 0 displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}' - - task: Bash@3 + - bash: ./bootstrap-vcpkg.sh displayName: 'Bootstrap vcpkg' - inputs: - filePath: bootstrap-vcpkg.sh + condition: eq('use default', '${{ parameters.vcpkgToolSha }}') + - bash: | + brew install cmake + ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }} + displayName: "Build vcpkg with CMake" + condition: ne('use default', '${{ parameters.vcpkgToolSha }}') - task: PowerShell@2 - displayName: '*** Test Modified Ports for x64-osx' + displayName: '*** Test Modified Ports' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' @@ -50,18 +55,13 @@ jobs: -WorkingRoot "${{ variables.WORKING_ROOT }}" -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" pwsh: true - - bash: | - df -h - displayName: 'Report on Disk Space After Build' - condition: always() - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: failure logs for x64-osx' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs' ArtifactName: 'failure logs for x64-osx${{ variables.Postfix }}' condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') - - bash: | - python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/ + - bash: python3 scripts/file_script.py /Users/vagrant/Data/installed/vcpkg/info/ displayName: 'Build a file list for all packages' condition: always() - task: PublishBuildArtifacts@1 diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json index 45917784a5..75cb83719e 100644 --- a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json +++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json @@ -1,10 +1,11 @@ { "$schema": "./vagrant-box-configuration.schema.json", "brew": [ - "autoconf", "autoconf-archive", + "autoconf", "automake", "bison", + "cmake", "gettext", "gfortran", "gperf", @@ -13,6 +14,7 @@ "meson", "mono", "nasm", + "ninja", "pkg-config", "texinfo", "yasm" diff --git a/scripts/azure-pipelines/windows-unstable/job.yml b/scripts/azure-pipelines/windows-unstable/job.yml index bac59834b2..325da69e1f 100644 --- a/scripts/azure-pipelines/windows-unstable/job.yml +++ b/scripts/azure-pipelines/windows-unstable/job.yml @@ -8,7 +8,7 @@ jobs: name: $(unstable-pool) workspace: clean: resources - timeoutInMinutes: 1440 # 1 day + timeoutInMinutes: 2880 # 2 days variables: - name: WORKING_ROOT value: D:\ @@ -59,30 +59,18 @@ jobs: filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1' arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret' pwsh: true - - task: PowerShell@2 - displayName: 'Report on Disk Space' - condition: always() - inputs: - filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' - pwsh: true - script: .\bootstrap-vcpkg.bat - displayName: 'Build vcpkg' + displayName: 'Bootstrap vcpkg' - task: PowerShell@2 - displayName: '*** Test Modified Ports and Prepare Test Logs ***' + displayName: '*** Test Modified Ports' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -PassingIsPassing' pwsh: true - - task: PowerShell@2 - displayName: 'Report on Disk Space After Build' - condition: always() - inputs: - filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' - pwsh: true - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs' ArtifactName: 'failure logs for ${{ parameters.triplet }}' - condition: always() + condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 540d5af008..6f9220b0c9 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -2,6 +2,20 @@ # SPDX-License-Identifier: MIT # +parameters: + - name: vcpkgToolSha + displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' + type: string + default: 'use default' + - name: triplet + type: string + default: 'x86-windows' + - name: jobName + type: string + default: 'x86_windows' + - name: poolName + type: string + jobs: - job: ${{ parameters.jobName }} pool: @@ -26,14 +40,12 @@ jobs: value: "x-azblob,$(root-bin-url-ea),$(sas-bin-ea)" steps: - - task: PowerShell@2 - displayName: 'Report on Disk Space' - condition: always() - inputs: - filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' - pwsh: true - script: .\bootstrap-vcpkg.bat displayName: 'Bootstrap vcpkg' + condition: eq('use default', '${{ parameters.vcpkgToolSha }}') + - script: .\scripts\azure-pipelines\windows\bootstrap-from-source.cmd ${{ parameters.vcpkgToolSha }} + displayName: "Build vcpkg with CMake" + condition: ne('use default', '${{ parameters.vcpkgToolSha }}') # Note that we run docs checks before PR checks because they are likely to invalidate a whole run anyway - task: Powershell@2 displayName: 'Generate Documentation' @@ -59,7 +71,7 @@ jobs: PathtoPublish: '$(DiffFile)' ArtifactName: 'format.diff' - task: PowerShell@2 - displayName: '*** Test Modified Ports for ${{ parameters.triplet }}' + displayName: '*** Test Modified Ports' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' @@ -69,43 +81,7 @@ jobs: displayName: 'Validate version files' condition: eq('${{ parameters.triplet }}', '${{ variables.ExtraChecksTriplet }}') inputs: - targetType: inline - script: | - ./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose | - ForEach-Object -Begin { - $long_error = '' - } -Process { - if ($long_error -ne '' -and $_ -match '^$|^ ') { - # Extend multi-line message - $long_error = -join($long_error, "%0D%0A", $_ -replace '^ ','' ` - -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' ) - } else { - if ($long_error -ne '') { - # Flush multi-line message - $long_error - $long_error = '' - } - if ($_ -match '^Error: ') { - # Start multi-line message - $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' ` - -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5' - } else { - # Normal line - $_ - } - } - } -End { - if ($long_error -ne '') { - # Flush multi-line message - $long_error - } - } - pwsh: true - - task: PowerShell@2 - displayName: 'Report on Disk Space After Build' - condition: always() - inputs: - filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' + filePath: 'scripts/azure-pipelines/windows/validate-version-files.ps1' pwsh: true - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}' diff --git a/scripts/azure-pipelines/windows/bootstrap-from-source.cmd b/scripts/azure-pipelines/windows/bootstrap-from-source.cmd new file mode 100644 index 0000000000..8f8aae3374 --- /dev/null +++ b/scripts/azure-pipelines/windows/bootstrap-from-source.cmd @@ -0,0 +1,7 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86 +git clone https://github.com/microsoft/vcpkg-tool vcpkg-tool +git -C vcpkg-tool switch -d %1 +rmdir /s /q build.x86.release > nul 2> nul +cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=OFF -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=OFF -B build.x86.release -S vcpkg-tool +ninja.exe -C build.x86.release +move build.x86.release\vcpkg.exe vcpkg.exe diff --git a/scripts/azure-pipelines/windows/validate-version-files.ps1 b/scripts/azure-pipelines/windows/validate-version-files.ps1 new file mode 100644 index 0000000000..1bcb05a7bd --- /dev/null +++ b/scripts/azure-pipelines/windows/validate-version-files.ps1 @@ -0,0 +1,29 @@ +./vcpkg.exe --feature-flags=versions x-ci-verify-versions --verbose | +ForEach-Object -Begin { + $long_error = '' +} -Process { + if ($long_error -ne '' -and $_ -match '^$|^ ') { + # Extend multi-line message + $long_error = -join($long_error, "%0D%0A", $_ -replace '^ ','' ` + -replace '(git add) [^ ]*\\ports\\([^ ]*)', '$1 ports/$2' ) + } else { + if ($long_error -ne '') { + # Flush multi-line message + $long_error + $long_error = '' + } + if ($_ -match '^Error: ') { + # Start multi-line message + $long_error = $_ -replace '^Error: ', '##vso[task.logissue type=error]' ` + -replace '(^##vso[^\]]*)](.*) [^ ]*\\versions\\(.-)\\(.*.json)(.*)', '$1;sourcepath=versions/$3/$4;linenumber=2]$2 version/$3/$4$5' + } else { + # Normal line + $_ + } + } +} -End { + if ($long_error -ne '') { + # Flush multi-line message + $long_error + } +}