From 8054263f15c8400d6df5fff55fae97394e187368 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Thu, 27 Aug 2020 12:27:39 -0700 Subject: [PATCH] [skia, ompl, capnproto] Update VMs for 2020-08-12 (including VS 16.7) (#12884) Co-authored-by: Nicole Mazzuca --- ports/llvm/0004-fix-dr-1734.patch | 14 ++++ ports/llvm/CONTROL | 3 +- ports/llvm/portfile.cmake | 1 + ports/skia/CONTROL | 2 +- ports/skia/add-missing-tuple.patch | 12 +++ ports/skia/portfile.cmake | 5 ++ scripts/azure-pipelines/azure-pipelines.yml | 6 +- scripts/azure-pipelines/linux/create-vmss.ps1 | 4 +- .../azure-pipelines/linux/provision-image.sh | 5 +- .../windows/azure-pipelines.yml | 6 ++ .../azure-pipelines/windows/create-vmss.ps1 | 8 +- .../windows/provision-image.txt | 83 ++++--------------- scripts/azure-pipelines/windows/sysprep.ps1 | 2 +- scripts/ci.baseline.txt | 18 ++++ 14 files changed, 95 insertions(+), 74 deletions(-) create mode 100644 ports/llvm/0004-fix-dr-1734.patch create mode 100644 ports/skia/add-missing-tuple.patch diff --git a/ports/llvm/0004-fix-dr-1734.patch b/ports/llvm/0004-fix-dr-1734.patch new file mode 100644 index 0000000000..adfbe5a131 --- /dev/null +++ b/ports/llvm/0004-fix-dr-1734.patch @@ -0,0 +1,14 @@ +diff --git a/llvm/include/llvm/Support/type_traits.h b/llvm/include/llvm/Support/type_traits.h +index b7d48e8e1ad..53ba24efc00 100644 +--- a/llvm/include/llvm/Support/type_traits.h ++++ b/llvm/include/llvm/Support/type_traits.h +@@ -177,7 +177,8 @@ class is_trivially_copyable { + (has_deleted_copy_assign || has_trivial_copy_assign) && + (has_deleted_copy_constructor || has_trivial_copy_constructor); + +-#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE ++ // due to DR 1734, a type can be std::is_trivially_copyable but not llvm::is_trivially_copyable ++#if 0 + static_assert(value == std::is_trivially_copyable::value, + "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable"); + #endif diff --git a/ports/llvm/CONTROL b/ports/llvm/CONTROL index 3c25713791..f461f1cb51 100644 --- a/ports/llvm/CONTROL +++ b/ports/llvm/CONTROL @@ -1,5 +1,6 @@ Source: llvm -Version: 10.0.0-4 +Version: 10.0.0 +Port-Version: 5 Homepage: https://llvm.org/ Description: The LLVM Compiler Infrastructure Supports: !uwp diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake index 9351047e65..305e6bf78a 100644 --- a/ports/llvm/portfile.cmake +++ b/ports/llvm/portfile.cmake @@ -12,6 +12,7 @@ vcpkg_from_github( 0001-allow-to-use-commas.patch 0002-fix-install-paths.patch 0003-fix-vs2019-v16.6.patch + 0004-fix-dr-1734.patch ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/ports/skia/CONTROL b/ports/skia/CONTROL index 6b6677ab9e..917ca1199d 100644 --- a/ports/skia/CONTROL +++ b/ports/skia/CONTROL @@ -1,6 +1,6 @@ Source: skia Version: 2020-05-18 -Port-Version: 2 +Port-Version: 3 Homepage: https://skia.org Description: Skia is an open source 2D graphics library which provides common APIs that work across a variety of hardware and software platforms. It serves as the graphics engine for Google Chrome and Chrome OS, Android, Mozilla Firefox and Firefox OS, and many other products. diff --git a/ports/skia/add-missing-tuple.patch b/ports/skia/add-missing-tuple.patch new file mode 100644 index 0000000000..e33d44e3ae --- /dev/null +++ b/ports/skia/add-missing-tuple.patch @@ -0,0 +1,12 @@ +diff --git a/include/private/SkPathRef.h b/include/private/SkPathRef.h +index 7afe4f9..502f83e 100644 +--- a/include/private/SkPathRef.h ++++ b/include/private/SkPathRef.h +@@ -21,6 +21,7 @@ + + #include + #include ++#include + + class SkRBuffer; + class SkWBuffer; diff --git a/ports/skia/portfile.cmake b/ports/skia/portfile.cmake index cdd7254cac..b7cc1f0930 100644 --- a/ports/skia/portfile.cmake +++ b/ports/skia/portfile.cmake @@ -163,6 +163,11 @@ if(CMAKE_HOST_WIN32) endif() +vcpkg_apply_patches( + SOURCE_PATH "${SOURCE_PATH}" + PATCHES add-missing-tuple.patch +) + vcpkg_configure_gn( SOURCE_PATH "${SOURCE_PATH}" OPTIONS_DEBUG "${OPTIONS_DBG}" diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index cda29f7e6a..01810eb0f3 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2020-06-30' - linux-pool: 'PrLin-2020-06-30' + windows-pool: 'PrWin-2020-08-12' + linux-pool: 'PrLin-2020-08-12' stages: - stage: check_cxx_formatting @@ -19,6 +19,7 @@ stages: inputs: filePath: 'scripts/azure-pipelines/windows/Check-CxxFormatting.ps1' arguments: '-Root .' + pwsh: true - stage: check_manifest_formatting displayName: Check the formatting of port manifests pool: $(windows-pool) @@ -36,6 +37,7 @@ stages: inputs: filePath: 'scripts/azure-pipelines/windows/Check-ManifestFormatting.ps1' arguments: '-Root .' + pwsh: true - stage: run_port_ci displayName: 'Run the Port CI' dependsOn: diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 index 84c83dfeae..169a80fc93 100755 --- a/scripts/azure-pipelines/linux/create-vmss.ps1 +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -220,7 +220,7 @@ Write-Progress ` -Status 'Running provisioning script provision-image.sh in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Invoke-AzVMRunCommand ` +$ProvisionImageResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunShellScript' ` @@ -228,6 +228,8 @@ Invoke-AzVMRunCommand ` -Parameter @{StorageAccountName=$StorageAccountName; ` StorageAccountKey=$StorageAccountKey;} +Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)" + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index 5a59161e6e..4936cf7190 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -27,7 +27,10 @@ APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \ APT_PACKAGES="$APT_PACKAGES libkrb5-dev" # Additionally required by mesa -APT_PACKAGES="$APT_PACKAGES python3-setuptools" +APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako" + +# Additionally required by some packages to install additional python packages +APT_PACKAGES="$APT_PACKAGES python3-pip" # Additionally required/installed by Azure DevOps Scale Set Agents APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60" diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 09a95a5194..bb8c7435c3 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -20,11 +20,13 @@ jobs: displayName: 'Initialize Environment' inputs: filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' + pwsh: true - task: PowerShell@2 displayName: 'Report on Disk Space' condition: always() inputs: filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' + pwsh: true # Note: D: is the Azure machines' temporary disk. - script: .\bootstrap-vcpkg.bat displayName: 'Build vcpkg' @@ -49,17 +51,20 @@ jobs: inputs: filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1' arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}' + pwsh: true - task: PowerShell@2 displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactsDirectory $(System.ArtifactsDirectory)' + pwsh: true - task: PowerShell@2 displayName: 'Report on Disk Space After Build' condition: always() inputs: filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' + pwsh: true - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}' inputs: @@ -74,6 +79,7 @@ jobs: script: | ./vcpkg.exe fetch python3 & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\ + pwsh: true - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: file lists for ${{ parameters.triplet }}' condition: always() diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index b1aa5d0ce8..3bb89ee259 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -198,7 +198,7 @@ Write-Progress ` -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Invoke-AzVMRunCommand ` +$ProvisionImageResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` @@ -207,6 +207,8 @@ Invoke-AzVMRunCommand ` StorageAccountName=$StorageAccountName; ` StorageAccountKey=$StorageAccountKey;} +Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)" + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` @@ -221,12 +223,14 @@ Write-Progress ` -Status 'Running provisioning script sysprep.ps1 in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Invoke-AzVMRunCommand ` +$SysprepResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` -ScriptPath "$PSScriptRoot\sysprep.ps1" +Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)" + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt index 798b96e9a1..b43b1bf87c 100644 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ b/scripts/azure-pipelines/windows/provision-image.txt @@ -7,7 +7,7 @@ Sets up a machine to be an image for a scale set. .DESCRIPTION provision-image.ps1 runs on an existing, freshly provisioned virtual machine, -and sets that virtual machine up as a vcpkg build machine. After this is done, +and sets up that virtual machine as a build machine. After this is done, (outside of this script), we take that machine and make it an image to be copied for setting up new VMs in the scale set. @@ -60,10 +60,14 @@ Function Get-TempFilePath { return Join-Path $tempPath $tempName } -if (-not [string]::IsNullOrEmpty($AdminUserPassword)) { - Write-Host "AdminUser password supplied; switching to AdminUser" +$TranscriptPath = 'C:\provision-image-transcript.txt' + +if ([string]::IsNullOrEmpty($AdminUserPassword)) { + Start-Transcript -Path $TranscriptPath +} else { + Write-Host 'AdminUser password supplied; switching to AdminUser.' $PsExecPath = Get-TempFilePath -Extension 'exe' - Write-Host "Downloading psexec to $PsExecPath" + Write-Host "Downloading psexec to: $PsExecPath" & curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe $PsExecArgs = @( '-u', @@ -89,8 +93,11 @@ if (-not [string]::IsNullOrEmpty($AdminUserPassword)) { $PsExecArgs += $StorageAccountKey } - Write-Host "Executing $PsExecPath " + @PsExecArgs - & $PsExecPath @PsExecArgs > C:\ProvisionLog.txt + Write-Host "Executing: $PsExecPath $PsExecArgs" + + $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru + Write-Host 'Reading transcript...' + Get-Content -Path $TranscriptPath Write-Host 'Cleaning up...' Remove-Item $PsExecPath exit $proc.ExitCode @@ -118,7 +125,6 @@ $WindowsSDKUrl = 'https://download.microsoft.com/download/1/c/3/1c3d5161-d9e9-4e $WindowsWDKUrl = 'https://download.microsoft.com/download/1/a/7/1a730121-7aa7-46f7-8978-7db729aa413d/wdk/wdksetup.exe' $MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' -$LlvmUrl = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/LLVM-10.0.0-win64.exe' $CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe' $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + ` @@ -127,7 +133,7 @@ $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_ad 'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + ` 'occupancy_calculator_10.1 fortran_examples_10.1' -$BinSkimUrl = 'https://www.nuget.org/api/v2/package/Microsoft.CodeAnalysis.BinSkim/1.6.0' +$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.0.3/PowerShell-7.0.3-win-x64.msi' $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' @@ -191,7 +197,7 @@ Function InstallVisualStudio { Write-Host 'Downloading Visual Studio...' [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe' curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl - Write-Host "Installing Visual Studio..." + Write-Host 'Installing Visual Studio...' $args = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache') foreach ($workload in $Workloads) { $args += '--add' @@ -362,59 +368,6 @@ Function InstallWindowsWDK { } } -<# -.SYNOPSIS -Installs LLVM. - -.DESCRIPTION -InstallLLVM installs LLVM from the supplied URL. - -.PARAMETER Url -The URL of the LLVM installer. -#> -Function InstallLLVM { - Param( - [String]$Url - ) - - try { - Write-Host 'Downloading LLVM...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing LLVM...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('/S') -NoNewWindow -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install LLVM! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs LLVM. - -.DESCRIPTION -InstallLLVM installs LLVM from the supplied URL. - -.PARAMETER Url -The URL of the LLVM installer. -#> -Function InstallLLVM { - try { - Write-Host 'Downloading LLVM...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing LLVM...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('/S') -NoNewWindow -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install LLVM! $($_.Exception.Message)" - } -} - <# .SYNOPSIS Installs MPI @@ -458,7 +411,7 @@ Installs NVIDIA's CUDA Toolkit. .DESCRIPTION InstallCuda installs the CUDA Toolkit with the features specified as a -space separated list of strings in $Features. +space-separated list of strings in $Features. .PARAMETER Url The URL of the CUDA installer. @@ -504,6 +457,7 @@ if ($null -eq $av) { Write-Host 'AntiVirus not installed, skipping exclusions.' } else { Write-Host 'Configuring AntiVirus exclusions...' + Add-MpPreference -ExclusionPath C:\agent Add-MPPreference -ExclusionPath D:\ Add-MPPreference -ExclusionProcess ninja.exe Add-MPPreference -ExclusionProcess clang-cl.exe @@ -515,10 +469,9 @@ if ($null -eq $av) { InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' InstallWindowsSDK -Url $WindowsSDKUrl InstallWindowsWDK -Url $WindowsWDKUrl -InstallLLVM -Url $LlvmUrl InstallMpi -Url $MpiUrl InstallCuda -Url $CudaUrl -Features $CudaFeatures -InstallZip -Url $BinSkimUrl -Name 'BinSkim' -Dir 'C:\BinSkim' +InstallMSI -Url $PwshUrl -Name 'PowerShell Core' if ([string]::IsNullOrWhiteSpace($StorageAccountName)) { Write-Host 'No storage account name configured.' } else { diff --git a/scripts/azure-pipelines/windows/sysprep.ps1 b/scripts/azure-pipelines/windows/sysprep.ps1 index c0965350d4..a29950044f 100644 --- a/scripts/azure-pipelines/windows/sysprep.ps1 +++ b/scripts/azure-pipelines/windows/sysprep.ps1 @@ -14,4 +14,4 @@ for more information. $ErrorActionPreference = 'Stop' Write-Host 'Running sysprep' -& C:\Windows\system32\sysprep\sysprep.exe /oobe /generalize /shutdown +& C:\Windows\system32\sysprep\sysprep.exe /oobe /generalize /mode:vm /shutdown diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 55f4c33c4e..38b5db2d92 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -153,6 +153,11 @@ caf:x64-uwp=fail caffe2:x86-windows=fail caffe2:arm64-windows=fail cairomm:x64-linux=fail +# capnproto currently triggers an ICE +capnproto:x86-windows=fail +capnproto:x64-windows=fail +capnproto:x64-windows-static=fail +# capnproto doesn't support arm, arm64, or UWP capnproto:arm64-windows=fail capnproto:arm-uwp=fail capnproto:x64-uwp=fail @@ -517,6 +522,8 @@ gmmlib:x64-uwp=fail gmmlib:x64-windows=fail gmmlib:x64-windows-static=fail gmmlib:x86-windows=fail +# VS 2019 16.7 broke gmp UWP https://github.com/microsoft/vcpkg/issues/13172 +gmp:x64-uwp=fail google-cloud-cpp:arm64-windows=fail google-cloud-cpp:arm-uwp=fail google-cloud-cpp:x64-uwp=fail @@ -1181,6 +1188,17 @@ ogre-next:x86-windows = skip ois:arm64-windows=fail ois:arm-uwp=fail ois:x64-uwp=fail +# ompl is vulnerable to some form of race in its dependent ports, and adding 'ode' as a dependency +# does not resolve the issue +# src/ompl/CMakeFiles/ompl.dir/extensions/ode/src/OpenDEStateValidityChecker.cpp.o +# -L/mnt/vcpkg-ci/packages/flann_x64-linux/debug/lib -L/mnt/vcpkg-ci/packages/ode_x64-linux/debug/lib +# -Wl,-rpath,/mnt/vcpkg-ci/packages/flann_x64-linux/debug/lib:/mnt/vcpkg-ci/packages/ode_x64-linux/debug/lib:::::::::::::::::::::::::::::::::::::::::::::::: +# -lode /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_serialization.a +# /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_filesystem.a +# /mnt/vcpkg-ci/installed/x64-linux/debug/lib/libboost_system.a -lpthread && : +# /usr/bin/ld: cannot find -lode +ompl:x64-osx=fail +ompl:x64-linux=fail open62541:arm-uwp=fail open62541:x64-uwp=fail openal-soft:arm-uwp=fail