From 909334f5a3a4c9fbdfba05158ae1cfc9b36030e7 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Wed, 16 Mar 2022 16:29:50 -0700 Subject: [PATCH] Update VMs for VS2022, CUDA 11.6 (#23011) * Update Linux VM CUDA to 11.6. * Update Intel OneAPI * Update Visual Studio to 2022, and remove deployment of v140 and v141 compilers. * Update CUDA to 11.6 on Windows. * Update pools. * cix ci.baseline.txt Co-authored-by: nicole mazzuca --- scripts/azure-pipelines/azure-pipelines.yml | 4 +- .../azure-pipelines/linux/provision-image.sh | 4 +- .../azure-pipelines/windows/create-image.ps1 | 2 +- .../azure-pipelines/windows/deploy-cuda.ps1 | 16 ++--- .../windows/deploy-inteloneapi.ps1 | 4 +- .../windows/deploy-visual-studio.ps1 | 5 +- scripts/ci.baseline.txt | 62 +++++++++++++++++-- 7 files changed, 74 insertions(+), 23 deletions(-) diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index c0f844fa8e..4ef04aa9b3 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2022-03-09' - linux-pool: 'PrLin-2022-03-09' + windows-pool: 'PrWin-2022-03-09-1' + linux-pool: 'PrLin-2022-03-09-1' osx-pool: 'PrOsx-2022-02-04' jobs: diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index 58ca5a4a67..d72c669cca 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -77,8 +77,8 @@ mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" apt-get -y update -apt-get install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 \ - cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev libnccl2 libnccl-dev +apt-get install -y --no-install-recommends cuda-compiler-11-6 cuda-libraries-dev-11-6 cuda-driver-dev-11-6 \ + cuda-cudart-dev-11-6 libcublas-11-6 libcurand-dev-11-6 libcudnn8-dev libnccl2 libnccl-dev # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb diff --git a/scripts/azure-pipelines/windows/create-image.ps1 b/scripts/azure-pipelines/windows/create-image.ps1 index 21fb8c19a8..5d07c59945 100644 --- a/scripts/azure-pipelines/windows/create-image.ps1 +++ b/scripts/azure-pipelines/windows/create-image.ps1 @@ -21,7 +21,7 @@ $VMSize = 'Standard_D32as_v4' $ProtoVMName = 'PROTOTYPE' $WindowsServerSku = '2022-datacenter-g2' $ErrorActionPreference = 'Stop' -$CudnnBaseUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-11.2-windows-x64-v8.1.1.33.zip' +$CudnnBaseUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.3.2.44_cuda11.5-archive.zip' $ProgressActivity = 'Creating Windows Image' $TotalProgress = 18 diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 index ab1cc1c54b..6a53c479c0 100644 --- a/scripts/azure-pipelines/windows/deploy-cuda.ps1 +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -7,17 +7,17 @@ # REPLACE WITH $CudnnUrl -$CudnnLocalZipPath = "$PSScriptRoot\cudnn-11.2-windows-x64-v8.1.1.33.zip" +$CudnnLocalZipPath = "$PSScriptRoot\cudnn-windows-x86_64-8.3.2.44_cuda11.5-archive.zip" -$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' +$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.6.0/network_installers/cuda_11.6.0_windows_network.exe' -$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` - 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` - 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` - 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` - 'occupancy_calculator_11.3 thrust_11.3 ' +$CudaFeatures = 'nvcc_11.6 cuobjdump_11.6 nvprune_11.6 cupti_11.6 memcheck_11.6 nvdisasm_11.6 nvprof_11.6 ' + ` + 'visual_studio_integration_11.6 visual_profiler_11.6 visual_profiler_11.6 cublas_11.6 cublas_dev_11.6 ' + ` + 'cudart_11.6 cufft_11.6 cufft_dev_11.6 curand_11.6 curand_dev_11.6 cusolver_11.6 cusolver_dev_11.6 ' + ` + 'cusparse_11.6 cusparse_dev_11.6 npp_11.6 npp_dev_11.6 nvrtc_11.6 nvrtc_dev_11.6 nvml_dev_11.6 ' + ` + 'occupancy_calculator_11.6 thrust_11.6 ' -$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.3" +$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.6" try { Write-Host 'Downloading CUDA...' diff --git a/scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 b/scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 index 45d8db3c10..a0b32cebb7 100644 --- a/scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 +++ b/scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 @@ -7,7 +7,7 @@ # Seems like only the HPC kit is really needed? #$oneAPIBaseUrl = 'https://registrationcenter-download.intel.com/akdlm/irc_nas/17768/w_BaseKit_p_2021.2.0.2871_offline.exe' -$oneAPIHPCUrl = 'https://registrationcenter-download.intel.com/akdlm/irc_nas/17762/w_HPCKit_p_2021.2.0.2901_offline.exe' +$oneAPIHPCUrl = 'https://registrationcenter-download.intel.com/akdlm/irc_nas/18417/w_HPCKit_p_2022.1.0.93_offline.exe' # Possible oneAPI Base components: #intel.oneapi.win.vtune 2021.1.1-68 true Intel® VTune(TM) Profiler @@ -55,7 +55,7 @@ Function InstallInteloneAPI { Write-Host 'Extracting Intel oneAPI...to folder: ' $extractionPath $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ', '-x ', '-f ' + $extractionPath , '--log extract.log') -Wait -PassThru Write-Host 'Install Intel oneAPI...from folder: ' $extractionPath - $proc = Start-Process -FilePath $extractionPath/bootstrapper.exe -ArgumentList @('-s ', '--action install', "--components=$Components" , '--eula=accept', '--continue-with-optional-error=yes', '-p=NEED_VS2017_INTEGRATION=0', '-p=NEED_VS2019_INTEGRATION=0', '--log-dir=.') -Wait -PassThru + $proc = Start-Process -FilePath $extractionPath/bootstrapper.exe -ArgumentList @('-s ', '--action install', "--components=$Components" , '--eula=accept', '-p=NEED_VS2017_INTEGRATION=0', '-p=NEED_VS2019_INTEGRATION=0', '-p=NEED_VS2022_INTEGRATION=0', '--log-dir=.') -Wait -PassThru $exitCode = $proc.ExitCode if ($exitCode -eq 0) { Write-Host 'Installation successful!' diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 index c8f0196057..2567d4c11d 100644 --- a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 +++ b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 @@ -5,7 +5,7 @@ # REPLACE WITH UTILITY-PREFIX.ps1 -$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe' +$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/17/release/vs_enterprise.exe' $Workloads = @( 'Microsoft.VisualStudio.Workload.NativeDesktop', 'Microsoft.VisualStudio.Workload.Universal', @@ -14,7 +14,6 @@ $Workloads = @( 'Microsoft.VisualStudio.Component.VC.Tools.ARM64', 'Microsoft.VisualStudio.Component.VC.ATL', 'Microsoft.VisualStudio.Component.VC.ATLMFC', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', 'Microsoft.VisualStudio.Component.Windows10SDK.18362', 'Microsoft.VisualStudio.Component.Windows10SDK.19041', 'Microsoft.Net.Component.4.8.SDK', @@ -22,8 +21,6 @@ $Workloads = @( 'Microsoft.Component.NetFX.Native', 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', 'Microsoft.VisualStudio.Component.VC.Llvm.Clang', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64', - 'Microsoft.VisualStudio.Component.VC.140', 'Microsoft.VisualStudio.ComponentGroup.UWP.VC.BuildTools' ) diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index ba428e4a06..306a21b66a 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -74,6 +74,18 @@ caffe2:x86-windows=fail caffe2:arm64-windows=fail c-ares:arm-uwp=fail c-ares:x64-uwp=fail + +# file conflict with dbg-macro +c-dbg-macro:x86-windows=skip +c-dbg-macro:x64-windows=skip +c-dbg-macro:x64-windows-static=skip +c-dbg-macro:x64-windows-static-md=skip +c-dbg-macro:x64-uwp=skip +c-dbg-macro:arm64-windows=skip +c-dbg-macro:arm-uwp=skip +c-dbg-macro:x64-osx=skip +c-dbg-macro:x64-linux=skip + casclib:arm-uwp=fail casclib:x64-uwp=fail catch-classic:arm64-windows = skip @@ -116,8 +128,14 @@ chartdir:x64-windows-static-md=fail chartdir:x64-osx=fail chmlib:arm-uwp=fail chmlib:x64-uwp=fail + +# chromium-base does not yet support VS2022 +chromium-base:x64-windows=fail +chromium-base:x64-windows-static=fail + # Chromium Base requires a recent version of Clang to be installed. chromium-base:x64-linux=skip + civetweb:arm64-windows = skip civetweb:arm-uwp = skip civetweb:x64-uwp = skip @@ -136,8 +154,14 @@ cmcstl2:x86-windows = skip coin:arm64-windows=fail coin:arm-uwp=fail coin:x64-uwp=fail + # https://github.com/colmap/colmap/issues/1421 colmap:x64-osx=fail +# there is an ICE in VS2022 with colmap in release mode +colmap:x86-windows=fail +colmap:x64-windows=fail +colmap:x64-windows-static=fail + concurrencpp:x64-linux=fail constexpr-contracts:x64-linux=fail coolprop:arm-uwp=fail @@ -343,6 +367,10 @@ gmmlib:x64-windows=fail gmmlib:x64-windows-static=fail gmmlib:x64-windows-static-md=fail gmmlib:x86-windows=fail + +# the msbuild for gmp:x64-uwp is broken on VS2022 due to TargetPlatformMinVersion not existing +gmp:x64-uwp=fail + google-cloud-cpp:arm-uwp=fail google-cloud-cpp:x64-uwp=fail gppanel:x64-osx=fail @@ -495,6 +523,10 @@ libgit2:x64-uwp=fail libgo:arm-uwp=fail libgo:x64-uwp=fail libgo:arm64-windows=fail + +# the msbuild for libgpg:x64-uwp is broken on VS2022 due to TargetPlatformMinVersion not existing +libgpg:x64-uwp=fail + libhdfs3:x64-linux=fail libhdfs3:x64-osx=fail libhydrogen:arm64-windows=fail @@ -622,6 +654,7 @@ libui:x64-linux=fail libui:x64-uwp=fail libusb:arm-uwp=fail libusb:x64-uwp=fail +libusb:arm64-windows=fail libusbmuxd:arm-uwp=fail libusbmuxd:x64-uwp=fail libusbmuxd:x64-linux=fail @@ -642,6 +675,7 @@ libvmdk:x64-windows-static-md=skip libvmdk:arm64=skip libvmdk:x64-linux=skip libvmdk:x64-osx=skip +libvpx:arm-uwp=fail libwandio:x86-windows=fail libwandio:x64-windows=fail libwandio:x64-windows-static=fail @@ -1002,6 +1036,10 @@ qpid-proton:arm-uwp=fail qpid-proton:x64-uwp=fail qpid-proton:x64-windows-static=fail qt5-base:arm64-windows=fail + +# qtwebengine:x64-windows has an ICE in VS2022 +qtwebengine:x64-windows=fail + # Skip deprecated Qt module # (remove after 1 year or longer due to vcpkg upgrade not handling removed ports correctly) qt5-canvas3d:x64-linux=skip @@ -1124,12 +1162,12 @@ rtlsdr:x64-linux=fail rtlsdr:x64-osx=fail rttr:arm-uwp=fail rttr:x64-uwp=fail -ryu:arm-uwp=fail -ryu:x64-uwp=fail + +# ryu does not support VS2022 yet +ryu:x64-windows=fail ryu:x64-windows-static=fail ryu:x64-windows-static-md=fail -ryu:x86-windows=fail -ryu::arm64-windows=fail + sciter:arm64-windows=fail sciter:arm-uwp=fail sciter:x64-linux=fail @@ -1247,8 +1285,18 @@ tcl:arm64-windows=fail tcl:x64-uwp=fail telnetpp:arm-uwp=fail telnetpp:x64-uwp=fail + +# tensorflow does not support VS2022 +tensorflow:x64-windows=fail +tensorflow:x64-windows-static=fail +tensorflow:x64-windows-static-md=fail +tensorflow-cc:x64-windows=fail +tensorflow-cc:x64-windows-static=fail +tensorflow-cc:x64-windows-static-md=fail + tensorflow:x64-osx=fail tensorflow-cc:x64-osx=fail + theia:arm64-windows = skip theia:arm-uwp = skip theia:x64-uwp = skip @@ -1393,7 +1441,13 @@ quantlib:x64-windows-static-md=fail sentencepiece:x64-windows-static-md=fail symengine:x64-windows-static-md=fail unicorn:x64-windows-static-md=fail + +# the version of v8 we have in the repo doesn't support VS2022 +v8:x86-windows=fail +v8:x64-windows=fail +v8:x64-windows-static=fail v8:x64-windows-static-md=fail + yato:x64-windows-static-md=fail zyre:x64-windows-static-md=fail usbmuxd:x64-windows-static-md=fail