diff --git a/scripts/azure-pipelines/android/Dockerfile b/scripts/azure-pipelines/android/Dockerfile index adbf124190..2a5ff46255 100644 --- a/scripts/azure-pipelines/android/Dockerfile +++ b/scripts/azure-pipelines/android/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 # DisableDockerDetector "Used to build the container deployed to Azure Container Registry" -FROM ubuntu:focal-20240123 +FROM ubuntu:focal-20240216 ADD https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb /packages-microsoft-prod.deb ADD https://dl.google.com/android/repository/android-ndk-r25c-linux.zip /android-ndk-r25c-linux.zip @@ -13,10 +13,10 @@ ENV APT_PACKAGES="git curl zip unzip tar" ## Common build prereqs ENV APT_PACKAGES="$APT_PACKAGES g++ vim pkg-config cmake ca-certificates" -ENV APT_PACKAGES="$APT_PACKAGES autoconf nasm bison python2 flex build-essential libtool libtool-bin gettext automake autoconf-archive" +ENV APT_PACKAGES="$APT_PACKAGES autoconf nasm bison python2 flex build-essential libtool libtool-bin libltdl-dev gettext automake autoconf-archive" ## Python related -ENV APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-pip python3-venv" +ENV APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-pip python3-venv python3-mako python3-jinja2" ## freeglut ENV APT_PACKAGES="$APT_PACKAGES libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxrandr-dev libxxf86vm-dev" diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 4290e6e422..b5b8692d4f 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -3,10 +3,10 @@ # variables: linux-pool: 'PrLin-1ES-Pool' - windows-pool: 'PrWin-2024-02-16' + windows-pool: 'PrWin-1ES' osx-pool: 'PrOsx-2024-01-18' osx-arm64-pool: 'PrOsx-2024-01-22-arm64' - linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-02-14' + linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-03-14' parameters: - name: vcpkgToolSha diff --git a/scripts/azure-pipelines/create-vmss-helpers.psm1 b/scripts/azure-pipelines/create-vmss-helpers.psm1 index 8ebe8070ef..96ceb066d5 100755 --- a/scripts/azure-pipelines/create-vmss-helpers.psm1 +++ b/scripts/azure-pipelines/create-vmss-helpers.psm1 @@ -55,60 +55,6 @@ function Find-ResourceGroupName { return $result } -<# -.SYNOPSIS -Returns whether there's a name collision for an image in the resource group. - -.DESCRIPTION -Find-ImageNameCollision takes a list of images, and checks if $Test -collides names with any of the image names. - -.PARAMETER Test -The name to test. - -.PARAMETER Images -The list of images. -#> -function Find-ImageNameCollision { - [CmdletBinding()] - Param([string]$Test, $Images) - - foreach ($resource in $Images) { - if ($resource.Name -eq $Test) { - return $true - } - } - - return $false -} - -<# -.SYNOPSIS -Attempts to find a name that does not collide with any images in the resource group. - -.DESCRIPTION -Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the -first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with -any of the resources in the resource group. - -.PARAMETER Prefix -The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?" -#> -function Find-ImageName { - [CmdLetBinding()] - Param([string]$ResourceGroupName, [string]$Prefix) - - $images = Get-AzImage -ResourceGroupName $ResourceGroupName - $result = $Prefix - $suffix = 0 - while (Find-ImageNameCollision -Test $result -Images $images) { - $suffix++ - $result = "$Prefix-$suffix" - } - - return $result -} - <# .SYNOPSIS Generates a random password. @@ -233,6 +179,20 @@ function Create-LockedDownNetwork { [string]$Location ) + $publicIp = New-AzPublicIpAddress ` + -Name "$ResourceGroupName-ip" ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Sku 'Standard' ` + -AllocationMethod 'Static' + + $natGateway = New-AzNatGateway ` + -Name "$ResourceGroupName-nat" ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Sku 'Standard' ` + -PublicIpAddress $publicIp + $allFirewallRules = @() $allFirewallRules += New-AzNetworkSecurityRuleConfig ` @@ -307,7 +267,8 @@ function Create-LockedDownNetwork { -Name $SubnetName ` -AddressPrefix "10.0.0.0/16" ` -NetworkSecurityGroup $NetworkSecurityGroup ` - -ServiceEndpoint "Microsoft.Storage" + -ServiceEndpoint "Microsoft.Storage" ` + -NatGateway $natGateway $VirtualNetworkName = $ResourceGroupName + 'Network' $VirtualNetwork = New-AzVirtualNetwork ` @@ -344,7 +305,6 @@ function Invoke-AzVMRunCommandWithRetries { } Export-ModuleMember -Function Find-ResourceGroupName -Export-ModuleMember -Function Find-ImageName Export-ModuleMember -Function New-Password Export-ModuleMember -Function Wait-Shutdown Export-ModuleMember -Function Sanitize-Name diff --git a/scripts/azure-pipelines/linux/managed-image.json b/scripts/azure-pipelines/linux/managed-image.json index 1f4696f1ed..d9075eb08e 100644 --- a/scripts/azure-pipelines/linux/managed-image.json +++ b/scripts/azure-pipelines/linux/managed-image.json @@ -3,7 +3,7 @@ { "name": "linux-install-packages", "parameters": { - "packages": "git curl zip unzip tar at libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full pkg-config meson nasm cmake ninja-build libxext-dev libxfixes-dev libxrender-dev libxcb1-dev libx11-xcb-dev libxcb-dri3-dev libxcb-glx0-dev libxcb-util0-dev libxkbcommon-dev libxcb-keysyms1-dev libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev libxcb-cursor-dev libkrb5-dev libxcb-res0-dev libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev python3-setuptools python3-mako python3-pip python3-venv nodejs libwayland-dev python-is-python3 guile-2.2-dev libxdamage-dev libdbus-1-dev libxtst-dev haskell-stack golang-go wayland-protocols" + "packages": "git curl zip unzip tar at libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin libltdl-dev flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full pkg-config meson nasm cmake ninja-build libxext-dev libxfixes-dev libxrender-dev libxcb1-dev libx11-xcb-dev libxcb-dri3-dev libxcb-present-dev libxcb-glx0-dev libxcb-util0-dev libxkbcommon-dev libxcb-keysyms1-dev libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev libxcb-cursor-dev libkrb5-dev libxcb-res0-dev libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev python3-setuptools python3-mako python3-pip python3-venv python3-jinja2 nodejs libwayland-dev python-is-python3 guile-2.2-dev libxdamage-dev libdbus-1-dev libxtst-dev haskell-stack golang-go wayland-protocols libbluetooth-dev" } }, { diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index ddcfbddba4..b2e8d888c9 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -31,7 +31,7 @@ APT_PACKAGES="git curl zip unzip tar" APT_PACKAGES="$APT_PACKAGES at libxt-dev gperf libxaw7-dev cifs-utils \ build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev \ libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev \ - libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin \ + libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin libltdl-dev \ flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \ xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \ pkg-config meson nasm cmake ninja-build" @@ -59,10 +59,10 @@ APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev" APT_PACKAGES="$APT_PACKAGES libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev" ## required by mesa -APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako libxcb-dri3-dev" +APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako libxcb-dri3-dev libxcb-present-dev" ## required by some packages to install additional python packages -APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv" +APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv python3-jinja2" ## required by qtwebengine APT_PACKAGES="$APT_PACKAGES nodejs" @@ -94,6 +94,9 @@ APT_PACKAGES="$APT_PACKAGES golang-go" ## required by libdecor and mesa APT_PACKAGES="$APT_PACKAGES wayland-protocols" +## required by robotraconteur +APT_PACKAGES="$APT_PACKAGES libbluetooth-dev" + ## CUDA APT_PACKAGES="$APT_PACKAGES cuda-compiler-12-1 cuda-libraries-dev-12-1 cuda-driver-dev-12-1 \ cuda-cudart-dev-12-1 libcublas-12-1 libcurand-dev-12-1 cuda-nvml-dev-12-1 libcudnn8-dev libnccl2 \ diff --git a/scripts/azure-pipelines/windows/create-image.ps1 b/scripts/azure-pipelines/windows/create-image.ps1 index ad08615895..68f7afddbe 100644 --- a/scripts/azure-pipelines/windows/create-image.ps1 +++ b/scripts/azure-pipelines/windows/create-image.ps1 @@ -15,39 +15,29 @@ or are running from Azure Cloud Shell. #> $Location = 'westus3' -$Prefix = 'Win-' -$Prefix += (Get-Date -Format 'yyyy-MM-dd') +$DatePrefixComponent = Get-Date -Format 'yyyy-MM-dd' +$Prefix = "Win-$DatePrefixComponent" +$GalleryImageVersion = $DatePrefixComponent.Replace('-','.') $VMSize = 'Standard_D8ads_v5' $ProtoVMName = 'PROTOTYPE' $WindowsServerSku = '2022-datacenter-azure-edition' $ErrorActionPreference = 'Stop' -$CudnnBaseUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip' $ProgressActivity = 'Creating Windows Image' -$TotalProgress = 18 +$TotalProgress = 17 $CurrentProgress = 1 -Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking +# Assigning this to another variable helps when running the commands in this script manually for +# debugging +$Root = $PSScriptRoot -#################################################################################################### -Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Creating resource group' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +Import-Module "$Root/../create-vmss-helpers.psm1" -DisableNameChecking -Force -$ResourceGroupName = Find-ResourceGroupName $Prefix $AdminPW = New-Password -New-AzResourceGroup -Name $ResourceGroupName -Location $Location $AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force $Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure) -#################################################################################################### -Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Creating virtual network' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) - -$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location +$VirtualNetwork = Get-AzVirtualNetwork -ResourceGroupName 'vcpkg-image-minting' -Name 'vcpkg-image-mintingNetwork' #################################################################################################### Write-Progress ` @@ -55,14 +45,15 @@ Write-Progress ` -Status 'Creating prototype VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -$NicName = $ResourceGroupName + 'NIC' +$NicName = $Prefix + 'NIC' $Nic = New-AzNetworkInterface ` -Name $NicName ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -Location $Location ` - -Subnet $VirtualNetwork.Subnets[0] + -Subnet $VirtualNetwork.Subnets[0] ` + -EnableAcceleratedNetworking -$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -SecurityType Standard +$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -SecurityType TrustedLaunch -IdentityType SystemAssigned $VM = Set-AzVMOperatingSystem ` -VM $VM ` -Windows ` @@ -81,10 +72,27 @@ $VM = Set-AzVMSourceImage ` $VM = Set-AzVMBootDiagnostic -VM $VM -Disable New-AzVm ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -Location $Location ` -VM $VM +$VMCreated = Get-AzVM -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName +$VMCreatedOsDisk = $VMCreated.StorageProfile.OsDisk.Name + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Granting permissions to use vcpkg-image-minting storage account' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$VcpkgImageMintingAccount = Get-AzStorageAccount -ResourceGroupName 'vcpkg-image-minting' -Name 'vcpkgimageminting' + +# Grant 'Storage Blob Data Reader' (RoleDefinitionId 2a2b9908-6ea1-4ae2-8e65-a410df84e7d1) to the VM +New-AzRoleAssignment ` + -Scope $VcpkgImageMintingAccount.ID ` + -RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' ` + -ObjectId $VMCreated.Identity.PrincipalId + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` @@ -92,10 +100,10 @@ Write-Progress ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $ProvisionImageResult = Invoke-AzVMRunCommandWithRetries ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\deploy-tlssettings.ps1" + -ScriptPath "$Root\deploy-tlssettings.ps1" Write-Host "deploy-tlssettings.ps1 output: $($ProvisionImageResult.value.Message)" Write-Host 'Waiting 1 minute for VM to reboot...' @@ -108,10 +116,10 @@ Write-Progress ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $DeployPsExecResult = Invoke-AzVMRunCommandWithRetries ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\deploy-psexec.ps1" + -ScriptPath "$Root\deploy-psexec.ps1" Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" @@ -119,8 +127,7 @@ Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" function Invoke-ScriptWithPrefix { param( [string]$ScriptName, - [switch]$AddAdminPw, - [string]$CudnnUrl + [switch]$AddAdminPw ) Write-Progress ` @@ -128,20 +135,16 @@ function Invoke-ScriptWithPrefix { -Status "Running provisioning script $ScriptName in VM" ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) - $DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw - $UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw + $DropToAdminUserPrefix = Get-Content "$Root\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw + $UtilityPrefixContent = Get-Content "$Root\utility-prefix.ps1" -Encoding utf8NoBOM -Raw - $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" + $tempScriptFilename = "$env:TEMP\temp-script.txt" try { - $script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw + $script = Get-Content "$Root\$ScriptName" -Encoding utf8NoBOM -Raw if ($AddAdminPw) { $script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix) } - if (-Not ([string]::IsNullOrWhiteSpace($CudnnUrl))) { - $script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnUrl'") - } - $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent); Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM @@ -151,7 +154,7 @@ function Invoke-ScriptWithPrefix { } $InvokeResult = Invoke-AzVMRunCommandWithRetries ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` -ScriptPath $tempScriptFilename ` @@ -163,6 +166,9 @@ function Invoke-ScriptWithPrefix { } } +Invoke-ScriptWithPrefix -ScriptName 'deploy-azcopy.ps1' + +#################################################################################################### Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-sdks.ps1' -AddAdminPw #################################################################################################### @@ -172,27 +178,7 @@ Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw #################################################################################################### -$StorageAccountKeys = Get-AzStorageAccountKey ` - -ResourceGroupName 'vcpkg-image-minting' ` - -Name 'vcpkgimageminting' - -$StorageContext = New-AzStorageContext ` - -StorageAccountName 'vcpkgimageminting' ` - -StorageAccountKey $StorageAccountKeys[0].Value - -$StartTime = [DateTime]::Now -$ExpiryTime = $StartTime.AddDays(1) - -$SetupSasToken = New-AzStorageAccountSASToken ` - -Service Blob ` - -Permission "r" ` - -Context $StorageContext ` - -StartTime $StartTime ` - -ExpiryTime $ExpiryTime ` - -ResourceType Object ` - -Protocol HttpsOnly - -Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -CudnnUrl ($CudnnBaseUrl + $SetupSasToken) +Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw #################################################################################################### Invoke-ScriptWithPrefix -ScriptName 'deploy-inteloneapi.ps1' -AddAdminPw @@ -207,13 +193,13 @@ Write-Progress ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $ProvisionImageResult = Invoke-AzVMRunCommandWithRetries ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\deploy-settings.txt" + -ScriptPath "$Root\deploy-settings.txt" Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)" -Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName +Restart-AzVM -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName #################################################################################################### Write-Progress ` @@ -222,10 +208,10 @@ Write-Progress ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $SysprepResult = Invoke-AzVMRunCommandWithRetries ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\sysprep.ps1" + -ScriptPath "$Root\sysprep.ps1" Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)" @@ -235,7 +221,7 @@ Write-Progress ` -Status 'Waiting for VM to shut down' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Wait-Shutdown -ResourceGroupName $ResourceGroupName -Name $ProtoVMName +Wait-Shutdown -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName #################################################################################################### Write-Progress ` @@ -244,19 +230,25 @@ Write-Progress ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) Stop-AzVM ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -Name $ProtoVMName ` -Force Set-AzVM ` - -ResourceGroupName $ResourceGroupName ` + -ResourceGroupName 'vcpkg-image-minting' ` -Name $ProtoVMName ` -Generalized -$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName -$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID -HyperVGeneration V2 -$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix -New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting' +New-AzGalleryImageVersion ` + -ResourceGroupName 'vcpkg-image-minting' ` + -GalleryName 'vcpkg_gallery_wus3' ` + -GalleryImageDefinitionName 'PrWinWus3-TrustedLaunch' ` + -Name $GalleryImageVersion ` + -Location $Location ` + -SourceImageId $VMCreated.ID ` + -ReplicaCount 1 ` + -StorageAccountType 'Premium_LRS' ` + -PublishingProfileExcludeFromLatest #################################################################################################### Write-Progress ` @@ -264,9 +256,16 @@ Write-Progress ` -Status 'Deleting unused temporary resources' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -Remove-AzResourceGroup $ResourceGroupName -Force +Remove-AzRoleAssignment ` + -Scope $VcpkgImageMintingAccount.ID ` + -RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' ` + -ObjectId $VMCreated.Identity.PrincipalId + +Remove-AzVM -Id $VMCreated.ID -Force +Remove-AzDisk -ResourceGroupName 'vcpkg-image-minting' -Name $VMCreatedOsDisk -Force +Remove-AzNetworkInterface -ResourceGroupName 'vcpkg-image-minting' -Name $NicName -Force #################################################################################################### Write-Progress -Activity $ProgressActivity -Completed -Write-Host "Generated Image: $ImageName" +Write-Host "Generated Image: $GalleryImageVersion" Write-Host 'Finished!' diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 deleted file mode 100644 index 5b87366111..0000000000 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT -# - -<# -.SYNOPSIS -Creates a Windows virtual machine scale set, set up for vcpkg's CI. - -.DESCRIPTION -create-vmss.ps1 creates an Azure Windows VM scale set, set up for vcpkg's CI -system. See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/overview -for more information. - -This script assumes you have installed Azure tools into PowerShell by following the instructions -at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1 -or are running from Azure Cloud Shell. - -.PARAMETER ImageName -The name of the image to deploy into the scale set. -#> - -[CmdLetBinding()] -Param( - [parameter(Mandatory=$true)] - [string]$ImageName -) - -$Location = 'westus3' -$Prefix = 'PrWin-' -$Prefix += (Get-Date -Format 'yyyy-MM-dd') -$VMSize = 'Standard_D32ads_v5' -$LiveVMPrefix = 'BUILD' -$ErrorActionPreference = 'Stop' - -Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking - -$ResourceGroupName = Find-ResourceGroupName $Prefix -$AdminPW = New-Password -$Image = Get-AzImage -ResourceGroupName 'vcpkg-image-minting' -ImageName $ImageName - -New-AzResourceGroup -Name $ResourceGroupName -Location $Location - -$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location -$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig' -$VmssIpConfig = New-AzVmssIpConfig -SubnetId $VirtualNetwork.Subnets[0].Id -Primary -Name $VmssIpConfigName -$VmssName = $ResourceGroupName + 'Vmss' -$Vmss = New-AzVmssConfig ` - -Location $Location ` - -SkuCapacity 0 ` - -SkuName $VMSize ` - -SkuTier 'Standard' ` - -Overprovision $false ` - -UpgradePolicyMode Automatic ` - -EvictionPolicy Delete ` - -Priority Spot ` - -MaxPrice -1 ` - -SecurityType Standard - -$NicName = $ResourceGroupName + 'NIC' -New-AzNetworkInterface ` - -Name $NicName ` - -ResourceGroupName $ResourceGroupName ` - -Location $Location ` - -Subnet $VirtualNetwork.Subnets[0] - -$Vmss = Add-AzVmssNetworkInterfaceConfiguration ` - -VirtualMachineScaleSet $Vmss ` - -Primary $true ` - -IpConfiguration $VmssIpConfig ` - -NetworkSecurityGroupId $VirtualNetwork.Subnets[0].NetworkSecurityGroup.Id ` - -Name $NicName - -$Vmss = Set-AzVmssOsProfile ` - -VirtualMachineScaleSet $Vmss ` - -ComputerNamePrefix $LiveVMPrefix ` - -AdminUsername 'AdminUser' ` - -AdminPassword $AdminPW ` - -WindowsConfigurationProvisionVMAgent $true ` - -WindowsConfigurationEnableAutomaticUpdate $false - -$Vmss = Set-AzVmssStorageProfile ` - -VirtualMachineScaleSet $Vmss ` - -OsDiskCreateOption 'FromImage' ` - -OsDiskCaching ReadOnly ` - -DiffDiskSetting Local ` - -ImageReferenceId $Image.Id - -$Vmss = Set-AzVmssBootDiagnostic ` - -VirtualMachineScaleSet $Vmss ` - -Enabled $false - -$VmssCreated = New-AzVmss ` - -ResourceGroupName $ResourceGroupName ` - -Name $VmssName ` - -VirtualMachineScaleSet $Vmss - -# Grant 'Virtual Machine Contributor' (RoleDefinitionId 9980e02c-c2be-4d73-94e8-173b1dc7cf3c) to -# 'dev-azure-com-vcpkg-scale-set-management' (ObjectId e4fe677f-f905-4f3c-b5c3-d8a2d6812a5b) -New-AzRoleAssignment ` - -Scope $VmssCreated.Id ` - -RoleDefinitionId '9980e02c-c2be-4d73-94e8-173b1dc7cf3c' ` - -ObjectId 'e4fe677f-f905-4f3c-b5c3-d8a2d6812a5b' - -Write-Host "Location: $Location" -Write-Host "Resource group name: $ResourceGroupName" -Write-Host 'Finished!' diff --git a/scripts/azure-pipelines/windows/deploy-azcopy.ps1 b/scripts/azure-pipelines/windows/deploy-azcopy.ps1 new file mode 100644 index 0000000000..4d1eb9b660 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-azcopy.ps1 @@ -0,0 +1,4 @@ +$azcopyZipPath = "$PSScriptRoot\azcopyv10.zip" +& curl.exe -L -o $azcopyZipPath 'https://azcopyvnext.azureedge.net/releases/release-10.23.0-20240129/azcopy_windows_amd64_10.23.0.zip' +Expand-Archive -LiteralPath $azcopyZipPath -DestinationPath $env:PROGRAMFILES +Remove-Item -LiteralPath $azcopyZipPath -Force diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 index 42f292a5bc..a3249d06c9 100644 --- a/scripts/azure-pipelines/windows/deploy-cuda.ps1 +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -5,7 +5,9 @@ # REPLACE WITH UTILITY-PREFIX.ps1 -# REPLACE WITH $CudnnUrl +# If you are running this script outside of our Azure VMs, you will need to download cudnn from NVIDIA and place +# it next to this script. +$CudnnUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip' $CudnnLocalZipPath = "$PSScriptRoot\cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip" @@ -82,16 +84,16 @@ catch { } try { - if ([string]::IsNullOrWhiteSpace($CudnnUrl)) { - if (-Not (Test-Path $CudnnLocalZipPath)) { - throw "CUDNN zip ($CudnnLocalZipPath) was missing, please download from NVidia and place next to this script." - } - + if (Test-Path $CudnnLocalZipPath) { $cudnnZipPath = $CudnnLocalZipPath } else { - Write-Host 'Downloading CUDNN...' + Write-Host 'Attempting to download cudnn. If this fails, you need to agree to NVidia''s EULA, download cudnn, and place it next to this script.' $cudnnZipPath = Get-TempFilePath -Extension 'zip' - curl.exe -L -o $cudnnZipPath -s -S $CudnnUrl + $env:AZCOPY_AUTO_LOGIN_TYPE = 'MSI' + & "$env:PROGRAMFILES\azcopy_windows_amd64_10.23.0\azcopy.exe" copy $CudnnUrl $cudnnZipPath + if ($LASTEXITCODE -ne 0) { + throw 'Failed to download cudnn!' + } } Write-Host "Installing CUDNN to $destination..." diff --git a/scripts/azure-pipelines/windows/provision-entire-image.ps1 b/scripts/azure-pipelines/windows/provision-entire-image.ps1 index 00c2d28cac..80062dd797 100644 --- a/scripts/azure-pipelines/windows/provision-entire-image.ps1 +++ b/scripts/azure-pipelines/windows/provision-entire-image.ps1 @@ -4,6 +4,7 @@ . "$PSScriptRoot\utility-prefix.ps1" . "$PSScriptRoot\deploy-tlssettings.ps1" -RebootIfRequired 0 +. "$PSScriptRoot\deploy-azcopy.ps1" . "$PSScriptRoot\deploy-windows-sdks.ps1" . "$PSScriptRoot\deploy-visual-studio.ps1" . "$PSScriptRoot\deploy-mpi.ps1" diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index e9cf77a963..b593613dda 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -990,9 +990,6 @@ rest-rpc:arm64-windows=skip rest-rpc:x64-linux=skip rest-rpc:x64-osx=skip rest-rpc:arm64-osx=skip -# Missing system libraries -robotraconteur:x64-linux=fail -robotraconteur-companion:x64-linux=fail rpclib:arm64-windows=fail rpclib:arm64-uwp=fail rpclib:x64-uwp=fail