mirror of
https://github.com/microsoft/vcpkg.git
synced 2024-12-27 02:11:58 +08:00
Patch Tuesday March 2024 (#37466)
This contains high priority active security things to adopt trusted launch, and managed identity rather than SAS tokens when minting the images, and 1ES Hosted Pools. Some instructions are rough around the edges because I'm not sure everything is repeatable yet while this is all in flux...
This commit is contained in:
parent
cf4ebef229
commit
9cbab417e4
@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:1.4
|
||||
# DisableDockerDetector "Used to build the container deployed to Azure Container Registry"
|
||||
FROM ubuntu:focal-20240123
|
||||
FROM ubuntu:focal-20240216
|
||||
|
||||
ADD https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb /packages-microsoft-prod.deb
|
||||
ADD https://dl.google.com/android/repository/android-ndk-r25c-linux.zip /android-ndk-r25c-linux.zip
|
||||
@ -13,10 +13,10 @@ ENV APT_PACKAGES="git curl zip unzip tar"
|
||||
## Common build prereqs
|
||||
ENV APT_PACKAGES="$APT_PACKAGES g++ vim pkg-config cmake ca-certificates"
|
||||
|
||||
ENV APT_PACKAGES="$APT_PACKAGES autoconf nasm bison python2 flex build-essential libtool libtool-bin gettext automake autoconf-archive"
|
||||
ENV APT_PACKAGES="$APT_PACKAGES autoconf nasm bison python2 flex build-essential libtool libtool-bin libltdl-dev gettext automake autoconf-archive"
|
||||
|
||||
## Python related
|
||||
ENV APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-pip python3-venv"
|
||||
ENV APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-pip python3-venv python3-mako python3-jinja2"
|
||||
|
||||
## freeglut
|
||||
ENV APT_PACKAGES="$APT_PACKAGES libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxrandr-dev libxxf86vm-dev"
|
||||
|
@ -3,10 +3,10 @@
|
||||
#
|
||||
variables:
|
||||
linux-pool: 'PrLin-1ES-Pool'
|
||||
windows-pool: 'PrWin-2024-02-16'
|
||||
windows-pool: 'PrWin-1ES'
|
||||
osx-pool: 'PrOsx-2024-01-18'
|
||||
osx-arm64-pool: 'PrOsx-2024-01-22-arm64'
|
||||
linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-02-14'
|
||||
linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-03-14'
|
||||
|
||||
parameters:
|
||||
- name: vcpkgToolSha
|
||||
|
@ -55,60 +55,6 @@ function Find-ResourceGroupName {
|
||||
return $result
|
||||
}
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Returns whether there's a name collision for an image in the resource group.
|
||||
|
||||
.DESCRIPTION
|
||||
Find-ImageNameCollision takes a list of images, and checks if $Test
|
||||
collides names with any of the image names.
|
||||
|
||||
.PARAMETER Test
|
||||
The name to test.
|
||||
|
||||
.PARAMETER Images
|
||||
The list of images.
|
||||
#>
|
||||
function Find-ImageNameCollision {
|
||||
[CmdletBinding()]
|
||||
Param([string]$Test, $Images)
|
||||
|
||||
foreach ($resource in $Images) {
|
||||
if ($resource.Name -eq $Test) {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
|
||||
return $false
|
||||
}
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Attempts to find a name that does not collide with any images in the resource group.
|
||||
|
||||
.DESCRIPTION
|
||||
Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the
|
||||
first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with
|
||||
any of the resources in the resource group.
|
||||
|
||||
.PARAMETER Prefix
|
||||
The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?"
|
||||
#>
|
||||
function Find-ImageName {
|
||||
[CmdLetBinding()]
|
||||
Param([string]$ResourceGroupName, [string]$Prefix)
|
||||
|
||||
$images = Get-AzImage -ResourceGroupName $ResourceGroupName
|
||||
$result = $Prefix
|
||||
$suffix = 0
|
||||
while (Find-ImageNameCollision -Test $result -Images $images) {
|
||||
$suffix++
|
||||
$result = "$Prefix-$suffix"
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Generates a random password.
|
||||
@ -233,6 +179,20 @@ function Create-LockedDownNetwork {
|
||||
[string]$Location
|
||||
)
|
||||
|
||||
$publicIp = New-AzPublicIpAddress `
|
||||
-Name "$ResourceGroupName-ip" `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-Location $Location `
|
||||
-Sku 'Standard' `
|
||||
-AllocationMethod 'Static'
|
||||
|
||||
$natGateway = New-AzNatGateway `
|
||||
-Name "$ResourceGroupName-nat" `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-Location $Location `
|
||||
-Sku 'Standard' `
|
||||
-PublicIpAddress $publicIp
|
||||
|
||||
$allFirewallRules = @()
|
||||
|
||||
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
|
||||
@ -307,7 +267,8 @@ function Create-LockedDownNetwork {
|
||||
-Name $SubnetName `
|
||||
-AddressPrefix "10.0.0.0/16" `
|
||||
-NetworkSecurityGroup $NetworkSecurityGroup `
|
||||
-ServiceEndpoint "Microsoft.Storage"
|
||||
-ServiceEndpoint "Microsoft.Storage" `
|
||||
-NatGateway $natGateway
|
||||
|
||||
$VirtualNetworkName = $ResourceGroupName + 'Network'
|
||||
$VirtualNetwork = New-AzVirtualNetwork `
|
||||
@ -344,7 +305,6 @@ function Invoke-AzVMRunCommandWithRetries {
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function Find-ResourceGroupName
|
||||
Export-ModuleMember -Function Find-ImageName
|
||||
Export-ModuleMember -Function New-Password
|
||||
Export-ModuleMember -Function Wait-Shutdown
|
||||
Export-ModuleMember -Function Sanitize-Name
|
||||
|
@ -3,7 +3,7 @@
|
||||
{
|
||||
"name": "linux-install-packages",
|
||||
"parameters": {
|
||||
"packages": "git curl zip unzip tar at libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full pkg-config meson nasm cmake ninja-build libxext-dev libxfixes-dev libxrender-dev libxcb1-dev libx11-xcb-dev libxcb-dri3-dev libxcb-glx0-dev libxcb-util0-dev libxkbcommon-dev libxcb-keysyms1-dev libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev libxcb-cursor-dev libkrb5-dev libxcb-res0-dev libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev python3-setuptools python3-mako python3-pip python3-venv nodejs libwayland-dev python-is-python3 guile-2.2-dev libxdamage-dev libdbus-1-dev libxtst-dev haskell-stack golang-go wayland-protocols"
|
||||
"packages": "git curl zip unzip tar at libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin libltdl-dev flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full pkg-config meson nasm cmake ninja-build libxext-dev libxfixes-dev libxrender-dev libxcb1-dev libx11-xcb-dev libxcb-dri3-dev libxcb-present-dev libxcb-glx0-dev libxcb-util0-dev libxkbcommon-dev libxcb-keysyms1-dev libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-xinerama0-dev libxcb-xkb-dev libxcb-xinput-dev libxcb-cursor-dev libkrb5-dev libxcb-res0-dev libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev python3-setuptools python3-mako python3-pip python3-venv python3-jinja2 nodejs libwayland-dev python-is-python3 guile-2.2-dev libxdamage-dev libdbus-1-dev libxtst-dev haskell-stack golang-go wayland-protocols libbluetooth-dev"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
@ -31,7 +31,7 @@ APT_PACKAGES="git curl zip unzip tar"
|
||||
APT_PACKAGES="$APT_PACKAGES at libxt-dev gperf libxaw7-dev cifs-utils \
|
||||
build-essential g++ gfortran libx11-dev libxkbcommon-x11-dev libxi-dev \
|
||||
libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxxf86vm-dev \
|
||||
libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin \
|
||||
libxcursor-dev yasm libnuma1 libnuma-dev libtool-bin libltdl-dev \
|
||||
flex bison libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \
|
||||
xutils-dev dh-autoreconf autoconf-archive libgles2-mesa-dev ruby-full \
|
||||
pkg-config meson nasm cmake ninja-build"
|
||||
@ -59,10 +59,10 @@ APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
|
||||
APT_PACKAGES="$APT_PACKAGES libxcb-keysyms1-dev libxcb-xkb-dev libxcb-record0-dev"
|
||||
|
||||
## required by mesa
|
||||
APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako libxcb-dri3-dev"
|
||||
APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako libxcb-dri3-dev libxcb-present-dev"
|
||||
|
||||
## required by some packages to install additional python packages
|
||||
APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv"
|
||||
APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv python3-jinja2"
|
||||
|
||||
## required by qtwebengine
|
||||
APT_PACKAGES="$APT_PACKAGES nodejs"
|
||||
@ -94,6 +94,9 @@ APT_PACKAGES="$APT_PACKAGES golang-go"
|
||||
## required by libdecor and mesa
|
||||
APT_PACKAGES="$APT_PACKAGES wayland-protocols"
|
||||
|
||||
## required by robotraconteur
|
||||
APT_PACKAGES="$APT_PACKAGES libbluetooth-dev"
|
||||
|
||||
## CUDA
|
||||
APT_PACKAGES="$APT_PACKAGES cuda-compiler-12-1 cuda-libraries-dev-12-1 cuda-driver-dev-12-1 \
|
||||
cuda-cudart-dev-12-1 libcublas-12-1 libcurand-dev-12-1 cuda-nvml-dev-12-1 libcudnn8-dev libnccl2 \
|
||||
|
@ -15,39 +15,29 @@ or are running from Azure Cloud Shell.
|
||||
#>
|
||||
|
||||
$Location = 'westus3'
|
||||
$Prefix = 'Win-'
|
||||
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
|
||||
$DatePrefixComponent = Get-Date -Format 'yyyy-MM-dd'
|
||||
$Prefix = "Win-$DatePrefixComponent"
|
||||
$GalleryImageVersion = $DatePrefixComponent.Replace('-','.')
|
||||
$VMSize = 'Standard_D8ads_v5'
|
||||
$ProtoVMName = 'PROTOTYPE'
|
||||
$WindowsServerSku = '2022-datacenter-azure-edition'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$CudnnBaseUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip'
|
||||
|
||||
$ProgressActivity = 'Creating Windows Image'
|
||||
$TotalProgress = 18
|
||||
$TotalProgress = 17
|
||||
$CurrentProgress = 1
|
||||
|
||||
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
|
||||
# Assigning this to another variable helps when running the commands in this script manually for
|
||||
# debugging
|
||||
$Root = $PSScriptRoot
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
-Activity $ProgressActivity `
|
||||
-Status 'Creating resource group' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
Import-Module "$Root/../create-vmss-helpers.psm1" -DisableNameChecking -Force
|
||||
|
||||
$ResourceGroupName = Find-ResourceGroupName $Prefix
|
||||
$AdminPW = New-Password
|
||||
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
|
||||
$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
|
||||
$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
-Activity $ProgressActivity `
|
||||
-Status 'Creating virtual network' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
|
||||
$VirtualNetwork = Get-AzVirtualNetwork -ResourceGroupName 'vcpkg-image-minting' -Name 'vcpkg-image-mintingNetwork'
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
@ -55,14 +45,15 @@ Write-Progress `
|
||||
-Status 'Creating prototype VM' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$NicName = $ResourceGroupName + 'NIC'
|
||||
$NicName = $Prefix + 'NIC'
|
||||
$Nic = New-AzNetworkInterface `
|
||||
-Name $NicName `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-Location $Location `
|
||||
-Subnet $VirtualNetwork.Subnets[0]
|
||||
-Subnet $VirtualNetwork.Subnets[0] `
|
||||
-EnableAcceleratedNetworking
|
||||
|
||||
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -SecurityType Standard
|
||||
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -SecurityType TrustedLaunch -IdentityType SystemAssigned
|
||||
$VM = Set-AzVMOperatingSystem `
|
||||
-VM $VM `
|
||||
-Windows `
|
||||
@ -81,10 +72,27 @@ $VM = Set-AzVMSourceImage `
|
||||
|
||||
$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
|
||||
New-AzVm `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-Location $Location `
|
||||
-VM $VM
|
||||
|
||||
$VMCreated = Get-AzVM -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName
|
||||
$VMCreatedOsDisk = $VMCreated.StorageProfile.OsDisk.Name
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
-Activity $ProgressActivity `
|
||||
-Status 'Granting permissions to use vcpkg-image-minting storage account' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$VcpkgImageMintingAccount = Get-AzStorageAccount -ResourceGroupName 'vcpkg-image-minting' -Name 'vcpkgimageminting'
|
||||
|
||||
# Grant 'Storage Blob Data Reader' (RoleDefinitionId 2a2b9908-6ea1-4ae2-8e65-a410df84e7d1) to the VM
|
||||
New-AzRoleAssignment `
|
||||
-Scope $VcpkgImageMintingAccount.ID `
|
||||
-RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' `
|
||||
-ObjectId $VMCreated.Identity.PrincipalId
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
-Activity $ProgressActivity `
|
||||
@ -92,10 +100,10 @@ Write-Progress `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-VMName $ProtoVMName `
|
||||
-CommandId 'RunPowerShellScript' `
|
||||
-ScriptPath "$PSScriptRoot\deploy-tlssettings.ps1"
|
||||
-ScriptPath "$Root\deploy-tlssettings.ps1"
|
||||
|
||||
Write-Host "deploy-tlssettings.ps1 output: $($ProvisionImageResult.value.Message)"
|
||||
Write-Host 'Waiting 1 minute for VM to reboot...'
|
||||
@ -108,10 +116,10 @@ Write-Progress `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$DeployPsExecResult = Invoke-AzVMRunCommandWithRetries `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-VMName $ProtoVMName `
|
||||
-CommandId 'RunPowerShellScript' `
|
||||
-ScriptPath "$PSScriptRoot\deploy-psexec.ps1"
|
||||
-ScriptPath "$Root\deploy-psexec.ps1"
|
||||
|
||||
Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)"
|
||||
|
||||
@ -119,8 +127,7 @@ Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)"
|
||||
function Invoke-ScriptWithPrefix {
|
||||
param(
|
||||
[string]$ScriptName,
|
||||
[switch]$AddAdminPw,
|
||||
[string]$CudnnUrl
|
||||
[switch]$AddAdminPw
|
||||
)
|
||||
|
||||
Write-Progress `
|
||||
@ -128,20 +135,16 @@ function Invoke-ScriptWithPrefix {
|
||||
-Status "Running provisioning script $ScriptName in VM" `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw
|
||||
$UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw
|
||||
$DropToAdminUserPrefix = Get-Content "$Root\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw
|
||||
$UtilityPrefixContent = Get-Content "$Root\utility-prefix.ps1" -Encoding utf8NoBOM -Raw
|
||||
|
||||
$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
|
||||
$tempScriptFilename = "$env:TEMP\temp-script.txt"
|
||||
try {
|
||||
$script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw
|
||||
$script = Get-Content "$Root\$ScriptName" -Encoding utf8NoBOM -Raw
|
||||
if ($AddAdminPw) {
|
||||
$script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix)
|
||||
}
|
||||
|
||||
if (-Not ([string]::IsNullOrWhiteSpace($CudnnUrl))) {
|
||||
$script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnUrl'")
|
||||
}
|
||||
|
||||
$script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent);
|
||||
Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
|
||||
|
||||
@ -151,7 +154,7 @@ function Invoke-ScriptWithPrefix {
|
||||
}
|
||||
|
||||
$InvokeResult = Invoke-AzVMRunCommandWithRetries `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-VMName $ProtoVMName `
|
||||
-CommandId 'RunPowerShellScript' `
|
||||
-ScriptPath $tempScriptFilename `
|
||||
@ -163,6 +166,9 @@ function Invoke-ScriptWithPrefix {
|
||||
}
|
||||
}
|
||||
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-azcopy.ps1'
|
||||
|
||||
####################################################################################################
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-sdks.ps1' -AddAdminPw
|
||||
|
||||
####################################################################################################
|
||||
@ -172,27 +178,7 @@ Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw
|
||||
|
||||
####################################################################################################
|
||||
$StorageAccountKeys = Get-AzStorageAccountKey `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-Name 'vcpkgimageminting'
|
||||
|
||||
$StorageContext = New-AzStorageContext `
|
||||
-StorageAccountName 'vcpkgimageminting' `
|
||||
-StorageAccountKey $StorageAccountKeys[0].Value
|
||||
|
||||
$StartTime = [DateTime]::Now
|
||||
$ExpiryTime = $StartTime.AddDays(1)
|
||||
|
||||
$SetupSasToken = New-AzStorageAccountSASToken `
|
||||
-Service Blob `
|
||||
-Permission "r" `
|
||||
-Context $StorageContext `
|
||||
-StartTime $StartTime `
|
||||
-ExpiryTime $ExpiryTime `
|
||||
-ResourceType Object `
|
||||
-Protocol HttpsOnly
|
||||
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -CudnnUrl ($CudnnBaseUrl + $SetupSasToken)
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw
|
||||
|
||||
####################################################################################################
|
||||
Invoke-ScriptWithPrefix -ScriptName 'deploy-inteloneapi.ps1' -AddAdminPw
|
||||
@ -207,13 +193,13 @@ Write-Progress `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-VMName $ProtoVMName `
|
||||
-CommandId 'RunPowerShellScript' `
|
||||
-ScriptPath "$PSScriptRoot\deploy-settings.txt"
|
||||
-ScriptPath "$Root\deploy-settings.txt"
|
||||
|
||||
Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)"
|
||||
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
|
||||
Restart-AzVM -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
@ -222,10 +208,10 @@ Write-Progress `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
$SysprepResult = Invoke-AzVMRunCommandWithRetries `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-VMName $ProtoVMName `
|
||||
-CommandId 'RunPowerShellScript' `
|
||||
-ScriptPath "$PSScriptRoot\sysprep.ps1"
|
||||
-ScriptPath "$Root\sysprep.ps1"
|
||||
|
||||
Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)"
|
||||
|
||||
@ -235,7 +221,7 @@ Write-Progress `
|
||||
-Status 'Waiting for VM to shut down' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
Wait-Shutdown -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
|
||||
Wait-Shutdown -ResourceGroupName 'vcpkg-image-minting' -Name $ProtoVMName
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
@ -244,19 +230,25 @@ Write-Progress `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
Stop-AzVM `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-Name $ProtoVMName `
|
||||
-Force
|
||||
|
||||
Set-AzVM `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-Name $ProtoVMName `
|
||||
-Generalized
|
||||
|
||||
$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
|
||||
$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID -HyperVGeneration V2
|
||||
$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix
|
||||
New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting'
|
||||
New-AzGalleryImageVersion `
|
||||
-ResourceGroupName 'vcpkg-image-minting' `
|
||||
-GalleryName 'vcpkg_gallery_wus3' `
|
||||
-GalleryImageDefinitionName 'PrWinWus3-TrustedLaunch' `
|
||||
-Name $GalleryImageVersion `
|
||||
-Location $Location `
|
||||
-SourceImageId $VMCreated.ID `
|
||||
-ReplicaCount 1 `
|
||||
-StorageAccountType 'Premium_LRS' `
|
||||
-PublishingProfileExcludeFromLatest
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress `
|
||||
@ -264,9 +256,16 @@ Write-Progress `
|
||||
-Status 'Deleting unused temporary resources' `
|
||||
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
|
||||
|
||||
Remove-AzResourceGroup $ResourceGroupName -Force
|
||||
Remove-AzRoleAssignment `
|
||||
-Scope $VcpkgImageMintingAccount.ID `
|
||||
-RoleDefinitionId '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' `
|
||||
-ObjectId $VMCreated.Identity.PrincipalId
|
||||
|
||||
Remove-AzVM -Id $VMCreated.ID -Force
|
||||
Remove-AzDisk -ResourceGroupName 'vcpkg-image-minting' -Name $VMCreatedOsDisk -Force
|
||||
Remove-AzNetworkInterface -ResourceGroupName 'vcpkg-image-minting' -Name $NicName -Force
|
||||
|
||||
####################################################################################################
|
||||
Write-Progress -Activity $ProgressActivity -Completed
|
||||
Write-Host "Generated Image: $ImageName"
|
||||
Write-Host "Generated Image: $GalleryImageVersion"
|
||||
Write-Host 'Finished!'
|
||||
|
@ -1,106 +0,0 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Creates a Windows virtual machine scale set, set up for vcpkg's CI.
|
||||
|
||||
.DESCRIPTION
|
||||
create-vmss.ps1 creates an Azure Windows VM scale set, set up for vcpkg's CI
|
||||
system. See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/overview
|
||||
for more information.
|
||||
|
||||
This script assumes you have installed Azure tools into PowerShell by following the instructions
|
||||
at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
|
||||
or are running from Azure Cloud Shell.
|
||||
|
||||
.PARAMETER ImageName
|
||||
The name of the image to deploy into the scale set.
|
||||
#>
|
||||
|
||||
[CmdLetBinding()]
|
||||
Param(
|
||||
[parameter(Mandatory=$true)]
|
||||
[string]$ImageName
|
||||
)
|
||||
|
||||
$Location = 'westus3'
|
||||
$Prefix = 'PrWin-'
|
||||
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
|
||||
$VMSize = 'Standard_D32ads_v5'
|
||||
$LiveVMPrefix = 'BUILD'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
|
||||
|
||||
$ResourceGroupName = Find-ResourceGroupName $Prefix
|
||||
$AdminPW = New-Password
|
||||
$Image = Get-AzImage -ResourceGroupName 'vcpkg-image-minting' -ImageName $ImageName
|
||||
|
||||
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
|
||||
|
||||
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
|
||||
$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig'
|
||||
$VmssIpConfig = New-AzVmssIpConfig -SubnetId $VirtualNetwork.Subnets[0].Id -Primary -Name $VmssIpConfigName
|
||||
$VmssName = $ResourceGroupName + 'Vmss'
|
||||
$Vmss = New-AzVmssConfig `
|
||||
-Location $Location `
|
||||
-SkuCapacity 0 `
|
||||
-SkuName $VMSize `
|
||||
-SkuTier 'Standard' `
|
||||
-Overprovision $false `
|
||||
-UpgradePolicyMode Automatic `
|
||||
-EvictionPolicy Delete `
|
||||
-Priority Spot `
|
||||
-MaxPrice -1 `
|
||||
-SecurityType Standard
|
||||
|
||||
$NicName = $ResourceGroupName + 'NIC'
|
||||
New-AzNetworkInterface `
|
||||
-Name $NicName `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-Location $Location `
|
||||
-Subnet $VirtualNetwork.Subnets[0]
|
||||
|
||||
$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
|
||||
-VirtualMachineScaleSet $Vmss `
|
||||
-Primary $true `
|
||||
-IpConfiguration $VmssIpConfig `
|
||||
-NetworkSecurityGroupId $VirtualNetwork.Subnets[0].NetworkSecurityGroup.Id `
|
||||
-Name $NicName
|
||||
|
||||
$Vmss = Set-AzVmssOsProfile `
|
||||
-VirtualMachineScaleSet $Vmss `
|
||||
-ComputerNamePrefix $LiveVMPrefix `
|
||||
-AdminUsername 'AdminUser' `
|
||||
-AdminPassword $AdminPW `
|
||||
-WindowsConfigurationProvisionVMAgent $true `
|
||||
-WindowsConfigurationEnableAutomaticUpdate $false
|
||||
|
||||
$Vmss = Set-AzVmssStorageProfile `
|
||||
-VirtualMachineScaleSet $Vmss `
|
||||
-OsDiskCreateOption 'FromImage' `
|
||||
-OsDiskCaching ReadOnly `
|
||||
-DiffDiskSetting Local `
|
||||
-ImageReferenceId $Image.Id
|
||||
|
||||
$Vmss = Set-AzVmssBootDiagnostic `
|
||||
-VirtualMachineScaleSet $Vmss `
|
||||
-Enabled $false
|
||||
|
||||
$VmssCreated = New-AzVmss `
|
||||
-ResourceGroupName $ResourceGroupName `
|
||||
-Name $VmssName `
|
||||
-VirtualMachineScaleSet $Vmss
|
||||
|
||||
# Grant 'Virtual Machine Contributor' (RoleDefinitionId 9980e02c-c2be-4d73-94e8-173b1dc7cf3c) to
|
||||
# 'dev-azure-com-vcpkg-scale-set-management' (ObjectId e4fe677f-f905-4f3c-b5c3-d8a2d6812a5b)
|
||||
New-AzRoleAssignment `
|
||||
-Scope $VmssCreated.Id `
|
||||
-RoleDefinitionId '9980e02c-c2be-4d73-94e8-173b1dc7cf3c' `
|
||||
-ObjectId 'e4fe677f-f905-4f3c-b5c3-d8a2d6812a5b'
|
||||
|
||||
Write-Host "Location: $Location"
|
||||
Write-Host "Resource group name: $ResourceGroupName"
|
||||
Write-Host 'Finished!'
|
4
scripts/azure-pipelines/windows/deploy-azcopy.ps1
Normal file
4
scripts/azure-pipelines/windows/deploy-azcopy.ps1
Normal file
@ -0,0 +1,4 @@
|
||||
$azcopyZipPath = "$PSScriptRoot\azcopyv10.zip"
|
||||
& curl.exe -L -o $azcopyZipPath 'https://azcopyvnext.azureedge.net/releases/release-10.23.0-20240129/azcopy_windows_amd64_10.23.0.zip'
|
||||
Expand-Archive -LiteralPath $azcopyZipPath -DestinationPath $env:PROGRAMFILES
|
||||
Remove-Item -LiteralPath $azcopyZipPath -Force
|
@ -5,7 +5,9 @@
|
||||
|
||||
# REPLACE WITH UTILITY-PREFIX.ps1
|
||||
|
||||
# REPLACE WITH $CudnnUrl
|
||||
# If you are running this script outside of our Azure VMs, you will need to download cudnn from NVIDIA and place
|
||||
# it next to this script.
|
||||
$CudnnUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip'
|
||||
|
||||
$CudnnLocalZipPath = "$PSScriptRoot\cudnn-windows-x86_64-8.8.1.3_cuda12-archive.zip"
|
||||
|
||||
@ -82,16 +84,16 @@ catch {
|
||||
}
|
||||
|
||||
try {
|
||||
if ([string]::IsNullOrWhiteSpace($CudnnUrl)) {
|
||||
if (-Not (Test-Path $CudnnLocalZipPath)) {
|
||||
throw "CUDNN zip ($CudnnLocalZipPath) was missing, please download from NVidia and place next to this script."
|
||||
}
|
||||
|
||||
if (Test-Path $CudnnLocalZipPath) {
|
||||
$cudnnZipPath = $CudnnLocalZipPath
|
||||
} else {
|
||||
Write-Host 'Downloading CUDNN...'
|
||||
Write-Host 'Attempting to download cudnn. If this fails, you need to agree to NVidia''s EULA, download cudnn, and place it next to this script.'
|
||||
$cudnnZipPath = Get-TempFilePath -Extension 'zip'
|
||||
curl.exe -L -o $cudnnZipPath -s -S $CudnnUrl
|
||||
$env:AZCOPY_AUTO_LOGIN_TYPE = 'MSI'
|
||||
& "$env:PROGRAMFILES\azcopy_windows_amd64_10.23.0\azcopy.exe" copy $CudnnUrl $cudnnZipPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw 'Failed to download cudnn!'
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Installing CUDNN to $destination..."
|
||||
|
@ -4,6 +4,7 @@
|
||||
. "$PSScriptRoot\utility-prefix.ps1"
|
||||
|
||||
. "$PSScriptRoot\deploy-tlssettings.ps1" -RebootIfRequired 0
|
||||
. "$PSScriptRoot\deploy-azcopy.ps1"
|
||||
. "$PSScriptRoot\deploy-windows-sdks.ps1"
|
||||
. "$PSScriptRoot\deploy-visual-studio.ps1"
|
||||
. "$PSScriptRoot\deploy-mpi.ps1"
|
||||
|
@ -990,9 +990,6 @@ rest-rpc:arm64-windows=skip
|
||||
rest-rpc:x64-linux=skip
|
||||
rest-rpc:x64-osx=skip
|
||||
rest-rpc:arm64-osx=skip
|
||||
# Missing system libraries
|
||||
robotraconteur:x64-linux=fail
|
||||
robotraconteur-companion:x64-linux=fail
|
||||
rpclib:arm64-windows=fail
|
||||
rpclib:arm64-uwp=fail
|
||||
rpclib:x64-uwp=fail
|
||||
|
Loading…
x
Reference in New Issue
Block a user