3party/vcpkg
0
0
Fork 0
mirror of https://github.com/microsoft/vcpkg.git synced 2024-12-27 18:31:15 +08:00
Code Issues Projects Releases Wiki Activity

Update pwsh to 7.1.14, separate image minting from vmss minting, and update VMs. (#20064)

Browse Source 
* Update Powershell-Core to 7.1.4.

* Add deployment of .NET 4.7.2 targeting pack, extracted from https://github.com/microsoft/vcpkg/pull/19320

* Separate image minting from vmss minting.

* Update pools.

* Fix image minting typo on Windows.
This commit is contained in:
Billy O'Neal 2021-09-11 13:43:19 -07:00 committed by GitHub
parent 1909fbc4eb
commit c26477297c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 670 additions and 808 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/azure-pipelines/azure-pipelines.yml
View File

@ -2,8 +2,8 @@
# SPDX-License-Identifier: MIT
#
variables:
windows-pool: 'PrWin-2021-08-12'
linux-pool: 'PrLin-2021-08-11'
windows-pool: 'PrWin-2021-09-08'
linux-pool: 'PrLin-2021-09-08'
osx-pool: 'PrOsx-2021-07-27'
stages:

182
scripts/azure-pipelines/create-vmss-helpers.psm1
View File

@ -55,6 +55,60 @@ function Find-ResourceGroupName {
return $result
}
<#
.SYNOPSIS
Returns whether there's a name collision for an image in the resource group.
.DESCRIPTION
Find-ImageNameCollision takes a list of images, and checks if $Test
collides names with any of the image names.
.PARAMETER Test
The name to test.
.PARAMETER Images
The list of images.
#>
function Find-ImageNameCollision {
[CmdletBinding()]
Param([string]$Test, $Images)
foreach ($resource in $Images) {
if ($resource.Name -eq $Test) {
return $true
}
}
return $false
}
<#
.SYNOPSIS
Attempts to find a name that does not collide with any images in the resource group.
.DESCRIPTION
Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the
first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with
any of the resources in the resource group.
.PARAMETER Prefix
The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?"
#>
function Find-ImageName {
[CmdLetBinding()]
Param([string]$ResourceGroupName, [string]$Prefix)
$images = Get-AzImage -ResourceGroupName $ResourceGroupName
$result = $Prefix
$suffix = 0
while (Find-ImageNameCollision -Test $result -Images $images) {
$suffix++
$result = "$Prefix-$suffix"
}
return $result
}
<#
.SYNOPSIS
Generates a random password.
@ -160,7 +214,135 @@ function Sanitize-Name {
return $result
}
<#
.SYNOPSIS
Creates a new Azure virtual network with locked down firewall rules.
.PARAMETER ResourceGroupName
The name of the resource group in which the virtual network should be created.
.PARAMETER Location
The location (region) where the network is to be created.
#>
function Create-LockedDownNetwork {
[CmdletBinding()]
Param(
[parameter(Mandatory=$true)]
[string]$ResourceGroupName,
[parameter(Mandatory=$true)]
[string]$Location
)
$allFirewallRules = @()
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowHTTP `
-Description 'Allow HTTP(S)' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1008 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(80, 443)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowSFTP `
-Description 'Allow (S)FTP' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1009 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(21, 22)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowDNS `
-Description 'Allow DNS' `
-Access Allow `
-Protocol * `
-Direction Outbound `
-Priority 1010 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 53
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowGit `
-Description 'Allow git' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1011 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 9418
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name DenyElse `
-Description 'Deny everything else' `
-Access Deny `
-Protocol * `
-Direction Outbound `
-Priority 1013 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange *
$NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
$NetworkSecurityGroup = New-AzNetworkSecurityGroup `
-Name $NetworkSecurityGroupName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-SecurityRules $allFirewallRules
$SubnetName = $ResourceGroupName + 'Subnet'
$Subnet = New-AzVirtualNetworkSubnetConfig `
-Name $SubnetName `
-AddressPrefix "10.0.0.0/16" `
-NetworkSecurityGroup $NetworkSecurityGroup `
-ServiceEndpoint "Microsoft.Storage"
$VirtualNetworkName = $ResourceGroupName + 'Network'
$VirtualNetwork = New-AzVirtualNetwork `
-Name $VirtualNetworkName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-AddressPrefix "10.0.0.0/16" `
-Subnet $Subnet
return $VirtualNetwork
}
function Invoke-AzVMRunCommandWithRetries {
try {
return Invoke-AzVMRunCommand @args
} catch {
for ($idx = 0; $idx -lt 5; $idx++) {
Write-Host "Running command failed. $_ Retrying after 10 seconds..."
Start-Sleep -Seconds 10
try {
return Invoke-AzVMRunCommand @args
} catch {
# ignore
}
}
Write-Host "Running command failed too many times. Giving up!"
throw $_
}
}
Export-ModuleMember -Function Find-ResourceGroupName
Export-ModuleMember -Function Find-ImageName
Export-ModuleMember -Function New-Password
Export-ModuleMember -Function Wait-Shutdown
Export-ModuleMember -Function Sanitize-Name
Export-ModuleMember -Function Create-LockedDownNetwork
Export-ModuleMember -Function Invoke-AzVMRunCommandWithRetries

161
scripts/azure-pipelines/linux/create-image.ps1 Normal file
View File

@ -0,0 +1,161 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: MIT
#
<#
.SYNOPSIS
Creates a Linux virtual machine image, set up for vcpkg's CI.
.DESCRIPTION
create-image.ps1 creates an Azure Linux VM image, set up for vcpkg's CI system.
This script assumes you have installed Azure tools into PowerShell by following the instructions
at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
or are running from Azure Cloud Shell.
This script assumes you have installed the OpenSSH Client optional Windows component.
#>
$Location = 'westus2'
$Prefix = 'Lin-'
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_D32as_v4'
$ProtoVMName = 'PROTOTYPE'
$ErrorActionPreference = 'Stop'
$ProgressActivity = 'Creating Linux Image'
$TotalProgress = 9
$CurrentProgress = 1
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating SSH key' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$sshDir = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName()
mkdir $sshDir
try {
ssh-keygen.exe -q -b 2048 -t rsa -f "$sshDir/key" -P [string]::Empty
$sshPublicKey = Get-Content "$sshDir/key.pub"
} finally {
Remove-Item $sshDir -Recurse -Force
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating resource group' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ResourceGroupName = Find-ResourceGroupName $Prefix
$AdminPW = New-Password
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating virtual network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating prototype VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$NicName = $ResourceGroupName + 'NIC'
$Nic = New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
$VM = Set-AzVMOperatingSystem `
-VM $VM `
-Linux `
-ComputerName $ProtoVMName `
-Credential $Credential `
-DisablePasswordAuthentication
$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
$VM = Set-AzVMSourceImage `
-VM $VM `
-PublisherName 'Canonical' `
-Offer '0001-com-ubuntu-server-focal' `
-Skus '20_04-lts-gen2' `
-Version latest
$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
$VM = Add-AzVMSshPublicKey `
-VM $VM `
-KeyData $sshPublicKey `
-Path "/home/AdminUser/.ssh/authorized_keys"
New-AzVm `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-VM $VM
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script provision-image.sh in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunShellScript' `
-ScriptPath "$PSScriptRoot\provision-image.sh"
Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Restarting VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Converting VM to Image' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Stop-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Force
Set-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Generalized
$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID -HyperVGeneration 'V2'
$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix
New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting'
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Deleting unused temporary resources' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Remove-AzResourceGroup $ResourceGroupName -Force
####################################################################################################
Write-Progress -Activity $ProgressActivity -Completed
Write-Host "Generated Image: $ImageName"
Write-Host 'Finished!'

348
scripts/azure-pipelines/linux/create-vmss.ps1
View File

@ -16,29 +16,27 @@ at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6
or are running from Azure Cloud Shell.
This script assumes you have installed the OpenSSH Client optional Windows component.
.PARAMETER ImageName
The name of the image to deploy into the scale set.
#>
[CmdLetBinding()]
Param(
[parameter(Mandatory=$true)]
[string]$ImageName
)
$Location = 'westus2'
$Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_D32ds_v4'
$ProtoVMName = 'PROTOTYPE'
$Prefix = 'PrLin-'
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_D32a_v4'
$LiveVMPrefix = 'BUILD'
$MakeInstalledDisk = $false
$InstalledDiskSizeInGB = 1024
$ErrorActionPreference = 'Stop'
$ProgressActivity = 'Creating Scale Set'
$TotalProgress = 11
$CurrentProgress = 1
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating SSH key' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$sshDir = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName()
mkdir $sshDir
try {
@ -47,279 +45,15 @@ try {
} finally {
Remove-Item $sshDir -Recurse -Force
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating resource group' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ResourceGroupName = Find-ResourceGroupName $Prefix
$AdminPW = New-Password
$Image = Get-AzImage -ResourceGroupName 'vcpkg-image-minting' -ImageName $ImageName
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating virtual network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$allFirewallRules = @()
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowHTTP `
-Description 'Allow HTTP(S)' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1008 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(80, 443)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowSFTP `
-Description 'Allow (S)FTP' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1009 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(21, 22)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowDNS `
-Description 'Allow DNS' `
-Access Allow `
-Protocol * `
-Direction Outbound `
-Priority 1010 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 53
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowGit `
-Description 'Allow git' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1011 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 9418
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name DenyElse `
-Description 'Deny everything else' `
-Access Deny `
-Protocol * `
-Direction Outbound `
-Priority 1013 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange *
$NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
$NetworkSecurityGroup = New-AzNetworkSecurityGroup `
-Name $NetworkSecurityGroupName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-SecurityRules $allFirewallRules
$SubnetName = $ResourceGroupName + 'Subnet'
$Subnet = New-AzVirtualNetworkSubnetConfig `
-Name $SubnetName `
-AddressPrefix "10.0.0.0/16" `
-NetworkSecurityGroup $NetworkSecurityGroup `
-ServiceEndpoint "Microsoft.Storage"
$VirtualNetworkName = $ResourceGroupName + 'Network'
$VirtualNetwork = New-AzVirtualNetwork `
-Name $VirtualNetworkName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-AddressPrefix "10.0.0.0/16" `
-Subnet $Subnet
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating archives storage account' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$StorageAccountName = Sanitize-Name $ResourceGroupName
New-AzStorageAccount `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Name $StorageAccountName `
-SkuName 'Standard_LRS' `
-Kind StorageV2 `
-MinimumTlsVersion TLS1_2
$StorageAccountKeys = Get-AzStorageAccountKey `
-ResourceGroupName $ResourceGroupName `
-Name $StorageAccountName
$StorageAccountKey = $StorageAccountKeys[0].Value
$StorageContext = New-AzStorageContext `
-StorageAccountName $StorageAccountName `
-StorageAccountKey $StorageAccountKey
New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
$StartTime = [DateTime]::Now
$ExpiryTime = $StartTime.AddMonths(6)
$SasToken = New-AzStorageAccountSASToken `
-Service Blob `
-Permission "racwdlup" `
-Context $StorageContext `
-StartTime $StartTime `
-ExpiryTime $ExpiryTime `
-ResourceType Service,Container,Object `
-Protocol HttpsOnly
$SasToken = $SasToken.Substring(1) # strip leading ?
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating storage account' `
-CurrentOperation 'Locking down network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
# Note that we put the storage account into the firewall after creating the above SAS token or we
# would be denied since the person running this script isn't one of the VMs we're creating here.
Set-AzStorageAccount `
-ResourceGroupName $ResourceGroupName `
-AccountName $StorageAccountName `
-NetworkRuleSet ( `
@{bypass="AzureServices"; `
virtualNetworkRules=( `
@{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
defaultAction="Deny"})
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating prototype VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$NicName = $ResourceGroupName + 'NIC'
$Nic = New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
$VM = Set-AzVMOperatingSystem `
-VM $VM `
-Linux `
-ComputerName $ProtoVMName `
-Credential $Credential `
-DisablePasswordAuthentication
$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
$VM = Set-AzVMSourceImage `
-VM $VM `
-PublisherName 'Canonical' `
-Offer '0001-com-ubuntu-server-focal' `
-Skus '20_04-lts' `
-Version latest
$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
$VM = Add-AzVMSshPublicKey `
-VM $VM `
-KeyData $sshPublicKey `
-Path "/home/AdminUser/.ssh/authorized_keys"
New-AzVm `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-VM $VM
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script provision-image.sh in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$tempScript = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".sh"
try {
$script = Get-Content "$PSScriptRoot\provision-image.sh" -Encoding utf8NoBOM
$script += "echo `"PROVISIONED_AZURE_STORAGE_NAME=\`"$StorageAccountName\`"`" | sudo tee -a /etc/environment"
$script += "echo `"PROVISIONED_AZURE_STORAGE_SAS_TOKEN=\`"$SasToken\`"`" | sudo tee -a /etc/environment"
Set-Content -Path $tempScript -Value $script -Encoding utf8NoBOM
$ProvisionImageResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunShellScript' `
-ScriptPath $tempScript
Write-Host "provision-image.sh output: $($ProvisionImageResult.value.Message)"
} finally {
Remove-Item $tempScript -Recurse -Force
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Restarting VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Converting VM to Image' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Stop-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Force
Set-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Generalized
$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
$PrototypeOSDiskName = $VM.StorageProfile.OsDisk.Name
$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID
$ImageName = "$Prefix-BaseImage"
$Image = New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName $ResourceGroupName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Deleting unused VM and disk' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Remove-AzVM -Id $VM.ID -Force
Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $PrototypeOSDiskName -Force
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating scale set' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig'
$VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id -Primary -Name $VmssIpConfigName
$VmssIpConfig = New-AzVmssIpConfig -SubnetId $VirtualNetwork.Subnets[0].Id -Primary -Name $VmssIpConfigName
$VmssName = $ResourceGroupName + 'Vmss'
$Vmss = New-AzVmssConfig `
-Location $Location `
@ -332,34 +66,30 @@ $Vmss = New-AzVmssConfig `
-Priority Spot `
-MaxPrice -1
$NicName = $ResourceGroupName + 'NIC'
New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
-VirtualMachineScaleSet $Vmss `
-Primary $true `
-IpConfiguration $VmssIpConfig `
-NetworkSecurityGroupId $NetworkSecurityGroup.Id `
-NetworkSecurityGroupId $VirtualNetwork.Subnets[0].NetworkSecurityGroup.Id `
-Name $NicName
$VmssPublicKey = New-Object -TypeName 'Microsoft.Azure.Management.Compute.Models.SshPublicKey' `
-ArgumentList @('/home/AdminUser/.ssh/authorized_keys', $sshPublicKey)
if ($MakeInstalledDisk) {
$Vmss = Set-AzVmssOsProfile `
-VirtualMachineScaleSet $Vmss `
-ComputerNamePrefix $LiveVMPrefix `
-AdminUsername AdminUser `
-AdminPassword $AdminPW `
-LinuxConfigurationDisablePasswordAuthentication $true `
-PublicKey @($VmssPublicKey) `
-CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n")))
} else {
$Vmss = Set-AzVmssOsProfile `
-VirtualMachineScaleSet $Vmss `
-ComputerNamePrefix $LiveVMPrefix `
-AdminUsername AdminUser `
-AdminPassword $AdminPW `
-LinuxConfigurationDisablePasswordAuthentication $true `
-PublicKey @($VmssPublicKey)
}
$Vmss = Set-AzVmssOsProfile `
-VirtualMachineScaleSet $Vmss `
-ComputerNamePrefix $LiveVMPrefix `
-AdminUsername AdminUser `
-AdminPassword $AdminPW `
-LinuxConfigurationDisablePasswordAuthentication $true `
-PublicKey @($VmssPublicKey)
$Vmss = Set-AzVmssStorageProfile `
-VirtualMachineScaleSet $Vmss `
@ -368,25 +98,11 @@ $Vmss = Set-AzVmssStorageProfile `
-DiffDiskSetting Local `
-ImageReferenceId $Image.Id
if ($MakeInstalledDisk) {
$Vmss = Add-AzVmssDataDisk `
-VirtualMachineScaleSet $Vmss `
-Lun 0 `
-Caching 'ReadWrite' `
-CreateOption Empty `
-DiskSizeGB $InstalledDiskSizeInGB `
-StorageAccountType 'StandardSSD_LRS'
}
New-AzVmss `
-ResourceGroupName $ResourceGroupName `
-Name $VmssName `
-VirtualMachineScaleSet $Vmss
####################################################################################################
Write-Progress -Activity $ProgressActivity -Completed
Write-Host "Location: $Location"
Write-Host "Resource group name: $ResourceGroupName"
Write-Host "User name: AdminUser"
Write-Host "Using generated password: $AdminPW"
Write-Host 'Finished!'

14
scripts/azure-pipelines/linux/provision-image.sh
View File

@ -81,17 +81,3 @@ sudo dpkg -i packages-microsoft-prod.deb
sudo apt update
sudo add-apt-repository universe
sudo apt install -y powershell
# Write script to provision disks used by cloud-init
echo "if [ ! -d \"/mnt/vcpkg-ci\" ]; then" > /etc/provision-disks.sh
echo "sudo parted /dev/sdb mklabel gpt" >> /etc/provision-disks.sh
echo "sudo parted /dev/sdb mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh
echo "sudo mkfs -t ext4 /dev/sdb1" >> /etc/provision-disks.sh
echo "sudo mkdir /mnt/vcpkg-ci -m=777" >> /etc/provision-disks.sh
echo "echo \"/dev/sdb1 /mnt/vcpkg-ci ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh
echo "sudo mount -a" >> /etc/provision-disks.sh
echo "sudo chmod 777 /mnt/vcpkg-ci" >> /etc/provision-disks.sh
echo "fi" >> /etc/provision-disks.sh
sudo chmod 700 /etc/provision-disks.sh
# provision-image.ps1 will append installation of the SAS token here

271
scripts/azure-pipelines/windows/create-image.ps1 Normal file
View File

@ -0,0 +1,271 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: MIT
#
<#
.SYNOPSIS
Creates a Windows virtual machine image, set up for vcpkg's CI.
.DESCRIPTION
create-image.ps1 creates an Azure Windows VM image, set up for vcpkg's CI system.
This script assumes you have installed Azure tools into PowerShell by following the instructions
at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
or are running from Azure Cloud Shell.
#>
$Location = 'westus2'
$Prefix = 'Win-'
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_D32as_v4'
$ProtoVMName = 'PROTOTYPE'
$WindowsServerSku = '2019-datacenter-gensecond'
$ErrorActionPreference = 'Stop'
$CudnnBaseUrl = 'https://vcpkgimageminting.blob.core.windows.net/assets/cudnn-11.2-windows-x64-v8.1.1.33.zip'
$ProgressActivity = 'Creating Windows Image'
$TotalProgress = 18
$CurrentProgress = 1
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating resource group' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ResourceGroupName = Find-ResourceGroupName $Prefix
$AdminPW = New-Password
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating virtual network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating prototype VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$NicName = $ResourceGroupName + 'NIC'
$Nic = New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
$VM = Set-AzVMOperatingSystem `
-VM $VM `
-Windows `
-ComputerName $ProtoVMName `
-Credential $Credential `
-ProvisionVMAgent
$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
$VM = Set-AzVMSourceImage `
-VM $VM `
-PublisherName 'MicrosoftWindowsServer' `
-Offer 'WindowsServer' `
-Skus $WindowsServerSku `
-Version latest
$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
New-AzVm `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-VM $VM
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-tlssettings.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-tlssettings.ps1"
Write-Host "deploy-tlssettings.ps1 output: $($ProvisionImageResult.value.Message)"
Write-Host 'Waiting 1 minute for VM to reboot...'
Start-Sleep -Seconds 60
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-psexec.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$DeployPsExecResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-psexec.ps1"
Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)"
####################################################################################################
function Invoke-ScriptWithPrefix {
param(
[string]$ScriptName,
[switch]$AddAdminPw,
[string]$CudnnUrl
)
Write-Progress `
-Activity $ProgressActivity `
-Status "Running provisioning script $ScriptName in VM" `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw
$UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw
$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
try {
$script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw
if ($AddAdminPw) {
$script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix)
}
if (-Not ([string]::IsNullOrWhiteSpace($CudnnUrl))) {
$script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnUrl'")
}
$script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent);
Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
$parameter = $null
if ($AddAdminPw) {
$parameter = @{AdminUserPassword = $AdminPW;}
}
$InvokeResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath $tempScriptFilename `
-Parameter $parameter
Write-Host "$ScriptName output: $($InvokeResult.value.Message)"
} finally {
Remove-Item $tempScriptFilename -Force
}
}
Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-sdks.ps1' -AddAdminPw
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw
####################################################################################################
$StorageAccountKeys = Get-AzStorageAccountKey `
-ResourceGroupName 'vcpkg-image-minting' `
-Name 'vcpkgimageminting'
$StorageContext = New-AzStorageContext `
-StorageAccountName 'vcpkgimageminting' `
-StorageAccountKey $StorageAccountKeys[0].Value
$StartTime = [DateTime]::Now
$ExpiryTime = $StartTime.AddDays(1)
$SetupSasToken = New-AzStorageAccountSASToken `
-Service Blob `
-Permission "r" `
-Context $StorageContext `
-StartTime $StartTime `
-ExpiryTime $ExpiryTime `
-ResourceType Object `
-Protocol HttpsOnly
Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -CudnnUrl ($CudnnBaseUrl + $SetupSasToken)
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-inteloneapi.ps1' -AddAdminPw
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-pwsh.ps1' -AddAdminPw
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-settings.txt (as a .ps1) in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ProvisionImageResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-settings.txt"
Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)"
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script sysprep.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$SysprepResult = Invoke-AzVMRunCommandWithRetries `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\sysprep.ps1"
Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)"
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Waiting for VM to shut down' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Wait-Shutdown -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Converting VM to Image' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Stop-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Force
Set-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Generalized
$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID -HyperVGeneration 'V2'
$ImageName = Find-ImageName -ResourceGroupName 'vcpkg-image-minting' -Prefix $Prefix
New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName 'vcpkg-image-minting'
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Deleting unused temporary resources' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Remove-AzResourceGroup $ResourceGroupName -Force
####################################################################################################
Write-Progress -Activity $ProgressActivity -Completed
Write-Host "Generated Image: $ImageName"
Write-Host 'Finished!'

487
scripts/azure-pipelines/windows/create-vmss.ps1
View File

@ -15,492 +15,34 @@ This script assumes you have installed Azure tools into PowerShell by following
at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
or are running from Azure Cloud Shell.
.PARAMETER CudnnPath
The path to a CUDNN zip file downloaded from NVidia official sources
(e.g. https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.1.1.33/11.2_20210301/cudnn-11.2-windows-x64-v8.1.1.33.zip
downloaded in a browser with an NVidia account logged in.)
.PARAMETER ImageName
The name of the image to deploy into the scale set.
#>
[CmdLetBinding()]
Param(
[parameter(Mandatory=$true)]
[string]$CudnnPath
[string]$ImageName
)
$Location = 'westus2'
$Prefix = 'PrWin-'
$Prefix += (Get-Date -Format 'yyyy-MM-dd')
$VMSize = 'Standard_D32ds_v4'
$ProtoVMName = 'PROTOTYPE'
$VMSize = 'Standard_D32a_v4'
$LiveVMPrefix = 'BUILD'
$WindowsServerSku = '2019-Datacenter'
$MakeInstalledDisk = $false
$InstalledDiskSizeInGB = 1024
$ErrorActionPreference = 'Stop'
$ProgressActivity = 'Creating Scale Set'
$TotalProgress = 20
if ($MakeInstalledDisk) {
$TotalProgress++
}
$CurrentProgress = 1
Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
if (-Not $CudnnPath.EndsWith('.zip')) {
Write-Error 'Expected CudnnPath to be a zip file.'
return
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating resource group' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ResourceGroupName = Find-ResourceGroupName $Prefix
$AdminPW = New-Password
$Image = Get-AzImage -ResourceGroupName 'vcpkg-image-minting' -ImageName $ImageName
New-AzResourceGroup -Name $ResourceGroupName -Location $Location
$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force
$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure)
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating virtual network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$allFirewallRules = @()
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowHTTP `
-Description 'Allow HTTP(S)' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1008 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(80, 443)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowSFTP `
-Description 'Allow (S)FTP' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1009 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange @(21, 22)
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowDNS `
-Description 'Allow DNS' `
-Access Allow `
-Protocol * `
-Direction Outbound `
-Priority 1010 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 53
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name AllowGit `
-Description 'Allow git' `
-Access Allow `
-Protocol Tcp `
-Direction Outbound `
-Priority 1011 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange 9418
$allFirewallRules += New-AzNetworkSecurityRuleConfig `
-Name DenyElse `
-Description 'Deny everything else' `
-Access Deny `
-Protocol * `
-Direction Outbound `
-Priority 1013 `
-SourceAddressPrefix * `
-SourcePortRange * `
-DestinationAddressPrefix * `
-DestinationPortRange *
$NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
$NetworkSecurityGroup = New-AzNetworkSecurityGroup `
-Name $NetworkSecurityGroupName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-SecurityRules $allFirewallRules
$SubnetName = $ResourceGroupName + 'Subnet'
$Subnet = New-AzVirtualNetworkSubnetConfig `
-Name $SubnetName `
-AddressPrefix "10.0.0.0/16" `
-NetworkSecurityGroup $NetworkSecurityGroup `
-ServiceEndpoint "Microsoft.Storage"
$VirtualNetworkName = $ResourceGroupName + 'Network'
$VirtualNetwork = New-AzVirtualNetwork `
-Name $VirtualNetworkName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-AddressPrefix "10.0.0.0/16" `
-Subnet $Subnet
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating storage account' `
-CurrentOperation 'Initial setup' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$StorageAccountName = Sanitize-Name $ResourceGroupName
New-AzStorageAccount `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Name $StorageAccountName `
-SkuName 'Standard_LRS' `
-Kind StorageV2 `
-MinimumTlsVersion TLS1_2
$StorageAccountKeys = Get-AzStorageAccountKey `
-ResourceGroupName $ResourceGroupName `
-Name $StorageAccountName
$StorageAccountKey = $StorageAccountKeys[0].Value
$StorageContext = New-AzStorageContext `
-StorageAccountName $StorageAccountName `
-StorageAccountKey $StorageAccountKey
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating storage account' `
-CurrentOperation 'Uploading cudnn.zip' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
New-AzStorageContainer -Name setup -Context $storageContext -Permission blob
Set-AzStorageBlobContent -File $CudnnPath `
-Container 'setup' `
-Blob 'cudnn.zip' `
-Context $StorageContext
$CudnnBlobUrl = "https://$StorageAccountName.blob.core.windows.net/setup/cudnn.zip"
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating storage account' `
-CurrentOperation 'Creating archives container' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off
$StartTime = [DateTime]::Now
$ExpiryTime = $StartTime.AddMonths(6)
$SasToken = New-AzStorageAccountSASToken `
-Service Blob `
-Permission "racwdlup" `
-Context $StorageContext `
-StartTime $StartTime `
-ExpiryTime $ExpiryTime `
-ResourceType Service,Container,Object `
-Protocol HttpsOnly
$SasToken = $SasToken.Substring(1) # strip leading ?
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating storage account' `
-CurrentOperation 'Locking down network' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++
# Note that we put the storage account into the firewall after creating the above SAS token or we
# would be denied since the person running this script isn't one of the VMs we're creating here.
Set-AzStorageAccount `
-ResourceGroupName $ResourceGroupName `
-AccountName $StorageAccountName `
-NetworkRuleSet ( `
@{bypass="AzureServices"; `
virtualNetworkRules=( `
@{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); `
defaultAction="Deny"})
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating prototype VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$NicName = $ResourceGroupName + 'NIC'
$Nic = New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
$VM = Set-AzVMOperatingSystem `
-VM $VM `
-Windows `
-ComputerName $ProtoVMName `
-Credential $Credential `
-ProvisionVMAgent
$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id
$VM = Set-AzVMSourceImage `
-VM $VM `
-PublisherName 'MicrosoftWindowsServer' `
-Offer 'WindowsServer' `
-Skus $WindowsServerSku `
-Version latest
$InstallDiskName = $ProtoVMName + "InstallDisk"
if ($MakeInstalledDisk) {
$VM = Add-AzVMDataDisk `
-Vm $VM `
-Name $InstallDiskName `
-Lun 0 `
-Caching ReadWrite `
-CreateOption Empty `
-DiskSizeInGB $InstalledDiskSizeInGB `
-StorageAccountType 'StandardSSD_LRS'
}
$VM = Set-AzVMBootDiagnostic -VM $VM -Disable
New-AzVm `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-VM $VM
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-tlssettings.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ProvisionImageResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-tlssettings.ps1"
Write-Host "deploy-tlssettings.ps1 output: $($ProvisionImageResult.value.Message)"
Write-Host 'Waiting 1 minute for VM to reboot...'
Start-Sleep -Seconds 60
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-psexec.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$DeployPsExecResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-psexec.ps1"
Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)"
####################################################################################################
function Invoke-ScriptWithPrefix {
param(
[string]$ScriptName,
[switch]$AddAdminPw,
[switch]$AddCudnnUrl
)
Write-Progress `
-Activity $ProgressActivity `
-Status "Running provisioning script $ScriptName in VM" `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw
$UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw
$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
try {
$script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw
if ($AddAdminPw) {
$script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix)
}
if ($AddCudnnUrl) {
$script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnBlobUrl'")
}
$script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent);
Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
$parameter = $null
if ($AddAdminPw) {
$parameter = @{AdminUserPassword = $AdminPW;}
}
$InvokeResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath $tempScriptFilename `
-Parameter $parameter
Write-Host "$ScriptName output: $($InvokeResult.value.Message)"
} finally {
Remove-Item $tempScriptFilename -Force
}
}
Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-sdks.ps1' -AddAdminPw
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -AddCudnnUrl
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-inteloneapi.ps1' -AddAdminPw
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Invoke-ScriptWithPrefix -ScriptName 'deploy-pwsh.ps1' -AddAdminPw
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script deploy-settings.txt (as a .ps1) in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$ProvisionImageResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\deploy-settings.txt"
Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)"
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Deploying SAS token into VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt"
try {
$script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " `
+ "-Name PROVISIONED_AZURE_STORAGE_NAME " `
+ "-Value '$StorageAccountName'`r`n" `
+ "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " `
+ "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " `
+ "-Value '$SasToken'`r`n"
Write-Host "Script content is:"
Write-Host $script
Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM
$InvokeResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath $tempScriptFilename
Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)"
} finally {
Remove-Item $tempScriptFilename -Force
}
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
if ($MakeInstalledDisk) {
Invoke-ScriptWithPrefix -ScriptName 'deploy-install-disk.ps1'
Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Running provisioning script sysprep.ps1 in VM' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$SysprepResult = Invoke-AzVMRunCommand `
-ResourceGroupName $ResourceGroupName `
-VMName $ProtoVMName `
-CommandId 'RunPowerShellScript' `
-ScriptPath "$PSScriptRoot\sysprep.ps1"
Write-Host "sysprep.ps1 output: $($SysprepResult.value.Message)"
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Waiting for VM to shut down' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Wait-Shutdown -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Converting VM to Image' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Stop-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Force
Set-AzVM `
-ResourceGroupName $ResourceGroupName `
-Name $ProtoVMName `
-Generalized
$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName
$PrototypeOSDiskName = $VM.StorageProfile.OsDisk.Name
$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID
$ImageName = "$Prefix-BaseImage"
$Image = New-AzImage -Image $ImageConfig -ImageName $ImageName -ResourceGroupName $ResourceGroupName
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Deleting unused VM and disk' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
Remove-AzVM -Id $VM.ID -Force
Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $PrototypeOSDiskName -Force
if ($MakeInstalledDisk) {
Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $InstallDiskName -Force
}
####################################################################################################
Write-Progress `
-Activity $ProgressActivity `
-Status 'Creating scale set' `
-PercentComplete (100 / $TotalProgress * $CurrentProgress++)
$VirtualNetwork = Create-LockedDownNetwork -ResourceGroupName $ResourceGroupName -Location $Location
$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig'
$VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id -Primary -Name $VmssIpConfigName
$VmssIpConfig = New-AzVmssIpConfig -SubnetId $VirtualNetwork.Subnets[0].Id -Primary -Name $VmssIpConfigName
$VmssName = $ResourceGroupName + 'Vmss'
$Vmss = New-AzVmssConfig `
-Location $Location `
@ -513,11 +55,18 @@ $Vmss = New-AzVmssConfig `
-Priority Spot `
-MaxPrice -1
$NicName = $ResourceGroupName + 'NIC'
New-AzNetworkInterface `
-Name $NicName `
-ResourceGroupName $ResourceGroupName `
-Location $Location `
-Subnet $VirtualNetwork.Subnets[0]
$Vmss = Add-AzVmssNetworkInterfaceConfiguration `
-VirtualMachineScaleSet $Vmss `
-Primary $true `
-IpConfiguration $VmssIpConfig `
-NetworkSecurityGroupId $NetworkSecurityGroup.Id `
-NetworkSecurityGroupId $VirtualNetwork.Subnets[0].NetworkSecurityGroup.Id `
-Name $NicName
$Vmss = Set-AzVmssOsProfile `
@ -540,10 +89,6 @@ New-AzVmss `
-Name $VmssName `
-VirtualMachineScaleSet $Vmss
####################################################################################################
Write-Progress -Activity $ProgressActivity -Completed
Write-Host "Location: $Location"
Write-Host "Resource group name: $ResourceGroupName"
Write-Host "User name: AdminUser"
Write-Host "Using generated password: $AdminPW"
Write-Host 'Finished!'

2
scripts/azure-pipelines/windows/deploy-pwsh.ps1
View File

@ -5,5 +5,5 @@
# REPLACE WITH UTILITY-PREFIX.ps1
$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi'
$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/PowerShell-7.1.4-win-x64.msi'
InstallMSI -Url $PwshUrl -Name 'PowerShell Core'

1
scripts/azure-pipelines/windows/deploy-visual-studio.ps1
View File

@ -18,6 +18,7 @@ $Workloads = @(
'Microsoft.VisualStudio.Component.Windows10SDK.18362',
'Microsoft.VisualStudio.Component.Windows10SDK.19041',
'Microsoft.Net.Component.4.8.SDK',
'Microsoft.Net.Component.4.7.2.TargetingPack',
'Microsoft.Component.NetFX.Native',
'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
'Microsoft.VisualStudio.Component.VC.Llvm.Clang',

8
scripts/vcpkgTools.xml
View File

@ -155,10 +155,10 @@
<archiveName>ninja-freebsd-1.8.2.zip</archiveName>
</tool>
<tool name="powershell-core" os="windows">
<version>7.1.3</version>
<version>7.1.4</version>
<exeRelativePath>pwsh.exe</exeRelativePath>
<url>https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x86.zip</url>
<sha512>8c2ce510b5c641aad2da6adefc92d47e09bc842d47db3b5d15e14859555a74fe13ad52eaeabf1b2954ca9af737e628b567731c8a3db9bbf0e4aad05279bc1fd8</sha512>
<archiveName>PowerShell-7.1.3-win-x86.zip</archiveName>
<url>https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/PowerShell-7.1.4-win-x86.zip</url>
<sha512>cf30f80edb57f37501c4a380a8ddd8adf016a51a988a315e15ef517fdae42313ddf5260d8374db65ef12808ec980118bc8b543256df0c1d641a5b8355a80ba7b</sha512>
<archiveName>PowerShell-7.1.4-win-x86.zip</archiveName>
</tool>
</tools>