From 4dc9ecd32b972af1da00d7011f9b9822b4254ef7 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Mon, 28 Sep 2020 13:54:10 -0700 Subject: [PATCH] Add MSVC-internal testing scripts to public GitHub. (#13748) --- .../azure-pipelines/test-modified-ports.ps1 | 56 +++++---- .../windows-unstable/README.md | 4 + .../windows-unstable/azure-pipelines.yml | 11 ++ .../azure-pipelines/windows-unstable/job.yml | 90 +++++++++++++++ .../rearrange-msvc-drop-layout.ps1 | 75 ++++++++++++ .../azure-pipelines/windows/create-vmss.ps1 | 109 ++++++++++++------ .../windows/initialize-environment.ps1 | 23 ++-- 7 files changed, 295 insertions(+), 73 deletions(-) create mode 100644 scripts/azure-pipelines/windows-unstable/README.md create mode 100644 scripts/azure-pipelines/windows-unstable/azure-pipelines.yml create mode 100644 scripts/azure-pipelines/windows-unstable/job.yml create mode 100644 scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1 diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 index 773c55d138..15cb8131f5 100755 --- a/scripts/azure-pipelines/test-modified-ports.ps1 +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -9,18 +9,20 @@ Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms. .PARAMETER Triplet The triplet to test. -.PARAMETER ArchivesRoot -The location where the binary caching archives are stored. Shared across runs of this script. - .PARAMETER WorkingRoot The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories. .PARAMETER ArtifactStagingDirectory The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory. +.PARAMETER ArchivesRoot +The location where the binary caching archives are stored. Shared across runs of this script. If +this parameter is not set, binary caching will not be used. + .PARAMETER BuildReason -The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not -supplied, binary caching will be used. +The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used). +If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set and this is not, +binary caching will default to read-write mode. #> [CmdletBinding()] @@ -30,12 +32,10 @@ Param( [string]$Triplet, [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] - $ArchivesRoot, - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] $WorkingRoot, [ValidateNotNullOrEmpty()] $ArtifactStagingDirectory = '.', + $ArchivesRoot = $null, $BuildReason = $null ) @@ -48,29 +48,39 @@ $env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads' $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees' $installRoot = Join-Path $WorkingRoot 'installed' $packagesRoot = Join-Path $WorkingRoot 'packages' -$commonArgs = @( - '--binarycaching', + +$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot)) +$commonArgs = @() +if ($usingBinaryCaching) { + $commonArgs += @('--binarycaching') +} else { + $commonArgs += @('--no-binarycaching') +} + +$commonArgs += @( "--x-buildtrees-root=$buildtreesRoot", "--x-install-root=$installRoot", "--x-packages-root=$packagesRoot", "--overlay-ports=scripts/test_ports" ) -$binaryCachingMode = 'readwrite' $skipFailures = $false -if ([string]::IsNullOrWhiteSpace($BuildReason)) { - Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.' -} -elseif ($BuildReason -eq 'PullRequest') { - Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.' - $skipFailures = $true -} -else { - Write-Host "Build reason was $BuildReason, using binary caching in write only mode." - $binaryCachingMode = 'write' -} +if ($usingBinaryCaching) { + $binaryCachingMode = 'readwrite' + if ([string]::IsNullOrWhiteSpace($BuildReason)) { + Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.' + } + elseif ($BuildReason -eq 'PullRequest') { + Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.' + $skipFailures = $true + } + else { + Write-Host "Build reason was $BuildReason, using binary caching in write only mode." + $binaryCachingMode = 'write' + } -$commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode") + $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode") +} if ($Triplet -eq 'x64-linux') { $env:HOME = '/home/agent' diff --git a/scripts/azure-pipelines/windows-unstable/README.md b/scripts/azure-pipelines/windows-unstable/README.md new file mode 100644 index 0000000000..d60367247a --- /dev/null +++ b/scripts/azure-pipelines/windows-unstable/README.md @@ -0,0 +1,4 @@ +The "unstable" build is used internally by Microsoft to test prerelease versions +of our C++ compiler; not seeing results from these build definitions in the +GitHub portal is normal as these builds depend on acquisition of private +compiler bits that aren't yet shipping. diff --git a/scripts/azure-pipelines/windows-unstable/azure-pipelines.yml b/scripts/azure-pipelines/windows-unstable/azure-pipelines.yml new file mode 100644 index 0000000000..189ab31acd --- /dev/null +++ b/scripts/azure-pipelines/windows-unstable/azure-pipelines.yml @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +variables: + unstable-pool: 'VcpkgUnstable-2020-09-01' + +jobs: +- template: job.yml + parameters: + triplet: x64-windows + jobName: x64_windows diff --git a/scripts/azure-pipelines/windows-unstable/job.yml b/scripts/azure-pipelines/windows-unstable/job.yml new file mode 100644 index 0000000000..c2eced1795 --- /dev/null +++ b/scripts/azure-pipelines/windows-unstable/job.yml @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +jobs: +- job: ${{ parameters.jobName }} + pool: + name: $(unstable-pool) + workspace: + clean: resources + timeoutInMinutes: 1440 # 1 day + variables: + - name: WORKING_ROOT + value: D:\ + - name: VCPKG_DOWNLOADS + value: D:\downloads + + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download DropBuildNumber if not specified' + inputs: + buildType: specific + project: '0bdbc590-a062-4c3f-b0f6-9383f67865ee' + pipeline: 8136 + buildVersionToDownload: latestFromBranch + branchName: 'refs/heads/$(MSVCBranchName)' + artifactName: BuildNumber + downloadPath: 'D:\msvc-drops' + condition: eq(variables['DropBuildNumber'], '') + - task: PowerShell@2 + displayName: 'Set DropBuildNumber if not specified' + inputs: + targetType: inline + script: | + $DropBuildNumber = Get-Content -Path D:\msvc-drops\BuildNumber\Build.BuildNumber.txt + Write-Host "##vso[task.setvariable variable=DropBuildNumber]$DropBuildNumber" + Write-Host "Build Number set to: $DropBuildNumber" + pwsh: true + condition: eq(variables['DropBuildNumber'], '') + - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 + displayName: 'Download msvc x86 ret' + inputs: + dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' + buildNumber: 'msvc/builds/$(DropBuildNumber)/x86ret' + destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.x86ret' + - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 + displayName: 'Download msvc amd64 ret' + inputs: + dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' + buildNumber: 'msvc/builds/$(DropBuildNumber)/amd64ret' + destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.amd64ret' + - task: PowerShell@2 + displayName: 'Rearrange MSVC Drop Layout' + inputs: + targetType: filePath + filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1' + arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret' + pwsh: true + - task: PowerShell@2 + displayName: 'Initialize Environment' + inputs: + filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' + pwsh: true + - task: PowerShell@2 + displayName: 'Report on Disk Space' + condition: always() + inputs: + filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' + pwsh: true + - script: .\bootstrap-vcpkg.bat + displayName: 'Build vcpkg' + - task: PowerShell@2 + displayName: '*** Test Modified Ports and Prepare Test Logs ***' + inputs: + failOnStderr: true + filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' + arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)' + pwsh: true + - task: PowerShell@2 + displayName: 'Report on Disk Space After Build' + condition: always() + inputs: + filePath: 'scripts/azure-pipelines/windows/disk-space.ps1' + pwsh: true + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs' + ArtifactName: 'failure logs for ${{ parameters.triplet }}' + condition: failed() diff --git a/scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1 b/scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1 new file mode 100644 index 0000000000..d409bc2085 --- /dev/null +++ b/scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1 @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# +<# +.SYNOPSIS +Moves files from an MSVC compiler drop to the locations where they are installed in a Visual Studio installation. + +.PARAMETER DropRoot +The location where the MSVC compiler drop has been downloaded. + +.PARAMETER BuildType +The MSVC drop build type set with /p:_BuildType when MSVC was built. Defaults to 'ret'. + +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$DropRoot, + [Parameter(Mandatory = $false)][ValidateSet('ret', 'chk')][string]$BuildType = 'ret' +) + +Set-StrictMode -Version Latest + +$MSVCRoot = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC" + +$ErrorActionPreference = "Stop" + +$tempRoot = "$DropRoot\readytodeploy" + +New-Item -ItemType Directory -Path $tempRoot | Out-Null + +Write-Host "Rearranging x86$BuildType" +New-Item -ItemType Directory -Path "$tempRoot\bin\HostX86" | Out-Null +Move-Item "$DropRoot\binaries.x86$BuildType\bin\i386" "$tempRoot\bin\HostX86\x86" +Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_amd64" "$tempRoot\bin\HostX86\x64" +Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_arm" "$tempRoot\bin\HostX86\arm" + +Write-Host "Rearranging amd64$BuildType" +New-Item -ItemType Directory -Path "$tempRoot\bin\HostX64" | Out-Null +Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64" "$tempRoot\bin\HostX64\x64" +Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_x86" "$tempRoot\bin\HostX64\x86" +Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_arm" "$tempRoot\bin\HostX64\arm" + +# Only copy files and directories that already exist in the VS installation. +Write-Host "Rearranging inc, lib" +New-Item -ItemType Directory -Path "$tempRoot\lib" | Out-Null +Move-Item "$DropRoot\binaries.x86$BuildType\inc" "$tempRoot\include" +Move-Item "$DropRoot\binaries.x86$BuildType\lib\i386" "$tempRoot\lib\x86" +Move-Item "$DropRoot\binaries.amd64$BuildType\lib\amd64" "$tempRoot\lib\x64" + +Write-Host "Rearranging atlmfc" +New-Item -ItemType Directory -Path "$tempRoot\atlmfc" | Out-Null +New-Item -ItemType Directory -Path "$tempRoot\atlmfc\lib" | Out-Null +Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\include" "$tempRoot\atlmfc\include" +Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\lib\i386" "$tempRoot\atlmfc\lib\x86" +Move-Item "$DropRoot\binaries.amd64$BuildType\atlmfc\lib\amd64" "$tempRoot\atlmfc\lib\x64" + +$toolsets = Get-ChildItem -Path $MSVCRoot -Directory | Sort-Object -Descending +if ($toolsets.Length -eq 0) { + throw "Could not find Visual Studio toolset!" +} + +Write-Host "Found toolsets:`n$($toolsets -join `"`n`")`n" +$selectedToolset = $toolsets[0] +Write-Host "Using toolset: $selectedToolset" +for ($idx = 1; $idx -lt $toolsets.Length; $idx++) { + $badToolset = $toolsets[$idx] + Write-Host "Deleting toolset: $badToolset" + Remove-Item $badToolset -Recurse -Force +} + +Write-Host "Deploying $tempRoot => $selectedToolset" +Copy-Item "$tempRoot\*" $selectedToolset -Recurse -Force +Write-Host "Deleting $DropRoot..." +Remove-Item $DropRoot -Recurse -Force +Write-Host "Done!" diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 3bb89ee259..1fcec50ed2 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -14,10 +14,28 @@ for more information. This script assumes you have installed Azure tools into PowerShell by following the instructions at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1 or are running from Azure Cloud Shell. + +.PARAMETER Unstable +If this parameter is set, the machine is configured for use in the "unstable" pool used for testing +the compiler rather than for testing vcpkg. Differences: +* The machine prefix is changed to VcpkgUnstable instead of PrWin. +* No storage account or "archives" share is provisioned. +* The firewall is not opened to allow communication with Azure Storage. #> +[CmdLetBinding()] +Param( + [switch]$Unstable = $false +) + $Location = 'westus2' -$Prefix = 'PrWin-' + (Get-Date -Format 'yyyy-MM-dd') +if ($Unstable) { + $Prefix = 'VcpkgUnstable-' +} else { + $Prefix = 'PrWin-' +} + +$Prefix += (Get-Date -Format 'yyyy-MM-dd') $VMSize = 'Standard_D16a_v4' $ProtoVMName = 'PROTOTYPE' $LiveVMPrefix = 'BUILD' @@ -26,6 +44,10 @@ $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' $TotalProgress = 12 +if ($Unstable) { + $TotalProgress -= 1 # skipping the archives share part +} + $CurrentProgress = 1 Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking @@ -84,17 +106,19 @@ $allowGit = New-AzNetworkSecurityRuleConfig ` -DestinationAddressPrefix * ` -DestinationPortRange 9418 -$allowStorage = New-AzNetworkSecurityRuleConfig ` - -Name AllowStorage ` - -Description 'Allow Storage' ` - -Access Allow ` - -Protocol * ` - -Direction Outbound ` - -Priority 1011 ` - -SourceAddressPrefix VirtualNetwork ` - -SourcePortRange * ` - -DestinationAddressPrefix Storage ` - -DestinationPortRange * +if (-Not $Unstable) { + $allowStorage = New-AzNetworkSecurityRuleConfig ` + -Name AllowStorage ` + -Description 'Allow Storage' ` + -Access Allow ` + -Protocol * ` + -Direction Outbound ` + -Priority 1011 ` + -SourceAddressPrefix VirtualNetwork ` + -SourcePortRange * ` + -DestinationAddressPrefix Storage ` + -DestinationPortRange * +} $denyEverythingElse = New-AzNetworkSecurityRuleConfig ` -Name DenyElse ` @@ -109,11 +133,18 @@ $denyEverythingElse = New-AzNetworkSecurityRuleConfig ` -DestinationPortRange * $NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity' +$securityRules = @($allowHttp, $allowDns, $allowGit); +if (-Not $Unstable) { + $securityRules += @($allowStorage) +} + +$securityRules += @($denyEverythingElse) + $NetworkSecurityGroup = New-AzNetworkSecurityGroup ` -Name $NetworkSecurityGroupName ` -ResourceGroupName $ResourceGroupName ` -Location $Location ` - -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse) + -SecurityRules $securityRules $SubnetName = $ResourceGroupName + 'Subnet' $Subnet = New-AzVirtualNetworkSubnetConfig ` @@ -130,32 +161,34 @@ $VirtualNetwork = New-AzVirtualNetwork ` -Subnet $Subnet #################################################################################################### -Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Creating archives storage account' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +if (-Not $Unstable) { + Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating archives storage account' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -$StorageAccountName = Sanitize-Name $ResourceGroupName + $StorageAccountName = Sanitize-Name $ResourceGroupName -New-AzStorageAccount ` - -ResourceGroupName $ResourceGroupName ` - -Location $Location ` - -Name $StorageAccountName ` - -SkuName 'Standard_LRS' ` - -Kind StorageV2 + New-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Name $StorageAccountName ` + -SkuName 'Standard_LRS' ` + -Kind StorageV2 -$StorageAccountKeys = Get-AzStorageAccountKey ` - -ResourceGroupName $ResourceGroupName ` - -Name $StorageAccountName + $StorageAccountKeys = Get-AzStorageAccountKey ` + -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName -$StorageAccountKey = $StorageAccountKeys[0].Value + $StorageAccountKey = $StorageAccountKeys[0].Value -$StorageContext = New-AzStorageContext ` - -StorageAccountName $StorageAccountName ` - -StorageAccountKey $StorageAccountKey + $StorageContext = New-AzStorageContext ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey -New-AzStorageShare -Name 'archives' -Context $StorageContext -Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048 + New-AzStorageShare -Name 'archives' -Context $StorageContext + Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048 +} #################################################################################################### Write-Progress ` @@ -198,14 +231,18 @@ Write-Progress ` -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +$provisionParameters = @{AdminUserPassword = $AdminPW;} +if (-Not $Unstable) { + $provisionParameters['StorageAccountName'] = $StorageAccountName + $provisionParameters['StorageAccountKey'] = $StorageAccountKey +} + $ProvisionImageResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` -ScriptPath "$PSScriptRoot\provision-image.txt" ` - -Parameter @{AdminUserPassword = $AdminPW; ` - StorageAccountName=$StorageAccountName; ` - StorageAccountKey=$StorageAccountKey;} + -Parameter $provisionParameters Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)" diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1 index 4a252df49b..24520802e4 100644 --- a/scripts/azure-pipelines/windows/initialize-environment.ps1 +++ b/scripts/azure-pipelines/windows/initialize-environment.ps1 @@ -8,23 +8,18 @@ Sets up the environment to run other vcpkg CI steps in an Azure Pipelines job. .DESCRIPTION This script maps network drives from infrastructure and cleans out anything that might have been leftover from a previous run. - -.PARAMETER ForceAllPortsToRebuildKey -A subdirectory / key to use to force a build without any previous run caching, -if necessary. #> -[CmdletBinding()] -Param( - [string]$ForceAllPortsToRebuildKey = '' -) +if ([string]::IsNullOrWhiteSpace($env:StorageAccountName) -or [string]::IsNullOrWhiteSpace($env:StorageAccountKey)) { + Write-Host 'No storage account and/or key set, skipping mount of W:\' +} else { + $StorageAccountName = $env:StorageAccountName + $StorageAccountKey = $env:StorageAccountKey -$StorageAccountName = $env:StorageAccountName -$StorageAccountKey = $env:StorageAccountKey - -Write-Host 'Setting up archives mount' -if (-Not (Test-Path W:)) { - net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey + Write-Host 'Setting up archives mount' + if (-Not (Test-Path W:)) { + net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey + } } Write-Host 'Creating downloads directory'