Add MSVC-internal testing scripts to public GitHub. (#13748)

scripts/azure-pipelines/test-modified-ports.ps1

@@ -9,18 +9,20 @@ Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms.
 .PARAMETER Triplet
 The triplet to test.
 
-.PARAMETER ArchivesRoot
-The location where the binary caching archives are stored. Shared across runs of this script.
-
 .PARAMETER WorkingRoot
 The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories.
 
 .PARAMETER ArtifactStagingDirectory
 The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory.
 
+.PARAMETER ArchivesRoot
+The location where the binary caching archives are stored. Shared across runs of this script. If
+this parameter is not set, binary caching will not be used.
+
 .PARAMETER BuildReason
-The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not
-supplied, binary caching will be used.
+The reason Azure Pipelines is running this script (controls in which mode Binary Caching is used).
+If ArchivesRoot is not set, this parameter has no effect. If ArchivesRoot is set and this is not,
+binary caching will default to read-write mode.
 #>
 
 [CmdletBinding()]
@@ -30,12 +32,10 @@ Param(
     [string]$Triplet,
     [Parameter(Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
-    $ArchivesRoot,
-    [Parameter(Mandatory = $true)]
-    [ValidateNotNullOrEmpty()]
     $WorkingRoot,
     [ValidateNotNullOrEmpty()]
     $ArtifactStagingDirectory = '.',
+    $ArchivesRoot = $null,
     $BuildReason = $null
 )
 
@@ -48,29 +48,39 @@ $env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads'
 $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'packages'
-$commonArgs = @(
-    '--binarycaching',
+
+$usingBinaryCaching = -Not ([string]::IsNullOrWhiteSpace($ArchivesRoot))
+$commonArgs = @()
+if ($usingBinaryCaching) {
+    $commonArgs += @('--binarycaching')
+} else {
+    $commonArgs += @('--no-binarycaching')
+}
+
+$commonArgs += @(
     "--x-buildtrees-root=$buildtreesRoot",
     "--x-install-root=$installRoot",
     "--x-packages-root=$packagesRoot",
     "--overlay-ports=scripts/test_ports"
 )
 
-$binaryCachingMode = 'readwrite'
 $skipFailures = $false
-if ([string]::IsNullOrWhiteSpace($BuildReason)) {
-    Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
-}
-elseif ($BuildReason -eq 'PullRequest') {
-    Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'
-    $skipFailures = $true
-}
-else {
-    Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
-    $binaryCachingMode = 'write'
-}
+if ($usingBinaryCaching) {
+    $binaryCachingMode = 'readwrite'
+    if ([string]::IsNullOrWhiteSpace($BuildReason)) {
+        Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
+    }
+    elseif ($BuildReason -eq 'PullRequest') {
+        Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.'
+        $skipFailures = $true
+    }
+    else {
+        Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
+        $binaryCachingMode = 'write'
+    }
 
-$commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+    $commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")
+}
 
 if ($Triplet -eq 'x64-linux') {
     $env:HOME = '/home/agent'

scripts/azure-pipelines/windows-unstable/README.md

@@ -0,0 +1,4 @@
+The "unstable" build is used internally by Microsoft to test prerelease versions
+of our C++ compiler; not seeing results from these build definitions in the 
+GitHub portal is normal as these builds depend on acquisition of private
+compiler bits that aren't yet shipping.

scripts/azure-pipelines/windows-unstable/azure-pipelines.yml

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+variables:
+  unstable-pool: 'VcpkgUnstable-2020-09-01'
+
+jobs:
+- template: job.yml
+  parameters:
+    triplet: x64-windows
+    jobName: x64_windows

scripts/azure-pipelines/windows-unstable/job.yml

@@ -0,0 +1,90 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+
+jobs:
+- job: ${{ parameters.jobName }}
+  pool:
+    name: $(unstable-pool)
+  workspace:
+    clean: resources
+  timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: D:\
+  - name: VCPKG_DOWNLOADS
+    value: D:\downloads
+
+  steps:
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download DropBuildNumber if not specified'
+    inputs:
+      buildType: specific
+      project: '0bdbc590-a062-4c3f-b0f6-9383f67865ee'
+      pipeline: 8136
+      buildVersionToDownload: latestFromBranch
+      branchName: 'refs/heads/$(MSVCBranchName)'
+      artifactName: BuildNumber
+      downloadPath: 'D:\msvc-drops'
+    condition: eq(variables['DropBuildNumber'], '')
+  - task: PowerShell@2
+    displayName: 'Set DropBuildNumber if not specified'
+    inputs:
+      targetType: inline
+      script: |
+        $DropBuildNumber = Get-Content -Path D:\msvc-drops\BuildNumber\Build.BuildNumber.txt
+        Write-Host "##vso[task.setvariable variable=DropBuildNumber]$DropBuildNumber"
+        Write-Host "Build Number set to: $DropBuildNumber"
+      pwsh: true
+    condition: eq(variables['DropBuildNumber'], '')
+  - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
+    displayName: 'Download msvc x86 ret'
+    inputs:
+      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
+      buildNumber: 'msvc/builds/$(DropBuildNumber)/x86ret'
+      destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.x86ret'
+  - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
+    displayName: 'Download msvc amd64 ret'
+    inputs:
+      dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
+      buildNumber: 'msvc/builds/$(DropBuildNumber)/amd64ret'
+      destinationPath: 'D:\msvc-drops\$(DropBuildNumber)\binaries.amd64ret'
+  - task: PowerShell@2
+    displayName: 'Rearrange MSVC Drop Layout'
+    inputs:
+      targetType: filePath
+      filePath: 'scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1'
+      arguments: '-DropRoot "D:\msvc-drops\$(DropBuildNumber)" -BuildType ret'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Initialize Environment'
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Report on Disk Space'
+    condition: always()
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
+      pwsh: true
+  - script: .\bootstrap-vcpkg.bat
+    displayName: 'Build vcpkg'
+  - task: PowerShell@2
+    displayName: '*** Test Modified Ports and Prepare Test Logs ***'
+    inputs:
+      failOnStderr: true
+      filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)'
+      pwsh: true
+  - task: PowerShell@2
+    displayName: 'Report on Disk Space After Build'
+    condition: always()
+    inputs:
+      filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
+      pwsh: true
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: failure logs for ${{ parameters.triplet }}'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs'
+      ArtifactName: 'failure logs for ${{ parameters.triplet }}'
+    condition: failed()

scripts/azure-pipelines/windows-unstable/rearrange-msvc-drop-layout.ps1

@@ -0,0 +1,75 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: MIT
+#
+<#
+.SYNOPSIS
+Moves files from an MSVC compiler drop to the locations where they are installed in a Visual Studio installation.
+
+.PARAMETER DropRoot
+The location where the MSVC compiler drop has been downloaded.
+
+.PARAMETER BuildType
+The MSVC drop build type set with /p:_BuildType when MSVC was built. Defaults to 'ret'.
+
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)][string]$DropRoot,
+    [Parameter(Mandatory = $false)][ValidateSet('ret', 'chk')][string]$BuildType = 'ret'
+)
+
+Set-StrictMode -Version Latest
+
+$MSVCRoot = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC"
+
+$ErrorActionPreference = "Stop"
+
+$tempRoot = "$DropRoot\readytodeploy"
+
+New-Item -ItemType Directory -Path $tempRoot | Out-Null
+
+Write-Host "Rearranging x86$BuildType"
+New-Item -ItemType Directory -Path "$tempRoot\bin\HostX86" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\i386" "$tempRoot\bin\HostX86\x86"
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_amd64" "$tempRoot\bin\HostX86\x64"
+Move-Item "$DropRoot\binaries.x86$BuildType\bin\x86_arm" "$tempRoot\bin\HostX86\arm"
+
+Write-Host "Rearranging amd64$BuildType"
+New-Item -ItemType Directory -Path "$tempRoot\bin\HostX64" | Out-Null
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64" "$tempRoot\bin\HostX64\x64"
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_x86" "$tempRoot\bin\HostX64\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\bin\amd64_arm" "$tempRoot\bin\HostX64\arm"
+
+# Only copy files and directories that already exist in the VS installation.
+Write-Host "Rearranging inc, lib"
+New-Item -ItemType Directory -Path "$tempRoot\lib" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\inc" "$tempRoot\include"
+Move-Item "$DropRoot\binaries.x86$BuildType\lib\i386" "$tempRoot\lib\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\lib\amd64" "$tempRoot\lib\x64"
+
+Write-Host "Rearranging atlmfc"
+New-Item -ItemType Directory -Path "$tempRoot\atlmfc" | Out-Null
+New-Item -ItemType Directory -Path "$tempRoot\atlmfc\lib" | Out-Null
+Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\include" "$tempRoot\atlmfc\include"
+Move-Item "$DropRoot\binaries.x86$BuildType\atlmfc\lib\i386" "$tempRoot\atlmfc\lib\x86"
+Move-Item "$DropRoot\binaries.amd64$BuildType\atlmfc\lib\amd64" "$tempRoot\atlmfc\lib\x64"
+
+$toolsets = Get-ChildItem -Path $MSVCRoot -Directory | Sort-Object -Descending
+if ($toolsets.Length -eq 0) {
+    throw "Could not find Visual Studio toolset!"
+}
+
+Write-Host "Found toolsets:`n$($toolsets -join `"`n`")`n"
+$selectedToolset = $toolsets[0]
+Write-Host "Using toolset: $selectedToolset"
+for ($idx = 1; $idx -lt $toolsets.Length; $idx++) {
+    $badToolset = $toolsets[$idx]
+    Write-Host "Deleting toolset: $badToolset"
+    Remove-Item $badToolset -Recurse -Force
+}
+
+Write-Host "Deploying $tempRoot => $selectedToolset"
+Copy-Item "$tempRoot\*" $selectedToolset -Recurse -Force
+Write-Host "Deleting $DropRoot..."
+Remove-Item $DropRoot -Recurse -Force
+Write-Host "Done!"

scripts/azure-pipelines/windows/create-vmss.ps1

@@ -14,10 +14,28 @@ for more information.
 This script assumes you have installed Azure tools into PowerShell by following the instructions
 at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1
 or are running from Azure Cloud Shell.
+
+.PARAMETER Unstable
+If this parameter is set, the machine is configured for use in the "unstable" pool used for testing
+the compiler rather than for testing vcpkg. Differences:
+* The machine prefix is changed to VcpkgUnstable instead of PrWin.
+* No storage account or "archives" share is provisioned.
+* The firewall is not opened to allow communication with Azure Storage.
 #>
 
+[CmdLetBinding()]
+Param(
+  [switch]$Unstable = $false
+)
+
 $Location = 'westus2'
-$Prefix = 'PrWin-' + (Get-Date -Format 'yyyy-MM-dd')
+if ($Unstable) {
+  $Prefix = 'VcpkgUnstable-'
+} else {
+  $Prefix = 'PrWin-'
+}
+
+$Prefix += (Get-Date -Format 'yyyy-MM-dd')
 $VMSize = 'Standard_D16a_v4'
 $ProtoVMName = 'PROTOTYPE'
 $LiveVMPrefix = 'BUILD'
@@ -26,6 +44,10 @@ $ErrorActionPreference = 'Stop'
 
 $ProgressActivity = 'Creating Scale Set'
 $TotalProgress = 12
+if ($Unstable) {
+  $TotalProgress -= 1 # skipping the archives share part
+}
+
 $CurrentProgress = 1
 
 Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking
@@ -84,17 +106,19 @@ $allowGit = New-AzNetworkSecurityRuleConfig `
   -DestinationAddressPrefix * `
   -DestinationPortRange 9418
 
-$allowStorage = New-AzNetworkSecurityRuleConfig `
-  -Name AllowStorage `
-  -Description 'Allow Storage' `
-  -Access Allow `
-  -Protocol * `
-  -Direction Outbound `
-  -Priority 1011 `
-  -SourceAddressPrefix VirtualNetwork `
-  -SourcePortRange * `
-  -DestinationAddressPrefix Storage `
-  -DestinationPortRange *
+if (-Not $Unstable) {
+  $allowStorage = New-AzNetworkSecurityRuleConfig `
+    -Name AllowStorage `
+    -Description 'Allow Storage' `
+    -Access Allow `
+    -Protocol * `
+    -Direction Outbound `
+    -Priority 1011 `
+    -SourceAddressPrefix VirtualNetwork `
+    -SourcePortRange * `
+    -DestinationAddressPrefix Storage `
+    -DestinationPortRange *
+}
 
 $denyEverythingElse = New-AzNetworkSecurityRuleConfig `
   -Name DenyElse `
@@ -109,11 +133,18 @@ $denyEverythingElse = New-AzNetworkSecurityRuleConfig `
   -DestinationPortRange *
 
 $NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity'
+$securityRules = @($allowHttp, $allowDns, $allowGit);
+if (-Not $Unstable) {
+  $securityRules += @($allowStorage)
+}
+
+$securityRules += @($denyEverythingElse)
+
 $NetworkSecurityGroup = New-AzNetworkSecurityGroup `
   -Name $NetworkSecurityGroupName `
   -ResourceGroupName $ResourceGroupName `
   -Location $Location `
-  -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse)
+  -SecurityRules $securityRules
 
 $SubnetName = $ResourceGroupName + 'Subnet'
 $Subnet = New-AzVirtualNetworkSubnetConfig `
@@ -130,32 +161,34 @@ $VirtualNetwork = New-AzVirtualNetwork `
   -Subnet $Subnet
 
 ####################################################################################################
-Write-Progress `
-  -Activity $ProgressActivity `
-  -Status 'Creating archives storage account' `
-  -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
+if (-Not $Unstable) {
+  Write-Progress `
+    -Activity $ProgressActivity `
+    -Status 'Creating archives storage account' `
+    -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
 
-$StorageAccountName = Sanitize-Name $ResourceGroupName
+  $StorageAccountName = Sanitize-Name $ResourceGroupName
 
-New-AzStorageAccount `
-  -ResourceGroupName $ResourceGroupName `
-  -Location $Location `
-  -Name $StorageAccountName `
-  -SkuName 'Standard_LRS' `
-  -Kind StorageV2
+  New-AzStorageAccount `
+    -ResourceGroupName $ResourceGroupName `
+    -Location $Location `
+    -Name $StorageAccountName `
+    -SkuName 'Standard_LRS' `
+    -Kind StorageV2
 
-$StorageAccountKeys = Get-AzStorageAccountKey `
-  -ResourceGroupName $ResourceGroupName `
-  -Name $StorageAccountName
+  $StorageAccountKeys = Get-AzStorageAccountKey `
+    -ResourceGroupName $ResourceGroupName `
+    -Name $StorageAccountName
 
-$StorageAccountKey = $StorageAccountKeys[0].Value
+  $StorageAccountKey = $StorageAccountKeys[0].Value
 
-$StorageContext = New-AzStorageContext `
-  -StorageAccountName $StorageAccountName `
-  -StorageAccountKey $StorageAccountKey
+  $StorageContext = New-AzStorageContext `
+    -StorageAccountName $StorageAccountName `
+    -StorageAccountKey $StorageAccountKey
 
-New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048
+  New-AzStorageShare -Name 'archives' -Context $StorageContext
+  Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048
+}
 
 ####################################################################################################
 Write-Progress `
@@ -198,14 +231,18 @@ Write-Progress `
   -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' `
   -PercentComplete (100 / $TotalProgress * $CurrentProgress++)
 
+$provisionParameters = @{AdminUserPassword = $AdminPW;}
+if (-Not $Unstable) {
+  $provisionParameters['StorageAccountName'] = $StorageAccountName
+  $provisionParameters['StorageAccountKey'] = $StorageAccountKey
+}
+
 $ProvisionImageResult = Invoke-AzVMRunCommand `
   -ResourceGroupName $ResourceGroupName `
   -VMName $ProtoVMName `
   -CommandId 'RunPowerShellScript' `
   -ScriptPath "$PSScriptRoot\provision-image.txt" `
-  -Parameter @{AdminUserPassword = $AdminPW; `
-    StorageAccountName=$StorageAccountName; `
-    StorageAccountKey=$StorageAccountKey;}
+  -Parameter $provisionParameters
 
 Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)"
 

scripts/azure-pipelines/windows/initialize-environment.ps1

@@ -8,23 +8,18 @@ Sets up the environment to run other vcpkg CI steps in an Azure Pipelines job.
 .DESCRIPTION
 This script maps network drives from infrastructure and cleans out anything that
 might have been leftover from a previous run.
-
-.PARAMETER ForceAllPortsToRebuildKey
-A subdirectory / key to use to force a build without any previous run caching,
-if necessary.
 #>
 
-[CmdletBinding()]
-Param(
-    [string]$ForceAllPortsToRebuildKey = ''
-)
+if ([string]::IsNullOrWhiteSpace($env:StorageAccountName) -or  [string]::IsNullOrWhiteSpace($env:StorageAccountKey)) {
+    Write-Host 'No storage account and/or key set, skipping mount of W:\'
+} else {
+    $StorageAccountName = $env:StorageAccountName
+    $StorageAccountKey = $env:StorageAccountKey
 
-$StorageAccountName = $env:StorageAccountName
-$StorageAccountKey = $env:StorageAccountKey
-
-Write-Host 'Setting up archives mount'
-if (-Not (Test-Path W:)) {
-    net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey
+    Write-Host 'Setting up archives mount'
+    if (-Not (Test-Path W:)) {
+        net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey
+    }
 }
 
 Write-Host 'Creating downloads directory'