[vcpkg] [llvm] [mpir] Bump Linux VM memory size and do all operations on the temporary disk. (#11174)

* [vcpkg] Bump Linux VM memory size and do all operations on the temporary disk.

* [llvm] disable optional dependencies to libxml2 and zlib

* [vcpkg] Reduce quotas on the share to something reasonable.

* [mpir] Skip on Linux because it conflicts with libgmp.

Co-authored-by: yurybura <yurybura@gmail.com>

ports/llvm/CONTROL

@@ -1,5 +1,5 @@
 Source: llvm
-Version: 10.0.0
+Version: 10.0.0-1
 Homepage: https://llvm.org/
 Description: The LLVM Compiler Infrastructure
 Supports: !uwp

ports/llvm/portfile.cmake

@@ -142,6 +142,9 @@ vcpkg_configure_cmake(
         -DLLVM_BUILD_EXAMPLES=OFF
         -DLLVM_INCLUDE_TESTS=OFF
         -DLLVM_BUILD_TESTS=OFF
+        # Disable optional dependencies to libxml2 and zlib
+        -DLLVM_ENABLE_LIBXML2=OFF
+        -DLLVM_ENABLE_ZLIB=OFF
         # Force TableGen to be built with optimization. This will significantly improve build time.
         -DLLVM_OPTIMIZED_TABLEGEN=ON
         # LLVM generates CMake error due to Visual Studio version 16.4 is known to miscompile part of LLVM.

scripts/azure-pipelines/azure-pipelines.yml

@@ -3,7 +3,7 @@
 #
 variables:
   windows-pool: 'PrWin-2020-04-28'
-  linux-pool: 'PrLin-2020-04-30'
+  linux-pool: 'PrLin-2020-05-07'
 
 jobs:
   - template: windows/azure-pipelines.yml

scripts/azure-pipelines/clean-tombstones.yml

@@ -3,7 +3,7 @@
 #
 variables:
   windows-pool: 'PrWin-2020-04-28'
-  linux-pool: 'PrLin-2020-04-30'
+  linux-pool: 'PrLin-2020-05-07'
 
 jobs:
   - job: windows

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -27,8 +27,7 @@ jobs:
   - powershell: |
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
       mkdir $(System.ArtifactsDirectory)/xml-results
-      mkdir /ci/home
+      $env:HOME = '/home/root'
-      $env:HOME = '/ci/home'
       ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2

scripts/azure-pipelines/linux/create-vmss.ps1

@@ -18,10 +18,9 @@ or are running from Azure Cloud Shell.
 
 $Location = 'westus2'
 $Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd')
-$VMSize = 'Standard_F16s_v2'
+$VMSize = 'Standard_D16a_v4'
 $ProtoVMName = 'PROTOTYPE'
 $LiveVMPrefix = 'BUILD'
-$InstalledDiskSizeInGB = 1024
 $ErrorActionPreference = 'Stop'
 
 $ProgressActivity = 'Creating Scale Set'
@@ -259,7 +258,7 @@ $StorageContext = New-AzStorageContext `
   -StorageAccountKey $StorageAccountKey
 
 New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120
+Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 1024
 
 ####################################################################################################
 Write-Progress `
@@ -273,7 +272,7 @@ $Nic = New-AzNetworkInterface `
   -Location $Location `
   -Subnet $VirtualNetwork.Subnets[0]
 
-$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize
+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
 $VM = Set-AzVMOperatingSystem `
   -VM $VM `
   -Linux `
@@ -377,8 +376,7 @@ $Vmss = Set-AzVmssOsProfile `
   -VirtualMachineScaleSet $Vmss `
   -ComputerNamePrefix $LiveVMPrefix `
   -AdminUsername AdminUser `
-  -AdminPassword $AdminPW `
+  -AdminPassword $AdminPW
-  -CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n")))
 
 $Vmss = Set-AzVmssStorageProfile `
   -VirtualMachineScaleSet $Vmss `
@@ -386,14 +384,6 @@ $Vmss = Set-AzVmssStorageProfile `
   -OsDiskCaching ReadWrite `
   -ImageReferenceId $Image.Id
 
-$Vmss = Add-AzVmssDataDisk `
-  -VirtualMachineScaleSet $Vmss `
-  -Lun 0 `
-  -Caching 'ReadWrite' `
-  -CreateOption Empty `
-  -DiskSizeGB 1024 `
-  -StorageAccountType 'StandardSSD_LRS'
-
 New-AzVmss `
   -ResourceGroupName $ResourceGroupName `
   -Name $VmssName `

scripts/azure-pipelines/linux/initialize-environment.sh

@@ -4,12 +4,9 @@
 
 # Cleans up the environment to prevent contamination across builds.
 if [ ! -d "archives" ]; then
-    ln -s /ci/archives archives
+    ln -s /archives archives
-fi
-if [ ! -d "installed" ]; then
-    ln -s /ci/installed installed
 fi
 
-rm -rf installed/*
+rm -rf installed
 rm -rf buildtrees
-rm -rf packages
+rm -rf packages/*

scripts/azure-pipelines/linux/provision-image.sh

@@ -50,19 +50,12 @@ echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null
 echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null
 sudo chmod 600 $smbCredentialFile
 
-# Write script to provision disks used by cloud-init
+# Mount the archives SMB share to /archives
-echo "if [ ! -d \"/ci\" ]; then" > /etc/provision-disks.sh
+sudo mkdir /archives -m=777
-echo "sudo parted /dev/sdc mklabel gpt" >> /etc/provision-disks.sh
+echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab
-echo "sudo parted /dev/sdc mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh
+
-echo "sudo mkfs -t ext4 /dev/sdc1" >> /etc/provision-disks.sh
+# Create 'home' directory for haskell stack bits that want this
-echo "sudo mkdir /ci -m=777" >> /etc/provision-disks.sh
+sudo mkdir -p /home/root -m=777
-echo "sudo mkdir /ci/installed -m=777" >> /etc/provision-disks.sh
-echo "sudo mkdir /ci/archives -m=777" >> /etc/provision-disks.sh
-echo "echo \"/dev/sdc1 /ci/installed ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh
-echo "echo \"//$StorageAccountName.file.core.windows.net/archives /ci/archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh
-echo "sudo mount -a" >> /etc/provision-disks.sh
-echo "fi" >> /etc/provision-disks.sh
-sudo chmod 700 /etc/provision-disks.sh
 
 # Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above
 sudo rm /etc/debian_version
@@ -71,3 +64,8 @@ sudo rm /etc/debian_version
 # https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux
 # (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs)
 apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60
+
+# Create work trees for the Azure Pipelines agent so that it puts its work tree into temporary storage.
+sudo chmod 777 /mnt
+sudo mkdir /agent -m=777
+sudo ln -s /mnt /agent/_work

scripts/azure-pipelines/windows/create-vmss.ps1

@@ -294,7 +294,7 @@ $StorageContext = New-AzStorageContext `
   -StorageAccountKey $StorageAccountKey
 
 New-AzStorageShare -Name 'archives' -Context $StorageContext
-Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120
+Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 2048
 
 ####################################################################################################
 Write-Progress `
@@ -308,7 +308,7 @@ $Nic = New-AzNetworkInterface `
   -Location $Location `
   -Subnet $VirtualNetwork.Subnets[0]
 
-$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize
+$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize -Priority 'Spot' -MaxPrice -1
 $VM = Set-AzVMOperatingSystem `
   -VM $VM `
   -Windows `

scripts/ci.baseline.txt

@@ -1006,7 +1006,6 @@ llgl:arm-uwp=fail
 llgl:x64-uwp=fail
 llvm:arm-uwp=fail
 llvm:arm64-windows=fail
-llvm:x64-linux=fail
 llvm:x64-uwp=fail
 lmdb:arm64-windows=fail
 lmdb:arm-uwp=fail
@@ -1134,7 +1133,17 @@ mpg123:x64-uwp=fail
 mpir:arm64-windows=fail
 mpir:arm-uwp=fail
 mpir:x64-uwp=fail
-
+# Building package gmp[core]:x64-linux... done
+# Installing package gmp[core]:x64-linux...
+# The following files are already installed in /mnt/1/s/installed/x64-linux and are in conflict with gmp:x64-linux
+#
+# Installed by mpir:x64-linux
+#     debug/lib/libgmp.a
+#     debug/lib/libgmp.la
+#     include/gmp.h
+#     lib/libgmp.a
+#     lib/libgmp.la
+mpir:x64-linux=skip
 #Conflicts with angle and qt-5base
 ms-angle:arm64-windows      = skip
 ms-angle:arm-uwp            = skip
@@ -1868,8 +1877,8 @@ usd:x64-linux=ignore
 usd:x86-windows=fail
 usrsctp:arm-uwp=fail
 usrsctp:x64-uwp=fail
-uthenticode:arm-uwp=fail
+uthenticode:arm-uwp=fail
-uthenticode:x64-uwp=fail
+uthenticode:x64-uwp=fail
 uvatlas:arm64-windows=fail
 uvatlas:arm-uwp=fail
 uvatlas:x64-linux=fail