[ci] Update macOS to 11 (#17376)

* start 2021-04-16 process

Major new things:
* update to macOS Big Sur (11.*)
* switch from VirtualBox to Parallels due to ^ (and also ARM)

Minor new things:
* update from xcode CLT 12 to 12.4

This PR includes new stuff for creating the base box for parallels.
Still to do: using the new box type.

* update the vagrantfile stuff

* link the CI to the new version

* modify how macOS boxes are made

the Vagrantfile for the final VM will only download the azure pipeline stuff

this allows us to keep the versions of brew applications stable

* fix cpus and memory

* add vagrant plugins to installables

* Skip cppgraphqlgen ICE.

* [sdformat6] Remove unneeded include(FindBoost) which causes problems when the system cmake version doesn't match the one deployed by vcpkg.

* switch to dmg for installing osxfuse/sshfs

* Set cores to 11 (leaving 1 thread for host)

Co-authored-by: Billy Robert ONeal III <bion@microsoft.com>

ports/sdformat6/CONTROL

@@ -1,6 +0,0 @@
-Source: sdformat6
-Version: 6.2.0-1
-Homepage: http://sdformat.org/
-Build-Depends: boost-any, boost-variant, ignition-math4, urdfdom, tinyxml
-Description: Simulation Description Format (SDF) parser and description files.
-Supports: !(arm|uwp)

ports/sdformat6/disable-unneeded-include-findboost.patch

@@ -0,0 +1,12 @@
+diff --git a/cmake/SearchForStuff.cmake b/cmake/SearchForStuff.cmake
+index c2a0ee4..2735a07 100644
+--- a/cmake/SearchForStuff.cmake
++++ b/cmake/SearchForStuff.cmake
+@@ -13,7 +13,6 @@ if (WIN32)
+   set(Boost_USE_STATIC_RUNTIME    OFF)
+ endif()
+ 
+-include(FindBoost)
+ find_package(Boost ${MIN_BOOST_VERSION})
+ 
+ if (NOT Boost_FOUND)

ports/sdformat6/portfile.cmake

@@ -6,6 +6,8 @@ vcpkg_from_github(
     REF sdformat6_6.2.0
     SHA512 3d139ec4b4c9fbfd547ed8bfca0adb5cdca92c1b7cc4d4b554a7c51ccf755b9079c26a006ebfedc5bc5b1ba5e16ad950bb38c47ea97bf97e59a2fd7d12d60620
     HEAD_REF sdf6
+    PATCHES
+        disable-unneeded-include-findboost.patch
 )
 
 # Ruby is required by the sdformat build process

ports/sdformat6/vcpkg.json

@@ -0,0 +1,15 @@
+{
+  "name": "sdformat6",
+  "version": "6.2.0",
+  "port-version": 2,
+  "description": "Simulation Description Format (SDF) parser and description files.",
+  "homepage": "http://sdformat.org/",
+  "supports": "!(arm | uwp)",
+  "dependencies": [
+    "boost-any",
+    "boost-variant",
+    "ignition-math4",
+    "tinyxml",
+    "urdfdom"
+  ]
+}

scripts/azure-pipelines/azure-pipelines.yml

@@ -4,7 +4,7 @@
 variables:
   windows-pool: 'PrWin-2021-04-23'
   linux-pool: 'PrLin-2021-04-25'
-  osx-pool: 'PrOsx-2020-09-28'
+  osx-pool: 'PrOsx-2021-04-16'
 
 stages:
 - stage: FormatChecks

scripts/azure-pipelines/osx/Install-Prerequisites.ps1

@@ -8,10 +8,7 @@ Installs the set of prerequisites for the macOS CI hosts.
 .DESCRIPTION
 Install-Prerequisites.ps1 installs all of the necessary prerequisites
 to run the vcpkg macOS CI in a vagrant virtual machine,
-skipping all prerequisites that are already installed.
-
-.PARAMETER Force
-Don't skip the prerequisites that are already installed.
+skipping all prerequisites that are already installed and of the right version.
 
 .INPUTS
 None
@@ -20,10 +17,7 @@ None
 None
 #>
 [CmdletBinding()]
-Param(
-    [Parameter()]
-    [Switch]$Force
-)
+Param()
 
 Set-StrictMode -Version 2
 
@@ -37,21 +31,51 @@ Import-Module "$PSScriptRoot/Utilities.psm1"
 $Installables = Get-Content "$PSScriptRoot/configuration/installables.json" | ConvertFrom-Json
 
 $Installables.Applications | ForEach-Object {
-    if (-not (Get-CommandExists $_.TestCommand)) {
+    $VersionCommand = $_.VersionCommand
+    $InstalledVersion = (& $VersionCommand[0] $VersionCommand[1..$VersionCommand.Length])
+    if (-not $?) {
         Write-Host "$($_.Name) not installed; installing now"
-    } elseif ($Force) {
-        Write-Host "$($_.Name) found; attempting to upgrade or re-install"
     } else {
-        Write-Host "$($_.Name) already installed"
-        return
+        $InstalledVersion = $InstalledVersion -join "`n"
+        if ($InstalledVersion -match $_.VersionRegex) {
+            if ($Matches.Count -ne 2) {
+                Write-Error "$($_.Name) has a malformed version regex ($($_.VersionRegex)); it should have a single capture group
+    (it has $($Matches.Count - 1))"
+                throw
+            }
+            if ($Matches[1] -eq $_.Version) {
+                Write-Host "$($_.Name) already installed and at the correct version ($($Matches[1]))"
+                return
+            } else {
+                Write-Host "$($_.Name) already installed but with the incorrect version
+    installed version: '$($Matches[1])'
+    required version : '$($_.Version)'
+upgrading now."
+            }
+        } else {
+            Write-Warning "$($_.Name)'s version command ($($VersionCommand -join ' ')) returned a value we did not expect:
+    $InstalledVersion
+    expected a version matching the regex: $($_.VersionRegex)
+Installing anyways."
+        }
     }
 
-    $pathToDmg = "~/Downloads/$($_.Name).dmg"
-    Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256
+    if ($null -ne (Get-Member -InputObject $_ -Name 'DmgUrl')) {
+        $pathToDmg = "~/Downloads/$($_.Name).dmg"
+        Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256
 
-    hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer
-    sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target /
-    hdiutil detach /Volumes/setup-installer
+        hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer
+        sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target /
+        hdiutil detach /Volumes/setup-installer
+    } elseif ($null -ne (Get-Member -InputObject $_ -Name 'PkgUrl')) {
+        $pathToPkg = "~/Downloads/$($_.Name).pkg"
+        Get-RemoteFile -OutFile $pathToPkg -Uri $_.PkgUrl -Sha256 $_.Sha256
+
+        sudo installer -pkg $pathToPkg -target /
+    } else {
+        Write-Error "$($_.Name) does not have an installer in the configuration file."
+        throw
+    }
 }
 
 $Installables.Brew | ForEach-Object {
@@ -64,31 +88,45 @@ $Installables.Brew | ForEach-Object {
             default {
                 Write-Error "Invalid kind: $_. Expected either empty, or 'cask'."
             }
-         }
-     }
+        }
+    }
 }
+brew upgrade
 
-# Install plugins
-$installedExtensionPacks = Get-InstalledVirtualBoxExtensionPacks
-
-$Installables.VBoxExtensions | ForEach-Object {
-    $extension = $_
-    $installedExts = $installedExtensionPacks | Where-Object { $_.Pack -eq $extension.FullName -and $_.Usable -eq 'true' }
-
-    if ($null -eq $installedExts) {
-        Write-Host "VBox extension: $($extension.Name) not installed; installing now"
-    } elseif ($Force) {
-        Write-Host "VBox extension: $($extension.Name) found; attempting to upgrade or re-install"
+$installedVagrantPlugins = @{}
+vagrant plugin list --machine-readable | ForEach-Object {
+    $timestamp, $target, $type, $data = $_ -split ','
+    switch ($type) {
+        # these are not important
+        'ui' { }
+        'plugin-version-constraint' { }
+        'plugin-name' {
+            $installedVagrantPlugins[$data] = $Null
+        }
+        'plugin-version' {
+            $version = $data -replace '%!\(VAGRANT_COMMA\)',','
+            if ($version -notmatch '^(.*), global') {
+                Write-Error "Invalid version string for plugin ${target}: $version"
+                throw
+            }
+            $installedVagrantPlugins[$target] = $Matches[1]
+        }
+        default {
+            Write-Warning "Unknown plugin list member type $type for plugin $target"
+        }
+    }
+}
+$Installables.VagrantPlugins | ForEach-Object {
+    if (-not $installedVagrantPlugins.Contains($_.Name)) {
+        Write-Host "$($_.Name) not installed; installing now"
+    } elseif ($installedVagrantPlugins[$_.Name] -ne $_.Version) {
+        Write-Host "$($_.Name) already installed but with the incorrect version
+    installed version: '$($installedVagrantPlugins[$_.Name])'
+    required version:  '$($_.Version)'"
     } else {
-        Write-Host "VBox extension: $($extension.Name) already installed"
+        Write-Host "$($_.Name) already installed and at the correct version ($($_.Version))"
         return
     }
 
-    $pathToExt = "~/Downloads/$($extension.FullName -replace ' ','_').vbox-extpack"
-
-    Get-RemoteFile -OutFile $pathToExt -Uri $extension.Url -Sha256 $extension.Sha256 | Out-Null
-
-    Write-Host 'Attempting to install extension with sudo; you may need to enter your password'
-    sudo VBoxManage extpack install --replace $pathToExt
-    sudo VBoxManage extpack cleanup
+    vagrant plugin install $_.Name --plugin-version $_.Version
 }

scripts/azure-pipelines/osx/README.md

@@ -6,6 +6,10 @@
   - [Table of Contents](#table-of-contents)
   - [Basic Usage](#basic-usage)
   - [Setting up a new macOS machine](#setting-up-a-new-macos-machine)
+    - [Troubleshooting](#troubleshooting)
+  - [Creating a new Vagrant box](#creating-a-new-vagrant-box)
+    - [VM Software Versions](#vm-software-versions)
+    - [(Internal) Accessing the macOS fileshare](#internal-accessing-the-macos-fileshare)
 
 ## Basic Usage
 
@@ -70,7 +74,7 @@ for the physical machine; i.e., vcpkgmm-04 would use 04.
 $ ./Setup-VagrantMachines.ps1 \
   -MachineId XX \
   -DevopsPat '<get this from azure devops; it needs agent pool read and manage access>' \
-  -Date <this is the date of the pool; 2020-09-28 at time of writing>
+  -Date <this is the date of the pool; 2021-04-16 at time of writing>
 $ cd ~/vagrant/vcpkg-eg-mac
 $ vagrant up
 ```
@@ -89,6 +93,13 @@ $ sudo shutdown -r now
 
 and wait for the machine to start back up. Then, start again from where the error was emitted.
 
+### Troubleshooting
+
+The following are issues that we've run into:
+
+- (with a Parallels box) `vagrant up` doesn't work, and vagrant gives the error that the VM is `'stopped'`.
+  - Try logging into the GUI with the KVM, and retrying `vagrant up`.
+
 ## Creating a new Vagrant box
 
 Whenever we want to install updated versions of the command line tools,
@@ -100,74 +111,99 @@ you can set up your own vagrant boxes that are the same as ours by doing the fol
 
 You'll need some prerequisites:
 
-- macinbox - installable via `sudo gem install macinbox`
 - vagrant - found at <https://www.vagrantup.com/>
-- VirtualBox - found at <https://www.virtualbox.org/>
-- A macOS installer application - you can get this from the App Store (although I believe only the latest is available)
-- An Xcode Command Line Tools installer - you can get this from Apple's developer website,
+  - The vagrant-parallels plugin - `vagrant plugin install vagrant-parallels`
+- Parallels - found at <https://parallels.com>
+- An Xcode installer - you can get this from Apple's developer website,
   although you'll need to sign in first: <https://developer.apple.com/downloads>
 
-First, you'll need to create a base box;
+First, you'll need to create a base VM;
 this is where you determine what version of macOS is installed.
+Just follow the Parallels process for creating a macOS VM.
 
-```
-> sudo macinbox \
-  --box-format virtualbox \
-  --name macos-ci-base \
-  --installer <path to macOS installer> \
-  --no-gui
+Once you've done this, you can run through the installation of macOS onto a new VM.
+You should set the username to `vagrant`.
+
+Once it's finished installing, make sure to turn on the SSH server.
+Open System Preferences, then go to Sharing > Remote Login,
+and turn it on.
+You'll then want to add the vagrant SSH keys to the VM's vagrant user.
+Open the terminal application and run the following:
+
+```sh
+$ # basic stuff
+$ date | sudo tee '/etc/vagrant_box_build_time'
+$ printf 'vagrant\tALL=(ALL)\tNOPASSWD:\tALL\n' | sudo tee -a '/etc/sudoers.d/vagrant'
+$ sudo chmod 0440 '/etc/sudoers.d/vagrant'
+$ # then install vagrant keys
+$ mkdir -p ~/.ssh
+$ curl -fsSL 'https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub' >~/.ssh/authorized_keys
+$ chmod 0600 ~/.ssh/authorized_keys
 ```
 
-Once you've done that, create a Vagrantfile that looks like the following:
+Finally, you'll need to install the Parallel Tools.
+From your host, in the top bar,
+go to Actions > Install Parallels Tools...,
+and then follow the instructions.
 
-```rb
-Vagrant.configure('2') do |config|
-  config.vm.box = 'macos-ci-base'
-  config.vm.boot_timeout = 600
-  config.vm.synced_folder ".", "/vagrant", disabled: true
-end
+Now, let's package the VM into a base box.
+(The following instructions are adapted from
+[these official instructions][base-box-instructions]).
+
+Run the following commands:
+
+```sh
+$ cd ~/Parallels
+$ echo '{ "provider": "parallels" }' >metadata.json
+$ tar zgvf <current date>.box ./metadata.json ./<name of VM>.pvm
 ```
 
-then, run the following in that vagrant directory:
+This will create a box file which contains all the necessary data.
+You can delete the `metadata.json` file after.
+
+Once you've done that, you can upload it to the fileshare,
+under `share/boxes/vcpkg-ci-base`, add it to `share/boxes/vcpkg-ci-base.json`,
+and finally add it to vagrant:
+
+```sh
+$ vagrant box add ~/vagrant/share/boxes/vcpkg-ci-base.json
+```
+
+Then, we'll create the final box,
+which contains all the necessary programs for doing CI work.
+Copy `configuration/Vagrantfile-box.rb` as `Vagrantfile`, and
+`configuration/vagrant-box-configuration.json`
+into a new directory; into that same directory,
+download the Xcode command line tools dmg, and name it `clt.dmg`.
+Then, run the following in that directory:
 
 ```sh
 $ vagrant up
-$ vagrant scp <path to Command Line Tools for Xcode installer> :clt.dmg
-$ vagrant ssh -c 'hdiutil attach clt.dmg -mountpoint /Volumes/setup-installer'
-$ vagrant ssh -c 'sudo installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /'
-$ vagrant ssh -c 'hdiutil detach /Volumes/setup-installer'
-$ vagrant ssh -c 'rm clt.dmg'
-$ vagrant ssh -c '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"'
-$ vagrant reload
-```
-
-if that works, you can now package the box:
-
-```sh
-$ vagrant ssh -c 'umount testmnt && rmdir testmnt'
 $ vagrant package
 ```
 
 This will create a `package.box`, which is the box file for the base VM.
-Then, you can `vagrant box add <package.box> --name <name for the box>`,
-and you'll have the base vcpkg box added for purposes of `Setup-VagrantMachines.ps1`!
-
-Once you've created the base box, if you're making it the new base box for the CI,
-upload it to the fileshare, under `share/vcpkg-boxes`.
-Then, add the metadata about the box (the name and version) to the JSON file there.
+Once you've created this box, if you're making it the new box for the CI,
+upload it to the fileshare, under `share/boxes/vcpkg-ci`.
+Then, add the metadata about the box (the name and version) to
+`share/boxes/vcpkg-ci.json`.
 Once you've done that, add the software versions under [VM Software Versions](#vm-software-versions).
 
+[base-box-instructions]: https://parallels.github.io/vagrant-parallels/docs/boxes/base.html
+
 ### VM Software Versions
 
 * 2020-09-28:
   * macOS: 10.15.6
   * Xcode CLTs: 12
+* 2021-04-16:
+  * macOS: 11.2.3
+  * Xcode CLTs: 12.4
 
 ### (Internal) Accessing the macOS fileshare
 
 The fileshare is located on `vcpkgmm-01`, under the `fileshare` user, in the `share` directory.
 In order to get `sshfs` working on the physical machine,
-you'll need to do the same thing one needs to do for building the base box.
 You can run `Install-Prerequisites.ps1` to grab the right software, then either:
 
 ```sh

scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1

@@ -12,6 +12,8 @@ configuration JSON file into ~/vagrant/vcpkg-eg-mac.
 
 .PARAMETER MachineId
 The number to give the machine; should match [0-9]{2}.
+Defaults to the numbers at the end of the machine name,
+assuming that that machine name matches `VCPKGMM-[0-9]{2}`.
 
 .PARAMETER DevopsPat
 The personal access token which has Read & Manage permissions on the ADO pool.
@@ -36,12 +38,6 @@ Defaults to 'vcpkg-eg-mac'.
 The name of the box to use. Defaults to 'vcpkg/macos-ci',
 which is only available internally.
 
-.PARAMETER Force
-Delete any existing vagrant/vcpkg-eg-mac directory.
-
-.PARAMETER DiskSize
-The size to make the temporary disks in gigabytes. Defaults to 350.
-
 .INPUTS
 None
 
@@ -50,7 +46,7 @@ None
 #>
 [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='DefineDate')]
 Param(
-    [Parameter(Mandatory=$True)]
+    [Parameter(Mandatory=$False)]
     [String]$MachineId,
 
     [Parameter(Mandatory=$True)]
@@ -72,13 +68,7 @@ Param(
     [String]$BaseName = 'vcpkg-eg-mac',
 
     [Parameter()]
-    [String]$BoxName = 'vcpkg/macos-ci',
-
-    [Parameter()]
-    [Int]$DiskSize = 250,
-
-    [Parameter()]
-    [Switch]$Force
+    [String]$BoxName = 'vcpkg/macos-ci'
 )
 
 Set-StrictMode -Version 2
@@ -92,33 +82,43 @@ if (-not [String]::IsNullOrEmpty($Date)) {
     $AgentPool = "PrOsx-$Date"
 }
 
-if (Test-Path '~/vagrant/vcpkg-eg-mac') {
-    if ($Force) {
-        Write-Host 'Deleting existing directories'
-        Remove-Item -Recurse -Force -Path '~/vagrant/vcpkg-eg-mac' | Out-Null
+if ([String]::IsNullOrEmpty($MachineId)) {
+    $hostname = hostname -s
+    if ($hostname -match '^VCPKGMM-([0-9]{2})$') {
+        $MachineId = $matches[1]
     } else {
-        throw '~/vagrant/vcpkg-eg-mac already exists; try re-running with -Force'
+        Write-Error "Hostname ($hostname) does not match the expected format (VCPKGMM-NN). Please pass -MachineId in order to give the VM a number."
+    }
+}
+
+if (Test-Path '~/vagrant/vcpkg-eg-mac') {
+    Push-Location '~/vagrant/vcpkg-eg-mac'
+    try {
+        Write-Host 'Deleting existing directories'
+        vagrant destroy -f
+        Remove-Item -Recurse -Force -LiteralPath '~/vagrant/vcpkg-eg-mac' | Out-Null
+    } finally {
+        Pop-Location
     }
 }
 
 Write-Host 'Creating new directories'
 if (-not (Test-Path -Path '~/vagrant')) {
-	New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null
+    New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null
 }
 New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null
 
 Copy-Item `
-    -Path "$PSScriptRoot/configuration/Vagrantfile" `
+    -Path "$PSScriptRoot/configuration/Vagrantfile-vm.rb" `
     -Destination '~/vagrant/vcpkg-eg-mac/Vagrantfile'
 
 $configuration = @{
-    pat = $DevopsPat;
-    agent_pool = $AgentPool;
-    devops_url = $DevopsUrl;
-    machine_name = "${BaseName}-${MachineId}";
-    box_name = $BoxName;
-    box_version = $BoxVersion;
-    disk_size = $DiskSize;
+    pat = $DevopsPat
+    agent_pool = $AgentPool
+    devops_url = $DevopsUrl
+    machine_name = "${BaseName}-${MachineId}"
+    box_name = $BoxName
+    box_version = $BoxVersion
 }
 ConvertTo-Json -InputObject $configuration -Depth 5 `
     | Set-Content -Path '~/vagrant/vcpkg-eg-mac/vagrant-configuration.json'

scripts/azure-pipelines/osx/Utilities.psm1

@@ -88,64 +88,3 @@ Please make sure that the hash in the powershell file is correct.
     Get-Item $OutFile
 }
 
-<#
-.SYNOPSIS
-Gets the list of installed extensions as powershell objects.
-
-.DESCRIPTION
-Get-InstalledVirtualBoxExtensionPacks gets the installed extensions,
-returning objects that look like:
-
-{
-    Pack = 'Oracle VM VirtualBox Extension Pack';
-    Version = '6.1.10';
-    ...
-}
-
-.INPUTS
-None
-
-.OUTPUTS
-PSCustomObject
-    The list of VBox Extension objects that are installed.
-#>
-function Get-InstalledVirtualBoxExtensionPacks
-{
-    [CmdletBinding()]
-    [OutputType([PSCustomObject])]
-    Param()
-
-    $lines = VBoxManage list extpacks
-
-    $result = @()
-
-    $currentObject = $null
-    $currentKey = ""
-    $currentString = ""
-
-    $lines | ForEach-Object {
-        $Line = $_
-        if ($Line[0] -eq ' ') {
-            $currentString += "`n$($Line.Trim())"
-        } else {
-            if ($null -ne $currentObject) {
-                $currentObject.$currentKey = $currentString
-            }
-            $currentKey, $currentString = $Line -Split ':'
-            $currentString = $currentString.Trim()
-
-            if ($currentKey.StartsWith('Pack no')) {
-                $currentKey = 'Pack'
-                if ($null -ne $currentObject) {
-                    Write-Output ([PSCustomObject]$currentObject)
-                }
-                $currentObject = @{}
-            }
-        }
-    }
-
-    if ($null -ne $currentObject) {
-        $currentObject.$currentKey = $currentString
-        Write-Output ([PSCustomObject]$currentObject)
-    }
-}

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -14,7 +14,7 @@ jobs:
     value: /Users/vagrant/Data
   - name: VCPKG_DOWNLOADS
     value: /Users/vagrant/Data/downloads
-  - group: azblob-test-sas-group
+  - group: osx-2021-04-16
   - name: BINARY_SOURCE_STUB
     value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)"
 

scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb

@@ -0,0 +1,36 @@
+require 'json'
+
+configuration = JSON.parse(File.read("#{__dir__}/vagrant-box-configuration.json"))
+
+Vagrant.configure('2') do |config|
+  config.vm.box = 'vcpkg/macos-ci-base'
+  config.vm.box_version = configuration['box_version']
+  config.vm.synced_folder '.', '/Users/vagrant/shared'
+
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install Xcode Command Line Tools: attach dmg file',
+    inline: 'hdiutil attach shared/clt.dmg -mountpoint /Volumes/setup-installer',
+    privileged: false
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install Xcode Command Line Tools: run installer',
+    inline: 'installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /',
+    privileged: true
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install XCode Command Line Tools: detach dmg file',
+    inline: 'hdiutil detach /Volumes/setup-installer',
+    privileged: false
+
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install brew',
+    inline: '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"',
+    privileged: false
+  config.vm.provision 'shell',
+    run: 'once',
+    name: 'Install brew applications',
+    inline: "brew install #{configuration['brew'].join(' ')} && brew install --cask #{configuration['brew-cask'].join(' ')}",
+    privileged: false
+end

scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb

@@ -1,79 +1,35 @@
 require 'json'
 
-require "erb"
-include ERB::Util
-
 configuration = JSON.parse(File.read("#{__dir__}/vagrant-configuration.json"))
 
 server = {
   :hostname => configuration['machine_name'],
   :box => configuration['box_name'],
   :box_version => configuration['box_version'],
-  :disk_size => configuration['disk_size'],
   :ram => 12000,
-  :cpu => 5
+  :cpu => 11
 }
 
-brew_formulas = [
-  'autoconf',
-  'automake',
-  'bison',
-  'gfortran',
-  'gperf',
-  'libtool',
-  'meson',
-  'mono',
-  'nasm',
-  'pkg-config',
-  'yasm' ]
-
-brew_cask_formulas = [
-  'powershell' ]
-
-azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.179.0/vsts-agent-osx-x64-2.179.0.tar.gz'
+azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.185.1/vsts-agent-osx-x64-2.185.1.tar.gz'
 devops_url = configuration['devops_url']
 agent_pool = configuration['agent_pool']
 pat = configuration['pat']
 
 Vagrant.configure('2') do |config|
-  # give them extra time to boot up
-  config.vm.boot_timeout = 600
-
   config.vm.box = server[:box]
   config.vm.box_version = server[:box_version]
   config.vm.hostname = server[:hostname]
   config.vm.synced_folder '.', '/vagrant', disabled: true
 
-  diskname = "#{server[:hostname]}-data.vmdk"
-
-  # I don't like this, but as far as I can tell, it's the only way
-  # to do this. When vagrant finishes the `disk` feature, switch
-  # over to that -- Nicole Mazzuca
-  if (not File.exists? diskname) then
-    system "VBoxManage createmedium --filename #{diskname} --size #{1024 * server[:disk_size]} --variant Fixed"
-  end
-
-  config.vm.provider 'virtualbox' do |vb|
-    vb.memory = server[:ram]
-    vb.cpus = server[:cpu]
-    vb.customize ['modifyvm', :id, '--ioapic', 'on']
-    vb.customize ['storageattach', :id,
-      '--storagectl', 'SATA Controller',
-      '--port', '1', '--device', '0', '--type', 'hdd',
-      '--medium', "#{diskname}"
-    ]
+  config.vm.provider 'parallels' do |prl|
+    prl.memory = server[:ram]
+    prl.cpus = server[:cpu]
   end
 
   config.vm.provision 'shell',
     run: 'once',
-    name: 'Format and mount the data filesystem',
-    inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0',
-    privileged: true
-
-  config.vm.provision 'shell',
-    run: 'once',
-    name: 'Link the data filesystem to the home directory',
-    inline: "ln -s /Volumes/data ~/Data",
+    name: 'Create the data directory',
+    inline: "mkdir ~/Data",
     privileged: false
 
   config.vm.provision 'shell',
@@ -88,12 +44,6 @@ Vagrant.configure('2') do |config|
     inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz',
     privileged: false
 
-  config.vm.provision 'shell',
-    run: 'once',
-    name: 'Install brew applications',
-    inline: "brew install #{brew_formulas.join(' ')} && brew install --cask #{brew_cask_formulas.join(' ')}",
-    privileged: false
-
   config.vm.provision 'shell',
     run: 'once',
     name: 'Add VM to azure agent pool',

scripts/azure-pipelines/osx/configuration/installables.json

@@ -3,35 +3,38 @@
 
   "Applications": [
     {
-      "Name": "VirtualBox",
-      "TestCommand": "VBoxManage",
-      "DmgUrl": "https://download.virtualbox.org/virtualbox/6.1.16/VirtualBox-6.1.16-140961-OSX.dmg",
-      "Sha256": "D7DF0F05D9A9E7CBA50EA01DA264AC20948B1C9C0E0CCCD2D628085C9F434D45",
-      "InstallerPath": "VirtualBox.pkg"
+      "Name": "vagrant",
+      "VersionCommand": [ "vagrant", "-v" ],
+      "VersionRegex": "Vagrant (.*)",
+      "Version": "2.2.15",
+      "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.15/vagrant_2.2.15_x86_64.dmg",
+      "Sha256": "5C2B290C4FA2371E255C56B1E96DED3D0C838D54CB7F0E8E6CF154E9F206A20E",
+      "InstallerPath": "vagrant.pkg"
     },
     {
-      "Name": "vagrant",
-      "TestCommand": "vagrant",
-      "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.dmg",
-      "Sha256": "76B849B26E6D6187A7829212B05545D3B424E05F1BCD0F7163DA1E5117084FA6",
-      "InstallerPath": "vagrant.pkg"
+      "Name": "osxfuse",
+      "VersionCommand": [ "cat", "/Library/Filesystems/macfuse.fs/Contents/version.plist" ],
+      "VersionRegex": "<key>CFBundleVersion</key>[\\n\\t ]*<string>([0-9.]+)</string>",
+      "Version": "4.1.0",
+      "DmgUrl": "https://github.com/osxfuse/osxfuse/releases/download/macfuse-4.1.0/macfuse-4.1.0.dmg",
+      "Sha256": "3CB6A49406FD036C50EF1B4AD717A377F4DCF182811BDE172D69F1C289791085",
+      "InstallerPath": "Install macFUSE.pkg"
+    },
+    {
+      "Name": "sshfs",
+      "VersionCommand": [ "sshfs", "--version" ],
+      "VersionRegex": "SSHFS version [0-9.]* \\(OSXFUSE SSHFS (.*)\\)",
+      "Version": "2.5.0",
+      "PkgUrl": "https://github.com/osxfuse/sshfs/releases/download/osxfuse-sshfs-2.5.0/sshfs-2.5.0.pkg",
+      "Sha256": "F8F4F71814273EA42DBE6CD92199F7CFF418571FFD1B10C0608878D3472D2162"
     }
   ],
   "Brew": [
-    {
-      "Name": "osxfuse",
-      "Kind": "cask"
-    },
-    {
-      "Name": "sshfs"
-    }
   ],
-  "VBoxExtensions": [
+  "VagrantPlugins": [
     {
-      "Name": "Extension Pack",
-      "FullName": "Oracle VM VirtualBox Extension Pack",
-      "Url": "https://download.virtualbox.org/virtualbox/6.1.16/Oracle_VM_VirtualBox_Extension_Pack-6.1.16.vbox-extpack",
-      "Sha256": "9802482B77B95A954CB5111793DA10D009009A4E9A9C4EAA4BD1AE5DAFE9DB46"
+      "Name": "vagrant-parallels",
+      "Version": "2.2.1"
     }
   ]
 }

scripts/azure-pipelines/osx/configuration/installables.schema.json

@@ -10,7 +10,7 @@
   "required": [
     "Applications",
     "Brew",
-    "VBoxExtensions"
+    "VagrantPlugins"
   ],
   "properties": {
     "Applications": {
@@ -21,7 +21,18 @@
           "Name": {
             "type": "string"
           },
-          "TestCommand": {
+          "VersionCommand": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "minItems": 1
+          },
+          "VersionRegex": {
+            "type": "string",
+            "format": "regex"
+          },
+          "Version": {
             "type": "string"
           },
           "DmgUrl": {
@@ -53,23 +64,17 @@
         }
       }
     },
-    "VBoxExtensions": {
+    "VagrantPlugins": {
       "type": "array",
       "items": {
         "type": "object",
+        "required": [ "Name", "Version" ],
         "properties": {
           "Name": {
             "type": "string"
           },
-          "FullName": {
+          "Version": {
             "type": "string"
-          },
-          "Url": {
-            "type": "string",
-            "format": "uri"
-          },
-          "Sha256": {
-            "$ref": "#/definitions/sha256"
           }
         }
       }

scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "./vagrant-vm-configuration.schema.json",
+  "brew": [
+    "autoconf",
+    "automake",
+    "bison",
+    "gettext",
+    "gfortran",
+    "gperf",
+    "gtk-doc",
+    "libtool",
+    "meson",
+    "mono",
+    "nasm",
+    "pkg-config",
+    "yasm"
+  ],
+  "brew-cask": [
+    "powershell"
+  ]
+}

scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema",
+  "type": "object",
+  "required": [
+    "brew",
+    "brew-cask"
+  ],
+  "properties": {
+    "brew": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "brew-cask": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  }
+}

scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json

@@ -5,9 +5,11 @@
 
   "required": [
     "pat",
-    "base_name",
-    "disk_size",
-    "machine_identifiers"
+    "agent_pool",
+    "devops_url",
+    "machine_name",
+    "box_name",
+    "box_version"
   ],
 
   "properties": {
@@ -28,9 +30,6 @@
     },
     "box_version": {
       "type": "string"
-    },
-    "disk_size": {
-      "type": "integer"
     }
   }
 }

scripts/ci.baseline.txt

@@ -1834,3 +1834,6 @@ wangle:x64-windows-static-md=fail
 # See https://developercommunity.visualstudio.com/t/Toolset-169-regression-vcxproj-producin/1356639
 dimcli:x64-windows-static-md=fail
 dimcli:x64-windows-static=fail
+
+# cppgraphqlgen triggers an ICE on Apple Clang that comes with MacOS 11.
+cppgraphqlgen:x64-osx=fail

versions/baseline.json

@@ -5461,8 +5461,8 @@
       "port-version": 0
     },
     "sdformat6": {
-      "baseline": "6.2.0-1",
-      "port-version": 0
+      "baseline": "6.2.0",
+      "port-version": 2
     },
     "sdformat9": {
       "baseline": "9.4.0",

versions/s-/sdformat6.json

@@ -1,5 +1,10 @@
 {
   "versions": [
+    {
+      "git-tree": "9104c5c84ce079686624adeaa3032543415acda2",
+      "version": "6.2.0",
+      "port-version": 2
+    },
     {
       "git-tree": "5a9a49ae80212912ecf538b7646f233cdba24211",
       "version-string": "6.2.0-1",