From e6cabdece52f3ffc21cf1101b2c72275cac9dbba Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Thu, 29 Apr 2021 07:39:04 -0700 Subject: [PATCH] [ci] Update macOS to 11 (#17376) * start 2021-04-16 process Major new things: * update to macOS Big Sur (11.*) * switch from VirtualBox to Parallels due to ^ (and also ARM) Minor new things: * update from xcode CLT 12 to 12.4 This PR includes new stuff for creating the base box for parallels. Still to do: using the new box type. * update the vagrantfile stuff * link the CI to the new version * modify how macOS boxes are made the Vagrantfile for the final VM will only download the azure pipeline stuff this allows us to keep the versions of brew applications stable * fix cpus and memory * add vagrant plugins to installables * Skip cppgraphqlgen ICE. * [sdformat6] Remove unneeded include(FindBoost) which causes problems when the system cmake version doesn't match the one deployed by vcpkg. * switch to dmg for installing osxfuse/sshfs * Set cores to 11 (leaving 1 thread for host) Co-authored-by: Billy Robert ONeal III --- ports/sdformat6/CONTROL | 6 - .../disable-unneeded-include-findboost.patch | 12 ++ ports/sdformat6/portfile.cmake | 2 + ports/sdformat6/vcpkg.json | 15 +++ scripts/azure-pipelines/azure-pipelines.yml | 2 +- .../osx/Install-Prerequisites.ps1 | 116 ++++++++++++------ scripts/azure-pipelines/osx/README.md | 116 ++++++++++++------ .../osx/Setup-VagrantMachines.ps1 | 56 ++++----- scripts/azure-pipelines/osx/Utilities.psm1 | 61 --------- .../azure-pipelines/osx/azure-pipelines.yml | 2 +- .../osx/configuration/Vagrantfile-box.rb | 36 ++++++ .../{Vagrantfile => Vagrantfile-vm.rb} | 64 ++-------- .../osx/configuration/installables.json | 47 +++---- .../configuration/installables.schema.json | 27 ++-- .../vagrant-box-configuration.json | 21 ++++ .../vagrant-box-configuration.schema.json | 18 +++ .../vagrant-configuration.schema.json | 11 +- scripts/ci.baseline.txt | 3 + versions/baseline.json | 4 +- versions/s-/sdformat6.json | 5 + 20 files changed, 350 insertions(+), 274 deletions(-) delete mode 100644 ports/sdformat6/CONTROL create mode 100644 ports/sdformat6/disable-unneeded-include-findboost.patch create mode 100644 ports/sdformat6/vcpkg.json create mode 100644 scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb rename scripts/azure-pipelines/osx/configuration/{Vagrantfile => Vagrantfile-vm.rb} (51%) create mode 100644 scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json create mode 100644 scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json diff --git a/ports/sdformat6/CONTROL b/ports/sdformat6/CONTROL deleted file mode 100644 index 8f1e5ed585..0000000000 --- a/ports/sdformat6/CONTROL +++ /dev/null @@ -1,6 +0,0 @@ -Source: sdformat6 -Version: 6.2.0-1 -Homepage: http://sdformat.org/ -Build-Depends: boost-any, boost-variant, ignition-math4, urdfdom, tinyxml -Description: Simulation Description Format (SDF) parser and description files. -Supports: !(arm|uwp) \ No newline at end of file diff --git a/ports/sdformat6/disable-unneeded-include-findboost.patch b/ports/sdformat6/disable-unneeded-include-findboost.patch new file mode 100644 index 0000000000..6a92202927 --- /dev/null +++ b/ports/sdformat6/disable-unneeded-include-findboost.patch @@ -0,0 +1,12 @@ +diff --git a/cmake/SearchForStuff.cmake b/cmake/SearchForStuff.cmake +index c2a0ee4..2735a07 100644 +--- a/cmake/SearchForStuff.cmake ++++ b/cmake/SearchForStuff.cmake +@@ -13,7 +13,6 @@ if (WIN32) + set(Boost_USE_STATIC_RUNTIME OFF) + endif() + +-include(FindBoost) + find_package(Boost ${MIN_BOOST_VERSION}) + + if (NOT Boost_FOUND) diff --git a/ports/sdformat6/portfile.cmake b/ports/sdformat6/portfile.cmake index e5e1c04f6e..1b6babd87a 100644 --- a/ports/sdformat6/portfile.cmake +++ b/ports/sdformat6/portfile.cmake @@ -6,6 +6,8 @@ vcpkg_from_github( REF sdformat6_6.2.0 SHA512 3d139ec4b4c9fbfd547ed8bfca0adb5cdca92c1b7cc4d4b554a7c51ccf755b9079c26a006ebfedc5bc5b1ba5e16ad950bb38c47ea97bf97e59a2fd7d12d60620 HEAD_REF sdf6 + PATCHES + disable-unneeded-include-findboost.patch ) # Ruby is required by the sdformat build process diff --git a/ports/sdformat6/vcpkg.json b/ports/sdformat6/vcpkg.json new file mode 100644 index 0000000000..84c6969921 --- /dev/null +++ b/ports/sdformat6/vcpkg.json @@ -0,0 +1,15 @@ +{ + "name": "sdformat6", + "version": "6.2.0", + "port-version": 2, + "description": "Simulation Description Format (SDF) parser and description files.", + "homepage": "http://sdformat.org/", + "supports": "!(arm | uwp)", + "dependencies": [ + "boost-any", + "boost-variant", + "ignition-math4", + "tinyxml", + "urdfdom" + ] +} diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 9f6526cb79..5e667b2720 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -4,7 +4,7 @@ variables: windows-pool: 'PrWin-2021-04-23' linux-pool: 'PrLin-2021-04-25' - osx-pool: 'PrOsx-2020-09-28' + osx-pool: 'PrOsx-2021-04-16' stages: - stage: FormatChecks diff --git a/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 index cb82f83b0d..1202025834 100755 --- a/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 +++ b/scripts/azure-pipelines/osx/Install-Prerequisites.ps1 @@ -8,10 +8,7 @@ Installs the set of prerequisites for the macOS CI hosts. .DESCRIPTION Install-Prerequisites.ps1 installs all of the necessary prerequisites to run the vcpkg macOS CI in a vagrant virtual machine, -skipping all prerequisites that are already installed. - -.PARAMETER Force -Don't skip the prerequisites that are already installed. +skipping all prerequisites that are already installed and of the right version. .INPUTS None @@ -20,10 +17,7 @@ None None #> [CmdletBinding()] -Param( - [Parameter()] - [Switch]$Force -) +Param() Set-StrictMode -Version 2 @@ -37,21 +31,51 @@ Import-Module "$PSScriptRoot/Utilities.psm1" $Installables = Get-Content "$PSScriptRoot/configuration/installables.json" | ConvertFrom-Json $Installables.Applications | ForEach-Object { - if (-not (Get-CommandExists $_.TestCommand)) { + $VersionCommand = $_.VersionCommand + $InstalledVersion = (& $VersionCommand[0] $VersionCommand[1..$VersionCommand.Length]) + if (-not $?) { Write-Host "$($_.Name) not installed; installing now" - } elseif ($Force) { - Write-Host "$($_.Name) found; attempting to upgrade or re-install" } else { - Write-Host "$($_.Name) already installed" - return + $InstalledVersion = $InstalledVersion -join "`n" + if ($InstalledVersion -match $_.VersionRegex) { + if ($Matches.Count -ne 2) { + Write-Error "$($_.Name) has a malformed version regex ($($_.VersionRegex)); it should have a single capture group + (it has $($Matches.Count - 1))" + throw + } + if ($Matches[1] -eq $_.Version) { + Write-Host "$($_.Name) already installed and at the correct version ($($Matches[1]))" + return + } else { + Write-Host "$($_.Name) already installed but with the incorrect version + installed version: '$($Matches[1])' + required version : '$($_.Version)' +upgrading now." + } + } else { + Write-Warning "$($_.Name)'s version command ($($VersionCommand -join ' ')) returned a value we did not expect: + $InstalledVersion + expected a version matching the regex: $($_.VersionRegex) +Installing anyways." + } } - $pathToDmg = "~/Downloads/$($_.Name).dmg" - Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256 + if ($null -ne (Get-Member -InputObject $_ -Name 'DmgUrl')) { + $pathToDmg = "~/Downloads/$($_.Name).dmg" + Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256 - hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer - sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target / - hdiutil detach /Volumes/setup-installer + hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer + sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target / + hdiutil detach /Volumes/setup-installer + } elseif ($null -ne (Get-Member -InputObject $_ -Name 'PkgUrl')) { + $pathToPkg = "~/Downloads/$($_.Name).pkg" + Get-RemoteFile -OutFile $pathToPkg -Uri $_.PkgUrl -Sha256 $_.Sha256 + + sudo installer -pkg $pathToPkg -target / + } else { + Write-Error "$($_.Name) does not have an installer in the configuration file." + throw + } } $Installables.Brew | ForEach-Object { @@ -64,31 +88,45 @@ $Installables.Brew | ForEach-Object { default { Write-Error "Invalid kind: $_. Expected either empty, or 'cask'." } - } - } + } + } } +brew upgrade -# Install plugins -$installedExtensionPacks = Get-InstalledVirtualBoxExtensionPacks - -$Installables.VBoxExtensions | ForEach-Object { - $extension = $_ - $installedExts = $installedExtensionPacks | Where-Object { $_.Pack -eq $extension.FullName -and $_.Usable -eq 'true' } - - if ($null -eq $installedExts) { - Write-Host "VBox extension: $($extension.Name) not installed; installing now" - } elseif ($Force) { - Write-Host "VBox extension: $($extension.Name) found; attempting to upgrade or re-install" +$installedVagrantPlugins = @{} +vagrant plugin list --machine-readable | ForEach-Object { + $timestamp, $target, $type, $data = $_ -split ',' + switch ($type) { + # these are not important + 'ui' { } + 'plugin-version-constraint' { } + 'plugin-name' { + $installedVagrantPlugins[$data] = $Null + } + 'plugin-version' { + $version = $data -replace '%!\(VAGRANT_COMMA\)',',' + if ($version -notmatch '^(.*), global') { + Write-Error "Invalid version string for plugin ${target}: $version" + throw + } + $installedVagrantPlugins[$target] = $Matches[1] + } + default { + Write-Warning "Unknown plugin list member type $type for plugin $target" + } + } +} +$Installables.VagrantPlugins | ForEach-Object { + if (-not $installedVagrantPlugins.Contains($_.Name)) { + Write-Host "$($_.Name) not installed; installing now" + } elseif ($installedVagrantPlugins[$_.Name] -ne $_.Version) { + Write-Host "$($_.Name) already installed but with the incorrect version + installed version: '$($installedVagrantPlugins[$_.Name])' + required version: '$($_.Version)'" } else { - Write-Host "VBox extension: $($extension.Name) already installed" + Write-Host "$($_.Name) already installed and at the correct version ($($_.Version))" return } - $pathToExt = "~/Downloads/$($extension.FullName -replace ' ','_').vbox-extpack" - - Get-RemoteFile -OutFile $pathToExt -Uri $extension.Url -Sha256 $extension.Sha256 | Out-Null - - Write-Host 'Attempting to install extension with sudo; you may need to enter your password' - sudo VBoxManage extpack install --replace $pathToExt - sudo VBoxManage extpack cleanup + vagrant plugin install $_.Name --plugin-version $_.Version } diff --git a/scripts/azure-pipelines/osx/README.md b/scripts/azure-pipelines/osx/README.md index ec88b09c75..9f253a06c2 100644 --- a/scripts/azure-pipelines/osx/README.md +++ b/scripts/azure-pipelines/osx/README.md @@ -6,6 +6,10 @@ - [Table of Contents](#table-of-contents) - [Basic Usage](#basic-usage) - [Setting up a new macOS machine](#setting-up-a-new-macos-machine) + - [Troubleshooting](#troubleshooting) + - [Creating a new Vagrant box](#creating-a-new-vagrant-box) + - [VM Software Versions](#vm-software-versions) + - [(Internal) Accessing the macOS fileshare](#internal-accessing-the-macos-fileshare) ## Basic Usage @@ -70,7 +74,7 @@ for the physical machine; i.e., vcpkgmm-04 would use 04. $ ./Setup-VagrantMachines.ps1 \ -MachineId XX \ -DevopsPat '' \ - -Date + -Date $ cd ~/vagrant/vcpkg-eg-mac $ vagrant up ``` @@ -89,6 +93,13 @@ $ sudo shutdown -r now and wait for the machine to start back up. Then, start again from where the error was emitted. +### Troubleshooting + +The following are issues that we've run into: + +- (with a Parallels box) `vagrant up` doesn't work, and vagrant gives the error that the VM is `'stopped'`. + - Try logging into the GUI with the KVM, and retrying `vagrant up`. + ## Creating a new Vagrant box Whenever we want to install updated versions of the command line tools, @@ -100,74 +111,99 @@ you can set up your own vagrant boxes that are the same as ours by doing the fol You'll need some prerequisites: -- macinbox - installable via `sudo gem install macinbox` - vagrant - found at -- VirtualBox - found at -- A macOS installer application - you can get this from the App Store (although I believe only the latest is available) -- An Xcode Command Line Tools installer - you can get this from Apple's developer website, + - The vagrant-parallels plugin - `vagrant plugin install vagrant-parallels` +- Parallels - found at +- An Xcode installer - you can get this from Apple's developer website, although you'll need to sign in first: -First, you'll need to create a base box; +First, you'll need to create a base VM; this is where you determine what version of macOS is installed. +Just follow the Parallels process for creating a macOS VM. -``` -> sudo macinbox \ - --box-format virtualbox \ - --name macos-ci-base \ - --installer \ - --no-gui +Once you've done this, you can run through the installation of macOS onto a new VM. +You should set the username to `vagrant`. + +Once it's finished installing, make sure to turn on the SSH server. +Open System Preferences, then go to Sharing > Remote Login, +and turn it on. +You'll then want to add the vagrant SSH keys to the VM's vagrant user. +Open the terminal application and run the following: + +```sh +$ # basic stuff +$ date | sudo tee '/etc/vagrant_box_build_time' +$ printf 'vagrant\tALL=(ALL)\tNOPASSWD:\tALL\n' | sudo tee -a '/etc/sudoers.d/vagrant' +$ sudo chmod 0440 '/etc/sudoers.d/vagrant' +$ # then install vagrant keys +$ mkdir -p ~/.ssh +$ curl -fsSL 'https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub' >~/.ssh/authorized_keys +$ chmod 0600 ~/.ssh/authorized_keys ``` -Once you've done that, create a Vagrantfile that looks like the following: +Finally, you'll need to install the Parallel Tools. +From your host, in the top bar, +go to Actions > Install Parallels Tools..., +and then follow the instructions. -```rb -Vagrant.configure('2') do |config| - config.vm.box = 'macos-ci-base' - config.vm.boot_timeout = 600 - config.vm.synced_folder ".", "/vagrant", disabled: true -end +Now, let's package the VM into a base box. +(The following instructions are adapted from +[these official instructions][base-box-instructions]). + +Run the following commands: + +```sh +$ cd ~/Parallels +$ echo '{ "provider": "parallels" }' >metadata.json +$ tar zgvf .box ./metadata.json ./.pvm ``` -then, run the following in that vagrant directory: +This will create a box file which contains all the necessary data. +You can delete the `metadata.json` file after. + +Once you've done that, you can upload it to the fileshare, +under `share/boxes/vcpkg-ci-base`, add it to `share/boxes/vcpkg-ci-base.json`, +and finally add it to vagrant: + +```sh +$ vagrant box add ~/vagrant/share/boxes/vcpkg-ci-base.json +``` + +Then, we'll create the final box, +which contains all the necessary programs for doing CI work. +Copy `configuration/Vagrantfile-box.rb` as `Vagrantfile`, and +`configuration/vagrant-box-configuration.json` +into a new directory; into that same directory, +download the Xcode command line tools dmg, and name it `clt.dmg`. +Then, run the following in that directory: ```sh $ vagrant up -$ vagrant scp :clt.dmg -$ vagrant ssh -c 'hdiutil attach clt.dmg -mountpoint /Volumes/setup-installer' -$ vagrant ssh -c 'sudo installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /' -$ vagrant ssh -c 'hdiutil detach /Volumes/setup-installer' -$ vagrant ssh -c 'rm clt.dmg' -$ vagrant ssh -c '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"' -$ vagrant reload -``` - -if that works, you can now package the box: - -```sh -$ vagrant ssh -c 'umount testmnt && rmdir testmnt' $ vagrant package ``` This will create a `package.box`, which is the box file for the base VM. -Then, you can `vagrant box add --name `, -and you'll have the base vcpkg box added for purposes of `Setup-VagrantMachines.ps1`! - -Once you've created the base box, if you're making it the new base box for the CI, -upload it to the fileshare, under `share/vcpkg-boxes`. -Then, add the metadata about the box (the name and version) to the JSON file there. +Once you've created this box, if you're making it the new box for the CI, +upload it to the fileshare, under `share/boxes/vcpkg-ci`. +Then, add the metadata about the box (the name and version) to +`share/boxes/vcpkg-ci.json`. Once you've done that, add the software versions under [VM Software Versions](#vm-software-versions). +[base-box-instructions]: https://parallels.github.io/vagrant-parallels/docs/boxes/base.html + ### VM Software Versions * 2020-09-28: * macOS: 10.15.6 * Xcode CLTs: 12 +* 2021-04-16: + * macOS: 11.2.3 + * Xcode CLTs: 12.4 ### (Internal) Accessing the macOS fileshare The fileshare is located on `vcpkgmm-01`, under the `fileshare` user, in the `share` directory. In order to get `sshfs` working on the physical machine, -you'll need to do the same thing one needs to do for building the base box. You can run `Install-Prerequisites.ps1` to grab the right software, then either: ```sh diff --git a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 index c448580441..88b64594ad 100755 --- a/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 +++ b/scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1 @@ -12,6 +12,8 @@ configuration JSON file into ~/vagrant/vcpkg-eg-mac. .PARAMETER MachineId The number to give the machine; should match [0-9]{2}. +Defaults to the numbers at the end of the machine name, +assuming that that machine name matches `VCPKGMM-[0-9]{2}`. .PARAMETER DevopsPat The personal access token which has Read & Manage permissions on the ADO pool. @@ -36,12 +38,6 @@ Defaults to 'vcpkg-eg-mac'. The name of the box to use. Defaults to 'vcpkg/macos-ci', which is only available internally. -.PARAMETER Force -Delete any existing vagrant/vcpkg-eg-mac directory. - -.PARAMETER DiskSize -The size to make the temporary disks in gigabytes. Defaults to 350. - .INPUTS None @@ -50,7 +46,7 @@ None #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='DefineDate')] Param( - [Parameter(Mandatory=$True)] + [Parameter(Mandatory=$False)] [String]$MachineId, [Parameter(Mandatory=$True)] @@ -72,13 +68,7 @@ Param( [String]$BaseName = 'vcpkg-eg-mac', [Parameter()] - [String]$BoxName = 'vcpkg/macos-ci', - - [Parameter()] - [Int]$DiskSize = 250, - - [Parameter()] - [Switch]$Force + [String]$BoxName = 'vcpkg/macos-ci' ) Set-StrictMode -Version 2 @@ -92,33 +82,43 @@ if (-not [String]::IsNullOrEmpty($Date)) { $AgentPool = "PrOsx-$Date" } -if (Test-Path '~/vagrant/vcpkg-eg-mac') { - if ($Force) { - Write-Host 'Deleting existing directories' - Remove-Item -Recurse -Force -Path '~/vagrant/vcpkg-eg-mac' | Out-Null +if ([String]::IsNullOrEmpty($MachineId)) { + $hostname = hostname -s + if ($hostname -match '^VCPKGMM-([0-9]{2})$') { + $MachineId = $matches[1] } else { - throw '~/vagrant/vcpkg-eg-mac already exists; try re-running with -Force' + Write-Error "Hostname ($hostname) does not match the expected format (VCPKGMM-NN). Please pass -MachineId in order to give the VM a number." + } +} + +if (Test-Path '~/vagrant/vcpkg-eg-mac') { + Push-Location '~/vagrant/vcpkg-eg-mac' + try { + Write-Host 'Deleting existing directories' + vagrant destroy -f + Remove-Item -Recurse -Force -LiteralPath '~/vagrant/vcpkg-eg-mac' | Out-Null + } finally { + Pop-Location } } Write-Host 'Creating new directories' if (-not (Test-Path -Path '~/vagrant')) { - New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null + New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null } New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null Copy-Item ` - -Path "$PSScriptRoot/configuration/Vagrantfile" ` + -Path "$PSScriptRoot/configuration/Vagrantfile-vm.rb" ` -Destination '~/vagrant/vcpkg-eg-mac/Vagrantfile' $configuration = @{ - pat = $DevopsPat; - agent_pool = $AgentPool; - devops_url = $DevopsUrl; - machine_name = "${BaseName}-${MachineId}"; - box_name = $BoxName; - box_version = $BoxVersion; - disk_size = $DiskSize; + pat = $DevopsPat + agent_pool = $AgentPool + devops_url = $DevopsUrl + machine_name = "${BaseName}-${MachineId}" + box_name = $BoxName + box_version = $BoxVersion } ConvertTo-Json -InputObject $configuration -Depth 5 ` | Set-Content -Path '~/vagrant/vcpkg-eg-mac/vagrant-configuration.json' diff --git a/scripts/azure-pipelines/osx/Utilities.psm1 b/scripts/azure-pipelines/osx/Utilities.psm1 index b7ad489bb7..1b70d61f7a 100644 --- a/scripts/azure-pipelines/osx/Utilities.psm1 +++ b/scripts/azure-pipelines/osx/Utilities.psm1 @@ -88,64 +88,3 @@ Please make sure that the hash in the powershell file is correct. Get-Item $OutFile } -<# -.SYNOPSIS -Gets the list of installed extensions as powershell objects. - -.DESCRIPTION -Get-InstalledVirtualBoxExtensionPacks gets the installed extensions, -returning objects that look like: - -{ - Pack = 'Oracle VM VirtualBox Extension Pack'; - Version = '6.1.10'; - ... -} - -.INPUTS -None - -.OUTPUTS -PSCustomObject - The list of VBox Extension objects that are installed. -#> -function Get-InstalledVirtualBoxExtensionPacks -{ - [CmdletBinding()] - [OutputType([PSCustomObject])] - Param() - - $lines = VBoxManage list extpacks - - $result = @() - - $currentObject = $null - $currentKey = "" - $currentString = "" - - $lines | ForEach-Object { - $Line = $_ - if ($Line[0] -eq ' ') { - $currentString += "`n$($Line.Trim())" - } else { - if ($null -ne $currentObject) { - $currentObject.$currentKey = $currentString - } - $currentKey, $currentString = $Line -Split ':' - $currentString = $currentString.Trim() - - if ($currentKey.StartsWith('Pack no')) { - $currentKey = 'Pack' - if ($null -ne $currentObject) { - Write-Output ([PSCustomObject]$currentObject) - } - $currentObject = @{} - } - } - } - - if ($null -ne $currentObject) { - $currentObject.$currentKey = $currentString - Write-Output ([PSCustomObject]$currentObject) - } -} diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 52e974c933..6f327c1e9a 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -14,7 +14,7 @@ jobs: value: /Users/vagrant/Data - name: VCPKG_DOWNLOADS value: /Users/vagrant/Data/downloads - - group: azblob-test-sas-group + - group: osx-2021-04-16 - name: BINARY_SOURCE_STUB value: "x-azblob,$(azblob-root-url),$(azblob-test-sas)" diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb b/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb new file mode 100644 index 0000000000..90ad6c4c55 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile-box.rb @@ -0,0 +1,36 @@ +require 'json' + +configuration = JSON.parse(File.read("#{__dir__}/vagrant-box-configuration.json")) + +Vagrant.configure('2') do |config| + config.vm.box = 'vcpkg/macos-ci-base' + config.vm.box_version = configuration['box_version'] + config.vm.synced_folder '.', '/Users/vagrant/shared' + + config.vm.provision 'shell', + run: 'once', + name: 'Install Xcode Command Line Tools: attach dmg file', + inline: 'hdiutil attach shared/clt.dmg -mountpoint /Volumes/setup-installer', + privileged: false + config.vm.provision 'shell', + run: 'once', + name: 'Install Xcode Command Line Tools: run installer', + inline: 'installer -pkg "/Volumes/setup-installer/Command Line Tools.pkg" -target /', + privileged: true + config.vm.provision 'shell', + run: 'once', + name: 'Install XCode Command Line Tools: detach dmg file', + inline: 'hdiutil detach /Volumes/setup-installer', + privileged: false + + config.vm.provision 'shell', + run: 'once', + name: 'Install brew', + inline: '/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"', + privileged: false + config.vm.provision 'shell', + run: 'once', + name: 'Install brew applications', + inline: "brew install #{configuration['brew'].join(' ')} && brew install --cask #{configuration['brew-cask'].join(' ')}", + privileged: false +end diff --git a/scripts/azure-pipelines/osx/configuration/Vagrantfile b/scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb similarity index 51% rename from scripts/azure-pipelines/osx/configuration/Vagrantfile rename to scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb index 08c472fe6d..ed689ab7be 100644 --- a/scripts/azure-pipelines/osx/configuration/Vagrantfile +++ b/scripts/azure-pipelines/osx/configuration/Vagrantfile-vm.rb @@ -1,79 +1,35 @@ require 'json' -require "erb" -include ERB::Util - configuration = JSON.parse(File.read("#{__dir__}/vagrant-configuration.json")) server = { :hostname => configuration['machine_name'], :box => configuration['box_name'], :box_version => configuration['box_version'], - :disk_size => configuration['disk_size'], :ram => 12000, - :cpu => 5 + :cpu => 11 } -brew_formulas = [ - 'autoconf', - 'automake', - 'bison', - 'gfortran', - 'gperf', - 'libtool', - 'meson', - 'mono', - 'nasm', - 'pkg-config', - 'yasm' ] - -brew_cask_formulas = [ - 'powershell' ] - -azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.179.0/vsts-agent-osx-x64-2.179.0.tar.gz' +azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.185.1/vsts-agent-osx-x64-2.185.1.tar.gz' devops_url = configuration['devops_url'] agent_pool = configuration['agent_pool'] pat = configuration['pat'] Vagrant.configure('2') do |config| - # give them extra time to boot up - config.vm.boot_timeout = 600 - config.vm.box = server[:box] config.vm.box_version = server[:box_version] config.vm.hostname = server[:hostname] config.vm.synced_folder '.', '/vagrant', disabled: true - diskname = "#{server[:hostname]}-data.vmdk" - - # I don't like this, but as far as I can tell, it's the only way - # to do this. When vagrant finishes the `disk` feature, switch - # over to that -- Nicole Mazzuca - if (not File.exists? diskname) then - system "VBoxManage createmedium --filename #{diskname} --size #{1024 * server[:disk_size]} --variant Fixed" - end - - config.vm.provider 'virtualbox' do |vb| - vb.memory = server[:ram] - vb.cpus = server[:cpu] - vb.customize ['modifyvm', :id, '--ioapic', 'on'] - vb.customize ['storageattach', :id, - '--storagectl', 'SATA Controller', - '--port', '1', '--device', '0', '--type', 'hdd', - '--medium', "#{diskname}" - ] + config.vm.provider 'parallels' do |prl| + prl.memory = server[:ram] + prl.cpus = server[:cpu] end config.vm.provision 'shell', run: 'once', - name: 'Format and mount the data filesystem', - inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0', - privileged: true - - config.vm.provision 'shell', - run: 'once', - name: 'Link the data filesystem to the home directory', - inline: "ln -s /Volumes/data ~/Data", + name: 'Create the data directory', + inline: "mkdir ~/Data", privileged: false config.vm.provision 'shell', @@ -88,12 +44,6 @@ Vagrant.configure('2') do |config| inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz', privileged: false - config.vm.provision 'shell', - run: 'once', - name: 'Install brew applications', - inline: "brew install #{brew_formulas.join(' ')} && brew install --cask #{brew_cask_formulas.join(' ')}", - privileged: false - config.vm.provision 'shell', run: 'once', name: 'Add VM to azure agent pool', diff --git a/scripts/azure-pipelines/osx/configuration/installables.json b/scripts/azure-pipelines/osx/configuration/installables.json index 7a7ba400c3..1b2c2d015b 100644 --- a/scripts/azure-pipelines/osx/configuration/installables.json +++ b/scripts/azure-pipelines/osx/configuration/installables.json @@ -3,35 +3,38 @@ "Applications": [ { - "Name": "VirtualBox", - "TestCommand": "VBoxManage", - "DmgUrl": "https://download.virtualbox.org/virtualbox/6.1.16/VirtualBox-6.1.16-140961-OSX.dmg", - "Sha256": "D7DF0F05D9A9E7CBA50EA01DA264AC20948B1C9C0E0CCCD2D628085C9F434D45", - "InstallerPath": "VirtualBox.pkg" + "Name": "vagrant", + "VersionCommand": [ "vagrant", "-v" ], + "VersionRegex": "Vagrant (.*)", + "Version": "2.2.15", + "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.15/vagrant_2.2.15_x86_64.dmg", + "Sha256": "5C2B290C4FA2371E255C56B1E96DED3D0C838D54CB7F0E8E6CF154E9F206A20E", + "InstallerPath": "vagrant.pkg" }, { - "Name": "vagrant", - "TestCommand": "vagrant", - "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.14/vagrant_2.2.14_x86_64.dmg", - "Sha256": "76B849B26E6D6187A7829212B05545D3B424E05F1BCD0F7163DA1E5117084FA6", - "InstallerPath": "vagrant.pkg" + "Name": "osxfuse", + "VersionCommand": [ "cat", "/Library/Filesystems/macfuse.fs/Contents/version.plist" ], + "VersionRegex": "CFBundleVersion[\\n\\t ]*([0-9.]+)", + "Version": "4.1.0", + "DmgUrl": "https://github.com/osxfuse/osxfuse/releases/download/macfuse-4.1.0/macfuse-4.1.0.dmg", + "Sha256": "3CB6A49406FD036C50EF1B4AD717A377F4DCF182811BDE172D69F1C289791085", + "InstallerPath": "Install macFUSE.pkg" + }, + { + "Name": "sshfs", + "VersionCommand": [ "sshfs", "--version" ], + "VersionRegex": "SSHFS version [0-9.]* \\(OSXFUSE SSHFS (.*)\\)", + "Version": "2.5.0", + "PkgUrl": "https://github.com/osxfuse/sshfs/releases/download/osxfuse-sshfs-2.5.0/sshfs-2.5.0.pkg", + "Sha256": "F8F4F71814273EA42DBE6CD92199F7CFF418571FFD1B10C0608878D3472D2162" } ], "Brew": [ - { - "Name": "osxfuse", - "Kind": "cask" - }, - { - "Name": "sshfs" - } ], - "VBoxExtensions": [ + "VagrantPlugins": [ { - "Name": "Extension Pack", - "FullName": "Oracle VM VirtualBox Extension Pack", - "Url": "https://download.virtualbox.org/virtualbox/6.1.16/Oracle_VM_VirtualBox_Extension_Pack-6.1.16.vbox-extpack", - "Sha256": "9802482B77B95A954CB5111793DA10D009009A4E9A9C4EAA4BD1AE5DAFE9DB46" + "Name": "vagrant-parallels", + "Version": "2.2.1" } ] } diff --git a/scripts/azure-pipelines/osx/configuration/installables.schema.json b/scripts/azure-pipelines/osx/configuration/installables.schema.json index 13476e7764..b7ec7ae801 100644 --- a/scripts/azure-pipelines/osx/configuration/installables.schema.json +++ b/scripts/azure-pipelines/osx/configuration/installables.schema.json @@ -10,7 +10,7 @@ "required": [ "Applications", "Brew", - "VBoxExtensions" + "VagrantPlugins" ], "properties": { "Applications": { @@ -21,7 +21,18 @@ "Name": { "type": "string" }, - "TestCommand": { + "VersionCommand": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "VersionRegex": { + "type": "string", + "format": "regex" + }, + "Version": { "type": "string" }, "DmgUrl": { @@ -53,23 +64,17 @@ } } }, - "VBoxExtensions": { + "VagrantPlugins": { "type": "array", "items": { "type": "object", + "required": [ "Name", "Version" ], "properties": { "Name": { "type": "string" }, - "FullName": { + "Version": { "type": "string" - }, - "Url": { - "type": "string", - "format": "uri" - }, - "Sha256": { - "$ref": "#/definitions/sha256" } } } diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json new file mode 100644 index 0000000000..d800c257aa --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.json @@ -0,0 +1,21 @@ +{ + "$schema": "./vagrant-vm-configuration.schema.json", + "brew": [ + "autoconf", + "automake", + "bison", + "gettext", + "gfortran", + "gperf", + "gtk-doc", + "libtool", + "meson", + "mono", + "nasm", + "pkg-config", + "yasm" + ], + "brew-cask": [ + "powershell" + ] +} diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json new file mode 100644 index 0000000000..80845f70f9 --- /dev/null +++ b/scripts/azure-pipelines/osx/configuration/vagrant-box-configuration.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "brew", + "brew-cask" + ], + "properties": { + "brew": { + "type": "array", + "items": { "type": "string" } + }, + "brew-cask": { + "type": "array", + "items": { "type": "string" } + } + } +} diff --git a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json index 3d6b619e8c..91540d77c9 100644 --- a/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json +++ b/scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json @@ -5,9 +5,11 @@ "required": [ "pat", - "base_name", - "disk_size", - "machine_identifiers" + "agent_pool", + "devops_url", + "machine_name", + "box_name", + "box_version" ], "properties": { @@ -28,9 +30,6 @@ }, "box_version": { "type": "string" - }, - "disk_size": { - "type": "integer" } } } diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index f0b12d494d..08e0fb0866 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1834,3 +1834,6 @@ wangle:x64-windows-static-md=fail # See https://developercommunity.visualstudio.com/t/Toolset-169-regression-vcxproj-producin/1356639 dimcli:x64-windows-static-md=fail dimcli:x64-windows-static=fail + +# cppgraphqlgen triggers an ICE on Apple Clang that comes with MacOS 11. +cppgraphqlgen:x64-osx=fail diff --git a/versions/baseline.json b/versions/baseline.json index af1251abb6..fb304b2e6d 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -5461,8 +5461,8 @@ "port-version": 0 }, "sdformat6": { - "baseline": "6.2.0-1", - "port-version": 0 + "baseline": "6.2.0", + "port-version": 2 }, "sdformat9": { "baseline": "9.4.0", diff --git a/versions/s-/sdformat6.json b/versions/s-/sdformat6.json index 04e7f03d88..7a75fc099c 100644 --- a/versions/s-/sdformat6.json +++ b/versions/s-/sdformat6.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "9104c5c84ce079686624adeaa3032543415acda2", + "version": "6.2.0", + "port-version": 2 + }, { "git-tree": "5a9a49ae80212912ecf538b7646f233cdba24211", "version-string": "6.2.0-1",