From 5bdb9d60123df20c247a1654e6a1def51d6c5140 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Fri, 29 Oct 2021 22:39:14 -0700 Subject: [PATCH] Update vcpkg tool, add windows-arm64, and download rather than build on some POSIX (#20838) --- scripts/bootstrap.ps1 | 8 +- scripts/bootstrap.sh | 302 ++++++++++++++++------------------------ scripts/ci.baseline.txt | 2 - 3 files changed, 124 insertions(+), 188 deletions(-) diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 index e6f8416473..c3d46a02c3 100644 --- a/scripts/bootstrap.ps1 +++ b/scripts/bootstrap.ps1 @@ -45,7 +45,13 @@ while (!($vcpkgRootDir -eq "") -and !(Test-Path "$vcpkgRootDir\.vcpkg-root")) Write-Verbose "Examining $vcpkgRootDir for .vcpkg-root - Found" -& "$scriptsDir/tls12-download.exe" github.com "/microsoft/vcpkg-tool/releases/download/2021-09-10/vcpkg.exe" "$vcpkgRootDir\vcpkg.exe" +$versionDate = '2021-10-25' +if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { + & "$scriptsDir/tls12-download-arm64.exe" github.com "/microsoft/vcpkg-tool/releases/download/$versionDate/vcpkg-arm64.exe" "$vcpkgRootDir\vcpkg.exe" +} else { + & "$scriptsDir/tls12-download.exe" github.com "/microsoft/vcpkg-tool/releases/download/$versionDate/vcpkg.exe" "$vcpkgRootDir\vcpkg.exe" +} + Write-Host "" if ($LASTEXITCODE -ne 0) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index ef8138c990..f012007861 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,34 +1,34 @@ #!/bin/sh -# Find .vcpkg-root, which indicates the root of this repo +# Find .vcpkg-root. vcpkgRootDir=$(X= cd -- "$(dirname -- "$0")" && pwd -P) while [ "$vcpkgRootDir" != "/" ] && ! [ -e "$vcpkgRootDir/.vcpkg-root" ]; do vcpkgRootDir="$(dirname "$vcpkgRootDir")" done -# Argument parsing +# Parse arguments. vcpkgDisableMetrics="OFF" vcpkgUseSystem=false -vcpkgAllowAppleClang=false -vcpkgBuildTests="OFF" +vcpkgUseMuslC="OFF" for var in "$@" do if [ "$var" = "-disableMetrics" -o "$var" = "--disableMetrics" ]; then vcpkgDisableMetrics="ON" elif [ "$var" = "-useSystemBinaries" -o "$var" = "--useSystemBinaries" ]; then - vcpkgUseSystem=true + echo "Warning: -useSystemBinaries no longer has any effect; ignored. Note that the VCPKG_USE_SYSTEM_BINARIES environment variable behavior is not changed." elif [ "$var" = "-allowAppleClang" -o "$var" = "--allowAppleClang" ]; then - vcpkgAllowAppleClang=true + echo "Warning: -allowAppleClang no longer has any effect; ignored." elif [ "$var" = "-buildTests" ]; then - vcpkgBuildTests="ON" + echo "Warning: -buildTests no longer has any effect; ignored." + elif [ "$var" = "-musl" ]; then + vcpkgUseMuslC="ON" elif [ "$var" = "-help" -o "$var" = "--help" ]; then echo "Usage: ./bootstrap-vcpkg.sh [options]" echo echo "Options:" echo " -help Display usage help" - echo " -disableMetrics Do not build metrics reporting into the executable" - echo " -useSystemBinaries Force use of the system utilities for building vcpkg" - echo " -allowAppleClang Set VCPKG_ALLOW_APPLE_CLANG to build vcpkg in apple with clang anyway" + echo " -disableMetrics Mark this vcpkg root to disable metrics." + echo " -musl Use the musl binary rather than the glibc binary on Linux." exit 1 else echo "Unknown argument $var. Use '-help' for help." @@ -50,6 +50,7 @@ if [ "$unixName" = "MINGW_NT" ]; then exit 0 fi +# Determine the downloads directory. if [ -z ${VCPKG_DOWNLOADS+x} ]; then downloadsDir="$vcpkgRootDir/downloads" else @@ -61,34 +62,88 @@ else fi -extractStringBetweenDelimiters() -{ - input=$1;leftDelim=$2;rightDelim=$3 - output="${input##*$leftDelim}" - output="${output%%$rightDelim*}" - echo "$output" -} - +# Check for minimal prerequisites. vcpkgCheckRepoTool() { __tool=$1 if ! command -v "$__tool" >/dev/null 2>&1 ; then echo "Could not find $__tool. Please install it (and other dependencies) with:" - echo "sudo apt-get install curl zip unzip tar" + echo "On Debian and Ubuntu derivatives:" + echo " sudo apt-get install curl zip unzip tar" + echo "On recent Red Hat and Fedora derivatives:" + echo " sudo dnf install curl zip unzip tar" + echo "On older Red Hat and Fedora derivatives:" + echo " sudo yum install curl zip unzip tar" + echo "On SUSE Linux and derivatives:" + echo " sudo zypper install curl zip unzip tar" + echo "On Alpine:" + echo " apk add build-base cmake ninja zip unzip curl git" + echo " (and export VCPKG_FORCE_SYSTEM_BINARIES=1)" exit 1 fi } -vcpkgCheckBuildTool() -{ - __tool=$1 - if ! command -v "$__tool" >/dev/null 2>&1 ; then - echo "Could not find $__tool. Please install it (and other dependencies) with:" - echo "sudo apt-get install cmake ninja-build" - exit 1 - fi -} +vcpkgCheckRepoTool curl +vcpkgCheckRepoTool zip +vcpkgCheckRepoTool unzip +vcpkgCheckRepoTool tar +UNAME="$(uname)" +ARCH="$(uname -m)" + +if [ -e /etc/alpine-release ]; then + vcpkgUseSystem="ON" + vcpkgUseMuslC="ON" +fi + +if [ "$UNAME" = "OpenBSD" ]; then + vcpkgUseSystem="ON" + + if [ -z "$CXX" ]; then + CXX=/usr/bin/clang++ + fi + if [ -z "$CC" ]; then + CC=/usr/bin/clang + fi +fi + +if [ "$vcpkgUseSystem" = "ON" ]; then + vcpkgCheckRepoTool cmake + vcpkgCheckRepoTool ninja + vcpkgCheckRepoTool git + vcpkgCheckRepoTool gcc +fi + +# Determine what we are going to do to bootstrap: +# MacOS -> Download vcpkg-macos +# Linux +# useMuslC -> download vcpkg-muslc +# amd64 -> download vcpkg-glibc +# Otherwise +# Download and build from source + +# Choose the vcpkg binary to download +vcpkgDownloadTool="ON" +vcpkgToolReleaseTag="2021-10-25" +if [ "$UNAME" = "Darwin" ]; then + echo "Downloading vcpkg-macos..." + vcpkgToolReleaseSha="09bd5d6bab4d45952f43626562af3e959cb82c96324003f665b902ccf65f4600fa1f1e84cbd54ad1f6e390be99cde5b3a1e640a0c3280aface02fbd1e867773e" + vcpkgToolName="vcpkg-macos" +elif [ "$vcpkgUseMuslC" = "ON" ]; then + echo "Downloading vcpkg-muslc..." + vcpkgToolReleaseSha="a598e37855f72841f3cd36a7b3f67d3cdc25f0577d851cd8dbdd5ff16190972ce5b9d0ca60c6e54ed147d1315bdedcd84005dfabc427fbdaee5b74726a351ec7" + vcpkgToolName="vcpkg-muslc" +elif [ "$ARCH" = "x86_64" ]; then + echo "Downloading vcpkg-glibc..." + vcpkgToolReleaseSha="c8f40cf91512500176ce3f7569ec0c91cfc93693921fac2db04ce8af0a6b65bc9aca880b7ecc44223b814e894fef66147af321c45e1b75a8628a78d499c272a8" + vcpkgToolName="vcpkg-glibc" +else + echo "Unable to determine a binary release of vcpkg; attempting to build from source." + vcpkgDownloadTool="OFF" + vcpkgToolReleaseSha="b12cc1229c93f8745864ac67a76213ecb052fb06a233bad2d6ce7743f40e0e2a72d654660d4dadeae95b9d6dd663a8b6cd83ae723cd90d63e3c795a1f12692bb" +fi + +# Do the download or build. vcpkgCheckEqualFileHash() { url=$1; filePath=$2; expectedHash=$3 @@ -117,108 +172,26 @@ vcpkgCheckEqualFileHash() vcpkgDownloadFile() { url=$1; downloadPath=$2 sha512=$3 - vcpkgCheckRepoTool "curl" rm -rf "$downloadPath.part" curl -L $url --tlsv1.2 --create-dirs --retry 3 --output "$downloadPath.part" --silent --show-error --fail || exit 1 vcpkgCheckEqualFileHash $url "$downloadPath.part" $sha512 + chmod +x "$downloadPath.part" mv "$downloadPath.part" "$downloadPath" } -vcpkgExtractArchive() +vcpkgExtractTar() { archive=$1; toPath=$2 rm -rf "$toPath" "$toPath.partial" mkdir -p "$toPath.partial" - - archiveType="${archive##*.}" - if [ "$archiveType" = "zip" ]; then - vcpkgCheckRepoTool "unzip" - $(cd "$toPath.partial" && unzip -qqo "$archive") - else - vcpkgCheckRepoTool "tar" - $(cd "$toPath.partial" && tar xzf "$archive") - fi + $(cd "$toPath.partial" && tar xzf "$archive") mv "$toPath.partial" "$toPath" } -fetchTool() -{ - tool=$1; UNAME=$2; __output=$3 - - if [ "$tool" = "" ]; then - echo "No tool name provided" - return 1 - fi - - if [ "$UNAME" = "Linux" ]; then - os="linux" - elif [ "$UNAME" = "Darwin" ]; then - os="osx" - elif [ "$UNAME" = "FreeBSD" ]; then - os="freebsd" - else - echo "Unknown uname: $UNAME" - return 1 - fi - - xmlFileAsString=`cat "$vcpkgRootDir/scripts/vcpkgTools.xml"` - toolRegexStart="" - toolData="$(extractStringBetweenDelimiters "$xmlFileAsString" "$toolRegexStart" "")" - case "$toolData" in - "" | "" "")" - - toolPath="$downloadsDir/tools/$tool-$version-$os" - - exeRelativePath="$(extractStringBetweenDelimiters "$toolData" "" "")" - exePath="$toolPath/$exeRelativePath" - - if [ -e "$exePath" ]; then - eval $__output="'$exePath'" - return 0 - fi - - isArchive=true - if [ $isArchive = true ]; then - archiveName="$(extractStringBetweenDelimiters "$toolData" "" "")" - downloadPath="$downloadsDir/$archiveName" - else - echo "Non-archives not supported yet" - return 1 - fi - - url="$(extractStringBetweenDelimiters "$toolData" "" "")" - sha512="$(extractStringBetweenDelimiters "$toolData" "" "")" - if ! [ -e "$downloadPath" ]; then - echo "Downloading $tool..." - vcpkgDownloadFile $url "$downloadPath" $sha512 - echo "Downloading $tool... done." - else - vcpkgCheckEqualFileHash $url "$downloadPath" $sha512 - fi - - if [ $isArchive = true ]; then - echo "Extracting $tool..." - vcpkgExtractArchive "$downloadPath" "$toolPath" - echo "Extracting $tool... done." - fi - - if ! [ -e "$exePath" ]; then - echo "Could not detect or download $tool" - return 1 - fi - - eval $__output="'$exePath'" - return 0 -} - -selectCXX() -{ +if [ "$vcpkgDownloadTool" = "ON" ]; then + vcpkgDownloadFile "https://github.com/microsoft/vcpkg-tool/releases/download/$vcpkgToolReleaseTag/$vcpkgToolName" "$vcpkgRootDir/vcpkg" $vcpkgToolReleaseSha +else if [ "x$CXX" = "x" ]; then if which g++-11 >/dev/null 2>&1; then CXX=g++-11 @@ -237,81 +210,40 @@ selectCXX() fi # If we can't find g++, allow CMake to do the look-up fi -} -# Preparation -UNAME="$(uname)" -ARCH="$(uname -m)" + vcpkgToolReleaseTarball="$vcpkgToolReleaseTag.tar.gz" + vcpkgToolUrl="https://github.com/microsoft/vcpkg-tool/archive/$vcpkgToolReleaseTarball" + baseBuildDir="$vcpkgRootDir/buildtrees/_vcpkg" + buildDir="$baseBuildDir/build" + tarballPath="$downloadsDir/$vcpkgToolReleaseTarball" + srcBaseDir="$baseBuildDir/src" + srcDir="$srcBaseDir/vcpkg-tool-$vcpkgToolReleaseTag" -# Force using system utilities for building vcpkg if host arch is arm, arm64, s390x, or ppc64le. -if [ "$ARCH" = "armv7l" -o "$ARCH" = "aarch64" -o "$ARCH" = "s390x" -o "$ARCH" = "ppc64le" ]; then - vcpkgUseSystem=true -fi - -if [ "$UNAME" = "OpenBSD" ]; then - vcpkgUseSystem=true - - if [ -z "$CXX" ]; then - CXX=/usr/bin/clang++ - fi - if [ -z "$CC" ]; then - CC=/usr/bin/clang - fi -fi - -if $vcpkgUseSystem; then - cmakeExe="cmake" - ninjaExe="ninja" - vcpkgCheckBuildTool "$cmakeExe" - vcpkgCheckBuildTool "$ninjaExe" -else - fetchTool "cmake" "$UNAME" cmakeExe || exit 1 - fetchTool "ninja" "$UNAME" ninjaExe || exit 1 -fi -if [ "$os" = "osx" ]; then - if [ "$vcpkgAllowAppleClang" = "true" ] || [[ $(sw_vers -productVersion | awk -F '.' '{print $1}') -ge 11 ]]; then - CXX=clang++ + if [ -e "$tarballPath" ]; then + vcpkgCheckEqualFileHash "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha" else - selectCXX + echo "Downloading vcpkg tool sources" + vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha" fi -else - selectCXX + + echo "Building vcpkg-tool..." + rm -rf "$baseBuildDir" + mkdir -p "$buildDir" + vcpkgExtractTar "$tarballPath" "$srcBaseDir" + cmakeConfigOptions="-DCMAKE_BUILD_TYPE=Release -G 'Ninja' -DVCPKG_DEVELOPMENT_WARNINGS=OFF" + + if [ "${VCPKG_MAX_CONCURRENCY}" != "" ] ; then + cmakeConfigOptions=" $cmakeConfigOptions '-DCMAKE_JOB_POOL_COMPILE:STRING=compile' '-DCMAKE_JOB_POOL_LINK:STRING=link' '-DCMAKE_JOB_POOLS:STRING=compile=$VCPKG_MAX_CONCURRENCY;link=$VCPKG_MAX_CONCURRENCY' " + fi + + (cd "$buildDir" && CXX="$CXX" eval cmake "$srcDir" $cmakeConfigOptions) || exit 1 + (cd "$buildDir" && cmake --build .) || exit 1 + + rm -rf "$vcpkgRootDir/vcpkg" + cp "$buildDir/vcpkg" "$vcpkgRootDir/" fi -# Do the build -vcpkgToolReleaseTag="2021-09-10" -vcpkgToolReleaseSha="0bea4c7bdd91933d44a0214e2202eb5ef988826d32ae7a00a8868e510710e7de0b336b1cc6aa1ea20af2f6e24d92f2ab665046089bb4ec43bc2add94a901d5fc" -vcpkgToolReleaseTarball="$vcpkgToolReleaseTag.tar.gz" -vcpkgToolUrl="https://github.com/microsoft/vcpkg-tool/archive/$vcpkgToolReleaseTarball" -baseBuildDir="$vcpkgRootDir/buildtrees/_vcpkg" -buildDir="$baseBuildDir/build" -tarballPath="$downloadsDir/$vcpkgToolReleaseTarball" -srcBaseDir="$baseBuildDir/src" -srcDir="$srcBaseDir/vcpkg-tool-$vcpkgToolReleaseTag" - -if [ -e "$tarballPath" ]; then - vcpkgCheckEqualFileHash "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha" -else - echo "Downloading vcpkg tool sources" - vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha" -fi - -echo "Building vcpkg-tool..." -rm -rf "$baseBuildDir" -mkdir -p "$buildDir" -vcpkgExtractArchive "$tarballPath" "$srcBaseDir" -cmakeConfigOptions="-DCMAKE_BUILD_TYPE=Release -G 'Ninja' -DCMAKE_MAKE_PROGRAM='$ninjaExe'" - -if [ "${VCPKG_MAX_CONCURRENCY}" != "" ] ; then - cmakeConfigOptions=" $cmakeConfigOptions '-DCMAKE_JOB_POOL_COMPILE:STRING=compile' '-DCMAKE_JOB_POOL_LINK:STRING=link' '-DCMAKE_JOB_POOLS:STRING=compile=$VCPKG_MAX_CONCURRENCY;link=$VCPKG_MAX_CONCURRENCY' " -fi - -(cd "$buildDir" && CXX="$CXX" eval "$cmakeExe" "$srcDir" $cmakeConfigOptions "-DBUILD_TESTING=$vcpkgBuildTests" "-DVCPKG_DEVELOPMENT_WARNINGS=OFF" "-DVCPKG_ALLOW_APPLE_CLANG=$vcpkgAllowAppleClang") || exit 1 -(cd "$buildDir" && "$cmakeExe" --build .) || exit 1 - -rm -rf "$vcpkgRootDir/vcpkg" -cp "$buildDir/vcpkg" "$vcpkgRootDir/" - +# Apply the disable-metrics marker file. if [ "$vcpkgDisableMetrics" = "ON" ]; then touch "$vcpkgRootDir/vcpkg.disable-metrics" elif ! [ -f "$vcpkgRootDir/vcpkg.disable-metrics" ]; then diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index e288e7ebe4..f1eea42970 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -643,8 +643,6 @@ libmpeg2:x64-uwp=fail libmupdf:x64-osx=fail libmysql:x86-windows=fail libmysql:arm64-windows=fail -libopenmpt:x64-linux=fail -libopenmpt:x64-osx=fail libopusenc:arm-uwp=fail libopusenc:x64-uwp=fail libosip2:x64-windows-static-md=fail