From 11b4a16bcadd252fd9721204262ab2bb6d09464e Mon Sep 17 00:00:00 2001 From: Joachim Gehweiler <44170764+jgehw@users.noreply.github.com> Date: Thu, 12 Nov 2020 19:15:48 +0100 Subject: [PATCH] [tensorflow-cc] More or less completely rewritten port, update to 2.3.1 (#13028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix build issues occurring from default bazel config being used because customized bazel config is stored in wrong directory * [tensorflow-cc] Update CONTROL and ci.baseline.txt * fix also applies to windows static build * fix debug and static builds as well as library naming for non-linux - add patch to fix debug builds - add patch to fix exports for static linking - really build debug (instead of cloning release) - override bazel build options for debug (work around bazel bug) - bazel doesn't support static libraries: work around by building dynamic library and constructing static linkage commands from build log - Windows .pdb file can't be >4GB even on x64: work around using reduced debug information - Windows doesn't support .lib files >4GB even on x64, so split into multiple libs - vcpkg requires equal amount of libs for debug and release: work around using handcrafted empty dummy libs - fix naming of libs (.dll on Windows and .dylib on macOS) * upgrade tensorflow from v1.14 to v2.3 - adapt patch files to tensorflow code changes - update bazel from v0.25.2 to v3.1 - on Windows use python installed on the host instead of embedded python obtained via vcpkg because embedded python lacks pip, which we need to obtain numpy - on Windows add MSYS2 to the PATH so that bazel tools can access MSYS2 GIT - add support for custom CA certificates when using HTTPS_PROXY * fix execute process macro The existing implementation totally screwed up commands if the command's arguments contained semicolons (this is the case, e.g., in the FindPython modules of the cmake distribution). * extend overriden execute_process to more than one COMMAND as there actually are use cases for this * added another patch required for tensorflow v2.3, fixed path and working directory * Revert "incorporate changes from microsoft:master" * Revert "Revert "incorporate changes from microsoft:master"" * final fixes for static build + improving out messages * enabling linux and osx in CI to see if it works now * simplified code, fixed version numbers, fixed generated include cmake file * fix failing postbuild check on handcrafted empty dummy library by spreading the last real libraries contents over the required number of libraries * remove dead code commit by mistake again * improvements from code review * cleaner fix for debug code * find pip3 in PATH (PYTHON3_DIR apparently not valid for pip3) * fix error in python helper script * fix wrong libname in postbuild script * fix python detection + switch to python on msys2 (instead of embedded python) for Windows as we need numpy * fix order of arguments * fix command (it may contain spaces such as C:\Program Files\...) * revert last commit (root cause for CI failures is something different: there are line breaks in path) * fix regex comparision (value needs to be escaped as it may contains regex special characters such as brackets, eg C:/Program Files (x86)/...) * fix linebreaks in generated file * fix CRT linkage (macOS doesn't support static CRT linkage; it's set to dynamic even static target triplets for macOS and linux) * refactor implemenation to avoid as much code duplication as possible -- algorithmically identical * fix version numbers in helper scripts * enable work-around for Windows until bazel fix is available * install missing python3-pip on linux * fix linux build by patching * apply timeout feature now available via merged master branch * correct linux build patch * improve debug build patches (no functional difference because LOG(FATAL, ...) macro internally anyway calls abort(), which the compiler doesn't detect in debug mode... * improve linux patch * temporarily add debug to inspect what's going on on macOS CI * remove temporary debug code and fix static linking scripts for linux and macOS * fix regex escaping * fix ambiguous match while grepping for the framework link command * extend fix of ambiguous match while grepping for the framework link command * fix what merge of master broke * fix more what got broken by merging master (all packages and their dependencies are now maintained manually instead of using pacman...) * remove "unofficial" from filename * added switch do distinct classic and manifest mode when generating config.cmake file * create symlinks for libraries without version number * fix linux postbuild script * temporarily disable code making problems * add note for linking on Linux and macOS * forget to add README file in previous commit * add file forgotton in macro fixing patch * fix python library path * fix macOS static link command * update linkage instructions in README * Update ports/tensorflow-cc/CONTROL * Update ports/tensorflow-cc/portfile.cmake * Update scripts/ci.baseline.txt * use vcpkg_execute_required_process * pass C_FLAGS and CXX_FLAGS to bazel * fix INTERFACE_INCLUDE_DIRECTORIES * fix optional c/cxx arguments * also add linker opts * update README * merge static libs into one to support force_load (cannot force_load both due to duplicate symbols) * update README * quote python path (it might contain spaces that don't get escaped inside outer quotes of bash command) * fix python path also for static build * add arm(64) as currently unsupported arch * bazel 3.7 is available -> remove workaround * update README, remove necessary c-ares from deps * update msys package * add uwp specific options, and minor general improvements * fix string replace * fix control file and windows path separator * revert backslashes-fix -- the root cause was missing .exe extension * upgrade to tf 2.3.1 * fix hard-coded version * remove uwp work-in-progress code so that PR can be merged * update README and print out usage info in portfile * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-windows Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/portfile.cmake Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/portfile.cmake Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/portfile.cmake Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/portfile.cmake Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/portfile.cmake Co-authored-by: nicole mazzuca * improve usage hints as discussed in review * add comment * apply changes from review * make additional compiler / linker args space-proof * Update ports/tensorflow-cc/README-macos Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-linux Co-authored-by: nicole mazzuca * Update ports/tensorflow-cc/README-macos Co-authored-by: nicole mazzuca * apply changes from code review * maybe fix the config files * rob.maynard CRs * fix windows static lib naming for first part * Update ports/tensorflow-cc/generate_static_link_cmd_windows.py Co-authored-by: nicole mazzuca * apply rob maynards 1st suggestion * nicole CRs * format/fix-compile * fix missing string termination * prefer IMPORTED_LOCATION over IMPORTED_LOCATION_RELEASE to have default fall-back * hopefully fix the issue where no libraries are generated * final stuff Co-authored-by: Gehweiler Co-authored-by: wangli28 Co-authored-by: Jack·Boos·Yu <47264268+JackBoosY@users.noreply.github.com> Co-authored-by: Joachim Gehweiler Co-authored-by: nicole mazzuca --- ports/tensorflow-cc/CONTROL | 6 +- ports/tensorflow-cc/README-linux | 13 + ports/tensorflow-cc/README-macos | 12 + ports/tensorflow-cc/README-windows | 5 + ports/tensorflow-cc/TensorflowCCConfig.cmake | 28 -- .../change-macros-for-static-lib.patch | 266 ++++++++++ .../tensorflow-cc/convert_lib_params_linux.py | 29 ++ .../convert_lib_params_windows.py | 27 + ports/tensorflow-cc/file-exists.patch | 13 - .../tensorflow-cc/fix-dbg-build-errors.patch | 111 +++++ ports/tensorflow-cc/fix-linux-build.patch | 64 +++ .../tensorflow-cc/fix-more-build-errors.patch | 11 + .../generate_static_link_cmd_linux.py | 22 + .../generate_static_link_cmd_macos.py | 46 ++ .../generate_static_link_cmd_windows.py | 41 ++ ports/tensorflow-cc/portfile.cmake | 468 +++++++++++++++--- .../tensorflow-cc-config-shared.cmake.in | 88 ++++ .../tensorflow-cc-config-static.cmake.in | 56 +++ .../tensorflow-cc-config-windows-dll.cmake.in | 56 +++ .../tensorflow-cc-config-windows-lib.cmake.in | 19 + scripts/build_info.cmake | 5 +- scripts/ci.baseline.txt | 6 - scripts/cmake/vcpkg_acquire_msys.cmake | 149 ++++++ .../cmake/vcpkg_find_acquire_program.cmake | 8 +- toolsrc/VERSION.txt | 2 +- toolsrc/include/vcpkg/base/files.h | 7 + toolsrc/include/vcpkg/build.h | 13 +- toolsrc/src/vcpkg/base/files.cpp | 14 - toolsrc/src/vcpkg/build.cpp | 18 +- toolsrc/src/vcpkg/postbuildlint.cpp | 5 +- 30 files changed, 1450 insertions(+), 158 deletions(-) create mode 100644 ports/tensorflow-cc/README-linux create mode 100644 ports/tensorflow-cc/README-macos create mode 100644 ports/tensorflow-cc/README-windows delete mode 100644 ports/tensorflow-cc/TensorflowCCConfig.cmake create mode 100644 ports/tensorflow-cc/change-macros-for-static-lib.patch create mode 100644 ports/tensorflow-cc/convert_lib_params_linux.py create mode 100644 ports/tensorflow-cc/convert_lib_params_windows.py delete mode 100644 ports/tensorflow-cc/file-exists.patch create mode 100644 ports/tensorflow-cc/fix-dbg-build-errors.patch create mode 100644 ports/tensorflow-cc/fix-linux-build.patch create mode 100644 ports/tensorflow-cc/fix-more-build-errors.patch create mode 100644 ports/tensorflow-cc/generate_static_link_cmd_linux.py create mode 100644 ports/tensorflow-cc/generate_static_link_cmd_macos.py create mode 100644 ports/tensorflow-cc/generate_static_link_cmd_windows.py create mode 100644 ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in create mode 100644 ports/tensorflow-cc/tensorflow-cc-config-static.cmake.in create mode 100644 ports/tensorflow-cc/tensorflow-cc-config-windows-dll.cmake.in create mode 100644 ports/tensorflow-cc/tensorflow-cc-config-windows-lib.cmake.in diff --git a/ports/tensorflow-cc/CONTROL b/ports/tensorflow-cc/CONTROL index 2d0ff50073..d422260e33 100644 --- a/ports/tensorflow-cc/CONTROL +++ b/ports/tensorflow-cc/CONTROL @@ -1,6 +1,4 @@ Source: tensorflow-cc -Version: 1.14 -Port-Version: 5 +Version: 2.3.1 Description: Library for computation using data flow graphs for scalable machine learning -Build-Depends: c-ares -Supports: !x86 +Supports: !(x86|arm|uwp) diff --git a/ports/tensorflow-cc/README-linux b/ports/tensorflow-cc/README-linux new file mode 100644 index 0000000000..4e0cb514ac --- /dev/null +++ b/ports/tensorflow-cc/README-linux @@ -0,0 +1,13 @@ +Warning: Static TensorFlow build contains several external dependencies that may cause linking conflicts + (for example, one cannot use both openssl and TensorFlow in the same project, since TensorFlow contains boringssl). + +Note: Besides TensorFlow itself, you'll also need to pass its dependencies to the linker: + * libm + * libdl + * libpthread +for example, '-ltensorflow_cc -lm -ldl -lpthread'. + +Note: There is no separate libtensorflow_framework.a as it got merged into libtensorflow_cc.a to avoid linking conflicts. + +Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to tell the linker to include the whole library: + '-Wl,--whole-archive -ltensorflow_cc -Wl,--no-whole-archive [rest of linker arguments]' diff --git a/ports/tensorflow-cc/README-macos b/ports/tensorflow-cc/README-macos new file mode 100644 index 0000000000..6bbfd07811 --- /dev/null +++ b/ports/tensorflow-cc/README-macos @@ -0,0 +1,12 @@ +Warning: Static TensorFlow build contains several external dependencies that may cause linking conflicts + (for example, one cannot use both openssl and TensorFlow in the same project, since TensorFlow contains boringssl). + +Note: Besides TensorFlow itself, you'll also need to pass its dependencies to the linker: + * CoreFoundation + * Security (only when linking against the whole library, see below) +for example, '-ltensorflow_cc -framework CoreFoundation -framework Security'. + +Note: There is no separate libtensorflow_framework.a as it got merged into libtensorflow_cc.a to avoid linking conflicts. + +Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to tell the linker to include the whole library: + '-Wl,-force_load,path/to/libtensorflow_cc.a [rest of linker arguments]' diff --git a/ports/tensorflow-cc/README-windows b/ports/tensorflow-cc/README-windows new file mode 100644 index 0000000000..5a19529760 --- /dev/null +++ b/ports/tensorflow-cc/README-windows @@ -0,0 +1,5 @@ +Warning: Static TensorFlow build contains several external dependencies that may cause linking conflicts + (for example, one cannot use both openssl and TensorFlow in the same project, since TensorFlow contains boringssl). + +Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to convince the linker to include the whole library, i.e., link using options + '/WHOLEARCHIVE:tensorflow_cc.lib /WHOLEARCHIVE:tensorflow_cc-part2.lib /WHOLEARCHIVE:tensorflow_cc-part3.lib ...' diff --git a/ports/tensorflow-cc/TensorflowCCConfig.cmake b/ports/tensorflow-cc/TensorflowCCConfig.cmake deleted file mode 100644 index db12811d5c..0000000000 --- a/ports/tensorflow-cc/TensorflowCCConfig.cmake +++ /dev/null @@ -1,28 +0,0 @@ -set(tensorflow_cc_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../include") - -message(WARNING "Tensorflow has vendored dependencies. You may need to manually include files from tensorflow-external") -set(tensorflow_cc_INCLUDE_DIRS - ${tensorflow_cc_INCLUDE_DIR} - ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/ - ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/tensorflow/ - ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/com_google_absl - ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/bazel-out/k8-opt/bin/ - ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/protobuf_archive/src/ -) - -add_library(tensorflow_cc::tensorflow_framework SHARED IMPORTED) -set_target_properties(tensorflow_cc::tensorflow_framework - PROPERTIES - IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_framework.so.1.14.0 - INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}" -) - -add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED) -set_target_properties(tensorflow_cc::tensorflow_cc - PROPERTIES - IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_cc.so.1.14.0 - INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}" -) - -set(tensorflow_cc_FOUND TRUE) -set(tensorflow_framework_FOUND TRUE) \ No newline at end of file diff --git a/ports/tensorflow-cc/change-macros-for-static-lib.patch b/ports/tensorflow-cc/change-macros-for-static-lib.patch new file mode 100644 index 0000000000..b000d5f0ea --- /dev/null +++ b/ports/tensorflow-cc/change-macros-for-static-lib.patch @@ -0,0 +1,266 @@ +diff --git a/tensorflow/core/platform/macros.h b/tensorflow/core/platform/macros.h +--- a/tensorflow/core/platform/macros.h ++++ b/tensorflow/core/platform/macros.h +@@ -57,16 +57,7 @@ + #define TF_SCANF_ATTRIBUTE(string_index, first_to_check) + #endif + +-// Control visibility outside .so +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_EXPORT __declspec(dllexport) +-#else +-#define TF_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 ++#define TF_EXPORT + + #ifdef __has_builtin + #define TF_HAS_BUILTIN(x) __has_builtin(x) + +diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h +--- a/tensorflow/c/c_api.h ++++ b/tensorflow/c/c_api.h +@@ -70,24 +70,7 @@ + // and the API just provides high level controls over the number of + // devices of each type. + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes. +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h +--- a/tensorflow/c/c_api_experimental.h ++++ b/tensorflow/c/c_api_experimental.h +@@ -28,24 +28,7 @@ + // The API here is subject to changes in the future. + // -------------------------------------------------------------------------- + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes.$a +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/c_api_macros.h b/tensorflow/c/c_api_macros.h +--- a/tensorflow/c/c_api_macros.h ++++ b/tensorflow/c/c_api_macros.h +@@ -16,18 +16,6 @@ + #ifndef TENSORFLOW_C_C_API_MACROS_H_ + #define TENSORFLOW_C_C_API_MACROS_H_ + +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #endif // TENSORFLOW_C_C_API_MACROS_H_ + +diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h +--- a/tensorflow/c/kernels.h ++++ b/tensorflow/c/kernels.h +@@ -21,24 +21,7 @@ + #include "tensorflow/c/tf_datatype.h" + #include "tensorflow/c/tf_status.h" + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes. +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/ops.h b/tensorflow/c/ops.h +--- a/tensorflow/c/ops.h ++++ b/tensorflow/c/ops.h +@@ -76,19 +76,7 @@ + #include "tensorflow/c/tf_datatype.h" + #include "tensorflow/c/tf_status.h" + +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/tf_datatype.h b/tensorflow/c/tf_datatype.h +--- a/tensorflow/c/tf_datatype.h ++++ b/tensorflow/c/tf_datatype.h +@@ -18,24 +18,7 @@ + + #include + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes. +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/tf_status.h b/tensorflow/c/tf_status.h +--- a/tensorflow/c/tf_status.h ++++ b/tensorflow/c/tf_status.h +@@ -16,19 +16,7 @@ + #ifndef TENSORFLOW_C_TF_STATUS_H_ + #define TENSORFLOW_C_TF_STATUS_H_ + +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h +--- a/tensorflow/c/tf_tensor.h ++++ b/tensorflow/c/tf_tensor.h +@@ -22,24 +22,7 @@ + #include "tensorflow/c/tf_datatype.h" + #include "tensorflow/c/tf_status.h" + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes. +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { + +diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h +--- a/tensorflow/c/eager/c_api.h ++++ b/tensorflow/c/eager/c_api.h +@@ -22,24 +22,7 @@ + + #include "tensorflow/c/c_api.h" + +-// Macro to control visibility of exported symbols in the shared library (.so, +-// .dylib, .dll). +-// This duplicates the TF_EXPORT macro definition in +-// tensorflow/core/platform/macros.h in order to keep this .h file independent +-// of any other includes.$a +-#ifdef SWIG + #define TF_CAPI_EXPORT +-#else +-#if defined(_WIN32) +-#ifdef TF_COMPILE_LIBRARY +-#define TF_CAPI_EXPORT __declspec(dllexport) +-#else +-#define TF_CAPI_EXPORT __declspec(dllimport) +-#endif // TF_COMPILE_LIBRARY +-#else +-#define TF_CAPI_EXPORT __attribute__((visibility("default"))) +-#endif // _WIN32 +-#endif // SWIG + + #ifdef __cplusplus + extern "C" { diff --git a/ports/tensorflow-cc/convert_lib_params_linux.py b/ports/tensorflow-cc/convert_lib_params_linux.py new file mode 100644 index 0000000000..ba2c352701 --- /dev/null +++ b/ports/tensorflow-cc/convert_lib_params_linux.py @@ -0,0 +1,29 @@ +import sys + +version = sys.argv[1] + +with open(f"libtensorflow_cc.a.{version}-2.params", "w") as f_out: + parts = [] + with open(f"libtensorflow_framework.so.{version}-2.params", "r") as f_in: + skip_next = False + for line in f_in: + if skip_next: + skip_next = False + continue + if line.startswith("-o"): + skip_next = True + elif line.startswith("bazel-out"): + f_out.write(line) + parts.append(line) + parts = set(parts) + with open(f"libtensorflow_cc.so.{version}-2.params", "r") as f_in: + skip_next = False + for line in f_in: + if skip_next: + skip_next = False + continue + if line.startswith("-o"): + skip_next = True + elif line.startswith("bazel-out"): + if line not in parts: + f_out.write(line) diff --git a/ports/tensorflow-cc/convert_lib_params_windows.py b/ports/tensorflow-cc/convert_lib_params_windows.py new file mode 100644 index 0000000000..5fc2fdd683 --- /dev/null +++ b/ports/tensorflow-cc/convert_lib_params_windows.py @@ -0,0 +1,27 @@ +import os +import sys + +file_no = 1 +with open("tensorflow_cc.dll-2.params", "r") as f_in: + lib_name = None + acc_size = 0 + f_out = open("tensorflow_cc.lib-2.params-part1", "w") + for line in f_in: + if line.startswith("/OUT:"): + lib_name = line + line = line.replace(".dll", "-part1.lib") + f_out.write(line) + elif line.startswith("/WHOLEARCHIVE:"): + line = line[len("/WHOLEARCHIVE:"):] + size = os.stat(f"../../{line.strip()}").st_size + if acc_size + size > 0xFFFFFFFF: + # we need to split the library if it is >4GB, because it's not supported even on x64 Windows + f_out.close() + file_no += 1 + f_out = open(f"tensorflow_cc.lib-2.params-part{file_no}", "w") + acc_size = 0 + f_out.write(lib_name.replace(".dll", f"-part{file_no}.lib")) + acc_size += size + f_out.write(line) + f_out.close() + diff --git a/ports/tensorflow-cc/file-exists.patch b/ports/tensorflow-cc/file-exists.patch deleted file mode 100644 index 8cefb4828c..0000000000 --- a/ports/tensorflow-cc/file-exists.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/configure.py b/configure.py -index 43af22d..7989b1f 100644 ---- a/configure.py -+++ b/configure.py -@@ -183,7 +183,7 @@ def get_python_path(environ_cp, python_bin_path): - - paths = [] - for path in all_paths: -- if os.path.isdir(path): -+ if os.path.isdir(path) or True: - paths.append(path) - return paths - diff --git a/ports/tensorflow-cc/fix-dbg-build-errors.patch b/ports/tensorflow-cc/fix-dbg-build-errors.patch new file mode 100644 index 0000000000..63e95dbb87 --- /dev/null +++ b/ports/tensorflow-cc/fix-dbg-build-errors.patch @@ -0,0 +1,111 @@ +diff --git a/tensorflow/core/platform/path.cc b/tensorflow/core/platform/path.cc +--- a/tensorflow/core/platform/path.cc ++++ b/tensorflow/core/platform/path.cc +@@ -329,6 +329,7 @@ + } + LOG(FATAL) << "No temp directory found."; + #endif ++ std::abort(); + } + + bool GetTestUndeclaredOutputsDir(string* dir) { + +diff --git a/tensorflow/core/framework/device_base.cc b/tensorflow/core/framework/device_base.cc +--- a/tensorflow/core/framework/device_base.cc ++++ b/tensorflow/core/framework/device_base.cc +@@ -65,10 +65,12 @@ + + const DeviceAttributes& DeviceBase::attributes() const { + LOG(FATAL) << "Device does not implement attributes()"; ++ std::abort(); + } + + const string& DeviceBase::name() const { + LOG(FATAL) << "Device does not implement name()"; ++ std::abort(); + } + + void DeviceBase::set_eigen_cpu_device(Eigen::ThreadPoolDevice* d) { + +diff --git a/tensorflow/core/kernels/depthtospace_op.cc b/tensorflow/core/kernels/depthtospace_op.cc +--- a/tensorflow/core/kernels/depthtospace_op.cc ++++ b/tensorflow/core/kernels/depthtospace_op.cc +@@ -117,13 +117,13 @@ + // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32. + auto Tinput_v = input.template reinterpret_last_dimension(); + auto Toutput_v = outputs_tensor->reinterpret_last_dimension(); +- functor::DepthToSpaceOpFunctor functor; +- functor(context->eigen_device(), Tinput_v, block_size_, ++ functor::DepthToSpaceOpFunctor functor; ++ functor(context->eigen_device(), Tinput_v, block_size_, + Toutput_v); + return; + } else if (data_format_ == FORMAT_NCHW) { +- functor::DepthToSpaceOpFunctor functor; +- functor(context->eigen_device(), Tinput, block_size_, ++ functor::DepthToSpaceOpFunctor functor; ++ functor(context->eigen_device(), Tinput, block_size_, + Toutput); + return; + } +@@ -173,6 +173,15 @@ + } + } + }; ++#ifdef WIN32 ++template ++struct DepthToSpaceOpFunctor { ++ void operator()(const CPUDevice& d, typename TTypes::ConstTensor input, ++ int block_size, typename TTypes::Tensor output) { ++ LOG(FATAL) << "dummy implementation to make debug build compile"; ++ } ++}; ++#endif + } // namespace functor + + #define REGISTER(type) \ + +diff --git a/tensorflow/core/kernels/spacetodepth_op.cc b/tensorflow/core/kernels/spacetodepth_op.cc +--- a/tensorflow/core/kernels/spacetodepth_op.cc ++++ b/tensorflow/core/kernels/spacetodepth_op.cc +@@ -132,18 +132,18 @@ + // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32. + auto Tinput_v = input.template reinterpret_last_dimension(); + auto Toutput_v = outputs_tensor->reinterpret_last_dimension(); +- functor::SpaceToDepthOpFunctor functor; +- functor(context->eigen_device(), Tinput_v, block_size_, ++ functor::SpaceToDepthOpFunctor functor; ++ functor(context->eigen_device(), Tinput_v, block_size_, + Toutput_v); + } else if (data_format_ == FORMAT_NCHW) { + CHECK((std::is_same::value)); +- functor::SpaceToDepthOpFunctor functor; +- functor(context->eigen_device(), input.tensor(), ++ functor::SpaceToDepthOpFunctor functor; ++ functor(context->eigen_device(), input.tensor(), + block_size_, outputs_tensor->tensor()); + } else { + CHECK((std::is_same::value)); +- functor::SpaceToDepthOpFunctor functor; +- functor(context->eigen_device(), input.tensor(), ++ functor::SpaceToDepthOpFunctor functor; ++ functor(context->eigen_device(), input.tensor(), + block_size_, outputs_tensor->tensor()); + } + } else { +@@ -188,6 +188,15 @@ + } + } + }; ++#ifdef WIN32 ++template ++struct SpaceToDepthOpFunctor { ++ void operator()(const CPUDevice& d, typename TTypes::ConstTensor input, ++ int block_size, typename TTypes::Tensor output) { ++ LOG(FATAL) << "dummy implementation to make debug build compile"; ++ } ++}; ++#endif + } // namespace functor + + #define REGISTER(type) \ diff --git a/ports/tensorflow-cc/fix-linux-build.patch b/ports/tensorflow-cc/fix-linux-build.patch new file mode 100644 index 0000000000..81a6a3be8c --- /dev/null +++ b/ports/tensorflow-cc/fix-linux-build.patch @@ -0,0 +1,64 @@ +diff --git a/third_party/aws/aws-checksums.bazel b/third_party/aws/aws-checksums.bazel +--- a/third_party/aws/aws-checksums.bazel ++++ b/third_party/aws/aws-checksums.bazel +@@ -7,6 +7,13 @@ + + exports_files(["LICENSE"]) + ++load("@bazel_skylib//lib:selects.bzl", "selects") ++ ++selects.config_setting_group( ++ name = "is_linux_debug", ++ match_all = ["@org_tensorflow//tensorflow:linux_x86_64", "@org_tensorflow//tensorflow:debug"], ++) ++ + cc_library( + name = "aws-checksums", + srcs = select({ +@@ -27,4 +34,10 @@ + deps = [ + "@aws-c-common", + ], ++ defines = select({ ++ ":is_linux_debug": [ ++ "DEBUG_BUILD" ++ ], ++ "//conditions:default": [], ++ }), + ) + +diff --git a/tensorflow/core/kernels/data/experimental/io_ops.cc b/tensorflow/core/kernels/data/experimental/io_ops.cc +--- a/tensorflow/core/kernels/data/experimental/io_ops.cc ++++ b/tensorflow/core/kernels/data/experimental/io_ops.cc +@@ -28,6 +28,12 @@ + namespace data { + namespace experimental { + ++/*static*/ constexpr const char* const LoadDatasetOp::kCompression; ++/*static*/ constexpr const char* const LoadDatasetOp::kReaderFunc; ++/*static*/ constexpr const char* const LoadDatasetOp::kReaderFuncTarguments; ++ ++/*static*/ constexpr const int SaveDatasetOp::kFileFormatVersion; ++ + SaveDatasetOp::SaveDatasetOp(OpKernelConstruction* ctx) + : HybridAsyncOpKernel(ctx, "tf_data_save_dataset") { + OP_REQUIRES_OK(ctx, ctx->GetAttr(kCompression, &compression_)); + +diff --git a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc +--- a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc ++++ b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc +@@ -63,6 +63,14 @@ + namespace data { + namespace experimental { + ++/*static*/ constexpr const char* const SnapshotDatasetV2Op::kCompression; ++/*static*/ constexpr const char* const SnapshotDatasetV2Op::kReaderFunc; ++/*static*/ constexpr const char* const SnapshotDatasetV2Op::kShardFunc; ++/*static*/ constexpr const char* const SnapshotDatasetV2Op::kReaderFuncTarguments; ++/*static*/ constexpr const char* const SnapshotDatasetV2Op::kShardFuncTarguments; ++ ++/*static*/ constexpr const int SnapshotDatasetV2Op::kFileFormatVersion; ++ + // ==== Snapshot Implementation ==== + + /* The current snapshot on-disk layout is as follows: diff --git a/ports/tensorflow-cc/fix-more-build-errors.patch b/ports/tensorflow-cc/fix-more-build-errors.patch new file mode 100644 index 0000000000..c7d6a9b7fa --- /dev/null +++ b/ports/tensorflow-cc/fix-more-build-errors.patch @@ -0,0 +1,11 @@ +diff --git a/tensorflow/core/distributed_runtime/remote_device.cc b/tensorflow/core/distributed_runtime/remote_device.cc +--- a/tensorflow/core/distributed_runtime/remote_device.cc ++++ b/tensorflow/core/distributed_runtime/remote_device.cc +@@ -45,6 +45,7 @@ + ResourceMgr* resource_manager() override { + LOG(FATAL) << "Accessing the resource manager of a remote device is not " + << "supported."; ++ std::abort(); + } + + bool IsLocal() const override { return false; } diff --git a/ports/tensorflow-cc/generate_static_link_cmd_linux.py b/ports/tensorflow-cc/generate_static_link_cmd_linux.py new file mode 100644 index 0000000000..3b7c04f49c --- /dev/null +++ b/ports/tensorflow-cc/generate_static_link_cmd_linux.py @@ -0,0 +1,22 @@ +import re +import sys + +with open(sys.argv[1], "r") as f_in: + with open("static_link.sh", "w") as f_out: + p_cd = re.compile("^\\((cd .*) && \\\\$") + p_linker = re.compile("^\\s*(.+)gcc.+(@bazel-out\\S+libtensorflow_cc\\.so\\.\\d\\.\\d\\.\\d-2\\.params).*") + f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n") + env = [] + for line in f_in: + if line.startswith("(cd"): + # new command, reset + env = [line] + else: + m1 = p_linker.match(line) + if m1: + m2 = p_cd.match(env[0]) + f_out.write(m2.group(1) + "\n") + line = f'"{m1.group(1)}ar" rcs {m1.group(2)[1:-9].replace(".so", ".a")} {m1.group(2).replace(".so", ".a")}\n' + f_out.write(line) + else: + env.append(line) diff --git a/ports/tensorflow-cc/generate_static_link_cmd_macos.py b/ports/tensorflow-cc/generate_static_link_cmd_macos.py new file mode 100644 index 0000000000..f1dfbbc466 --- /dev/null +++ b/ports/tensorflow-cc/generate_static_link_cmd_macos.py @@ -0,0 +1,46 @@ +import re +import sys + +with open(sys.argv[1], "r") as f_in: + with open("static_link.sh", "w") as f_out: + p_cd = re.compile("^\\((cd .*) && \\\\$") + p_linker1 = re.compile("^.*cc_wrapper.sh.+-shared.+-o (bazel-out\\S+libtensorflow_cc\\.\\d\\.\\d\\.\\d\\.dylib)") + p_linker2 = re.compile("^.*cc_wrapper.sh.+-shared.+-o (bazel-out\\S+libtensorflow_framework\\.\\d\\.\\d\\.\\d\\.dylib)") + f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n") + env = [] + parts = None + for line in f_in: + if line.startswith("(cd"): + # new command, reset + env = [line] + else: + m1 = p_linker1.match(line) + m2 = p_linker2.match(line) + if m1: + tokens = line.split() + if parts is None: + parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] + else: + m = p_cd.match(env[0]) + f_out.write(m.group(1) + "\n") + tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] + old = set(parts) + parts += [t for t in tmp if t not in old] + line = f"libtool -static -o {m1.group(1).replace('.dylib', '.a')} {' '.join(parts)}\n" + f_out.write(line) + break + elif m2 and len(env) > 6: + tokens = line.split() + if parts is None: + parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] + else: + m = p_cd.match(env[0]) + f_out.write(m.group(1) + "\n") + tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")] + old = set(parts) + parts += [t for t in tmp if t not in old] + line = f"libtool -static -o {m2.group(1).replace('_framework', '_cc').replace('.dylib', '.a')} {' '.join(parts)}\n" + f_out.write(line) + break + else: + env.append(line) diff --git a/ports/tensorflow-cc/generate_static_link_cmd_windows.py b/ports/tensorflow-cc/generate_static_link_cmd_windows.py new file mode 100644 index 0000000000..363b3ad884 --- /dev/null +++ b/ports/tensorflow-cc/generate_static_link_cmd_windows.py @@ -0,0 +1,41 @@ +import os.path +import re +import sys + +with open(sys.argv[1], "r") as f_in: + with open("static_link.bat", "w") as f_out: + p_setenv = re.compile("^\s*(SET .+=.*)$") + p_linker = re.compile(".+link\\.exe.+tensorflow_cc\\.dll-2\\.params.*") + env = [] + for line in f_in: + if line.startswith("cd"): + # new command, reset + env = [] + else: + m = p_setenv.match(line) + if m: + env.append(m.group(1)) + else: + m = p_linker.match(line) + if m: + for e in env: + f_out.write(e + "\n") + tokens = line.split() + line = "\"" + params_file = None + for t in tokens: + if t.endswith("link.exe"): + t = t[:-len("link.exe")] + "lib.exe\"" + elif t == "/DLL" or t.lower()[1:].startswith("defaultlib:") or t.lower()[1:].startswith("ignore") or t.startswith("/OPT:") or t.startswith("/DEF:") or t.startswith("/DEBUG:") or t.startswith("/INCREMENTAL:"): + continue + elif t[0] == '@' and t.endswith("tensorflow_cc.dll-2.params"): + t = t[:-len("dll-2.params")] + "lib-2.params-part1" + params_file = t[1:-len("-part1")] + line += t + " " + f_out.write(line + "\n") + # check for more parts if library needs to be split + file_no = 2 + while os.path.isfile(f"{params_file}-part{file_no}"): + f_out.write(line.replace("lib-2.params-part1", f"lib-2.params-part{file_no}") + "\n") + file_no += 1 + break diff --git a/ports/tensorflow-cc/portfile.cmake b/ports/tensorflow-cc/portfile.cmake index 0aa53e7aed..35dbb5d1b0 100644 --- a/ports/tensorflow-cc/portfile.cmake +++ b/ports/tensorflow-cc/portfile.cmake @@ -1,61 +1,53 @@ -message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.") +vcpkg_fail_port_install(ON_ARCH "x86" "arm" ON_TARGET "UWP") -if (VCPKG_TARGET_ARCHITECTURE STREQUAL x86) - message(FATAL_ERROR "TensorFlow does not support 32bit systems.") -endif() +set(TF_VERSION 2.3.1) +set(TF_VERSION_SHORT 2.3) -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tensorflow/tensorflow - REF v1.14.0 - SHA512 ac9ea5a2d1c761aaafbdc335259e29c128127b8d069ec5b206067935180490aa95e93c7e13de57f7f54ce4ba4f34a822face22b4a028f60185edb380e5cd4787 - HEAD_REF master - PATCHES - file-exists.patch # required or otherwise it cant find python lib path on windows - fix-build-error.patch # Fix namespace error -) - -# due to https://github.com/bazelbuild/bazel/issues/8028, bazel must be version 25.0 or higher vcpkg_find_acquire_program(BAZEL) get_filename_component(BAZEL_DIR "${BAZEL}" DIRECTORY) vcpkg_add_to_path(PREPEND ${BAZEL_DIR}) set(ENV{BAZEL_BIN_PATH} "${BAZEL}") -vcpkg_find_acquire_program(PYTHON3) -get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY) -vcpkg_add_to_path(PREPEND ${PYTHON3_DIR}) -set(ENV{PYTHON_BIN_PATH} "${PYTHON3}") +function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE) + file(REMOVE_RECURSE ${PATH_TO_REMOVE}) + if(EXISTS "${PATH_TO_REMOVE}") + vcpkg_execute_required_process(COMMAND ${CMAKE_COMMAND} -E sleep 5 WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-sleep-${TARGET_TRIPLET}) + file(REMOVE_RECURSE ${PATH_TO_REMOVE}) + endif() +endfunction() vcpkg_find_acquire_program(GIT) get_filename_component(GIT_DIR "${GIT}" DIRECTORY) vcpkg_add_to_path(PREPEND ${GIT_DIR}) -function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE) - file(REMOVE_RECURSE ${PATH_TO_REMOVE}) - if (EXISTS "${PATH_TO_REMOVE}") - _execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 5) - file(REMOVE_RECURSE ${PATH_TO_REMOVE}) - endif() -endfunction() - if(CMAKE_HOST_WIN32) - vcpkg_acquire_msys(MSYS_ROOT PACKAGES bash unzip patch diffutils libintl gzip coreutils) - vcpkg_add_to_path(${MSYS_ROOT}/usr/bin) - set(BASH ${MSYS_ROOT}/usr/bin/bash.exe) - set(ENV{BAZEL_SH} ${MSYS_ROOT}/usr/bin/bash.exe) + vcpkg_acquire_msys(MSYS_ROOT PACKAGES bash unzip patch diffutils libintl gzip coreutils mingw-w64-x86_64-python-numpy) + vcpkg_add_to_path(${MSYS_ROOT}/usr/bin) + vcpkg_add_to_path(${MSYS_ROOT}/mingw64/bin) + set(BASH ${MSYS_ROOT}/usr/bin/bash.exe) - set(ENV{BAZEL_VS} $ENV{VSInstallDir}) - set(ENV{BAZEL_VC} $ENV{VCInstallDir}) - set(ENV{BAZEL_VC_FULL_VERSION} $ENV{VCToolsVersion}) + set(ENV{BAZEL_SH} ${MSYS_ROOT}/usr/bin/bash.exe) + set(ENV{BAZEL_VC} $ENV{VCInstallDir}) + set(ENV{BAZEL_VC_FULL_VERSION} $ENV{VCToolsVersion}) + + set(PYTHON3 "${MSYS_ROOT}/mingw64/bin/python3.exe") + vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getsitepackages()[0])" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH) +else() + vcpkg_find_acquire_program(PYTHON3) + get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY) + vcpkg_add_to_path(PREPEND ${PYTHON3_DIR}) + + vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET}) + vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH) endif() +set(ENV{PYTHON_BIN_PATH} "${PYTHON3}") +set(ENV{PYTHON_LIB_PATH} "${PYTHON_LIB_PATH}") -# we currently only support the release version -tensorflow_try_remove_recurse_wait(${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel) -file(RENAME ${SOURCE_PATH} ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel) -set(SOURCE_PATH "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel") +# check if numpy can be loaded +vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-numpy-${TARGET_TRIPLET}) # tensorflow has long file names, which will not work on windows -set(ENV{TEST_TMPDIR} ${CURRENT_BUILDTREES_DIR}/../.bzl) +set(ENV{TEST_TMPDIR} ${BUILDTREES_DIR}/.bzl) set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1) set(ENV{TF_NEED_KAFKA} 0) @@ -75,51 +67,379 @@ set(ENV{TF_NEED_IGNITE} 0) set(ENV{TF_NEED_ROCM} 0) set(ENV{TF_SET_ANDROID_WORKSPACE} 0) set(ENV{TF_DOWNLOAD_CLANG} 0) -set(ENV{TF_NCCL_VERSION} 2.3) +set(ENV{TF_NCCL_VERSION} ${TF_VERSION_SHORT}) set(ENV{NCCL_INSTALL_PATH} "") set(ENV{CC_OPT_FLAGS} "/arch:AVX") set(ENV{TF_NEED_CUDA} 0) set(ENV{TF_CONFIGURE_IOS} 0) -message(STATUS "Configuring TensorFlow") - -vcpkg_execute_required_process( - COMMAND ${PYTHON3} ${SOURCE_PATH}/configure.py --workspace "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel" - WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel - LOGNAME config-${TARGET_TRIPLET}-rel -) -message(STATUS "Warning: Building TensorFlow can take an hour or more.") - -if(CMAKE_HOST_WIN32) - vcpkg_execute_build_process( - COMMAND ${BASH} --noprofile --norc -c "${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true ///tensorflow:libtensorflow_cc.so ///tensorflow:install_headers" - WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel - LOGNAME build-${TARGET_TRIPLET}-rel - ) +if(VCPKG_TARGET_IS_WINDOWS) + set(BAZEL_LIB_NAME tensorflow_cc.dll) + set(PLATFORM_SUFFIX windows) + set(STATIC_LINK_CMD static_link.bat) +elseif(VCPKG_TARGET_IS_OSX) + set(BAZEL_LIB_NAME libtensorflow_cc.dylib) + set(PLATFORM_SUFFIX macos) + set(STATIC_LINK_CMD sh static_link.sh) + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + set(TF_LIB_NAME "libtensorflow_cc.dylib") + set(TF_LIB_NAME_SHORT "libtensorflow_cc.2.dylib") + set(TF_LIB_NAME_FULL "libtensorflow_cc.${TF_VERSION}.dylib") + set(TF_FRAMEWORK_NAME "libtensorflow_framework.dylib") + set(TF_FRAMEWORK_NAME_SHORT "libtensorflow_framework.2.dylib") + set(TF_FRAMEWORK_NAME_FULL "libtensorflow_framework.${TF_VERSION}.dylib") + else() + set(TF_LIB_NAME "libtensorflow_cc.a") + set(TF_LIB_NAME_SHORT "libtensorflow_cc.2.a") + set(TF_LIB_NAME_FULL "libtensorflow_cc.${TF_VERSION}.a") + endif() else() - vcpkg_execute_build_process( - COMMAND ${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true //tensorflow:libtensorflow_cc.so //tensorflow:install_headers - WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel - LOGNAME build-${TARGET_TRIPLET}-rel - ) + set(BAZEL_LIB_NAME libtensorflow_cc.so) + set(PLATFORM_SUFFIX linux) + set(STATIC_LINK_CMD sh static_link.sh) + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + set(TF_LIB_NAME "libtensorflow_cc.so") + set(TF_LIB_NAME_SHORT "libtensorflow_cc.so.2") + set(TF_LIB_NAME_FULL "libtensorflow_cc.so.${TF_VERSION}") + set(TF_FRAMEWORK_NAME "libtensorflow_framework.so") + set(TF_FRAMEWORK_NAME_SHORT "libtensorflow_framework.so.2") + set(TF_FRAMEWORK_NAME_FULL "libtensorflow_framework.so.${TF_VERSION}") + else() + set(TF_LIB_NAME "libtensorflow_cc.a") + set(TF_LIB_NAME_SHORT "libtensorflow_cc.a.2") + set(TF_LIB_NAME_FULL "libtensorflow_cc.a.${TF_VERSION}") + endif() endif() -file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-genfiles/tensorflow/include/ DESTINATION ${CURRENT_PACKAGES_DIR}/include/tensorflow-external) +foreach(BUILD_TYPE dbg rel) + # prefer repeated source extraction here for each build type over extracting once above the loop and copying because users reported issues with copying symlinks + set(STATIC_ONLY_PATCHES) + set(LINUX_ONLY_PATCHES) + if(VCPKG_LIBRARY_LINKAGE STREQUAL static) + set(STATIC_ONLY_PATCHES change-macros-for-static-lib.patch) # there is no static build option - change macros via patch and link library manually at the end + endif() + if(VCPKG_TARGET_IS_LINUX) + set(LINUX_ONLY_PATCHES fix-linux-build.patch) + endif() + vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO tensorflow/tensorflow + REF "v${TF_VERSION}" + SHA512 e497ef4564f50abf9f918be4522cf702f4cf945cb1ebf83af1386ac4ddc7373b3ba70c7f803f8ca06faf2c6b5396e60b1e0e9b97bfbd667e733b08b6e6d70ef0 + HEAD_REF master + PATCHES + fix-build-error.patch # Fix namespace error + fix-dbg-build-errors.patch # Fix no return statement + fix-more-build-errors.patch # Fix no return statement + ${STATIC_ONLY_PATCHES} + ${LINUX_ONLY_PATCHES} + ) -if(CMAKE_HOST_WIN32) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) -else() - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) - file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + message(STATUS "Configuring TensorFlow (${BUILD_TYPE})") + tensorflow_try_remove_recurse_wait(${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}) + file(RENAME ${SOURCE_PATH} ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}) + set(SOURCE_PATH "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}") + + vcpkg_execute_required_process( + COMMAND ${PYTHON3} ${SOURCE_PATH}/configure.py --workspace "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME config-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + + if(DEFINED ENV{BAZEL_CUSTOM_CACERTS}) + file(APPEND ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/.bazelrc "startup --host_jvm_args=-Djavax.net.ssl.trustStore='$ENV{BAZEL_CUSTOM_CACERTS}'\n") + message(STATUS "Using custom CA certificate store at: $ENV{BAZEL_CUSTOM_CACERTS}") + if(DEFINED ENV{BAZEL_CUSTOM_CACERTS_PASSWORD}) + file(APPEND ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/.bazelrc "startup --host_jvm_args=-Djavax.net.ssl.trustStorePassword='$ENV{BAZEL_CUSTOM_CACERTS_PASSWORD}'\n") + message(STATUS "Using supplied custom CA certificate store password.") + endif() + else() + if(DEFINED ENV{HTTPS_PROXY}) + message(STATUS "You are using HTTPS_PROXY. In case you encounter bazel certificate errors, you might want to set: BAZEL_CUSTOM_CACERTS=/path/to/trust.store (and optionally BAZEL_CUSTOM_CACERTS_PASSWORD), and to enable vcpkg to actually use it: VCPKG_KEEP_ENV_VARS=BAZEL_CUSTOM_CACERTS;BAZEL_CUSTOM_CACERTS_PASSWORD") + if(CMAKE_HOST_WIN32) + message(STATUS "(For BAZEL_CUSTOM_CACERTS please use forward slashes instead of backslashes on Windows systems.") + endif() + endif() + endif() + + message(STATUS "Warning: Building TensorFlow can take an hour or more.") + set(COPTS) + set(CXXOPTS) + set(LINKOPTS) + if(VCPKG_TARGET_IS_WINDOWS) + set(PLATFORM_COMMAND WINDOWS_COMMAND) + else() + set(PLATFORM_COMMAND UNIX_COMMAND) + endif() + if(BUILD_TYPE STREQUAL dbg) + if(VCPKG_TARGET_IS_WINDOWS) + set(BUILD_OPTS "--compilation_mode=dbg --features=fastbuild") # link with /DEBUG:FASTLINK instead of /DEBUG:FULL to avoid .pdb >4GB error + else() + set(BUILD_OPTS "--compilation_mode=dbg") + endif() + + separate_arguments(VCPKG_C_FLAGS ${PLATFORM_COMMAND} ${VCPKG_C_FLAGS}) + separate_arguments(VCPKG_C_FLAGS_DEBUG ${PLATFORM_COMMAND} ${VCPKG_C_FLAGS_DEBUG}) + foreach(OPT IN LISTS VCPKG_C_FLAGS VCPKG_C_FLAGS_DEBUG) + list(APPEND COPTS "--copt='${OPT}'") + endforeach() + separate_arguments(VCPKG_CXX_FLAGS ${PLATFORM_COMMAND} ${VCPKG_CXX_FLAGS}) + separate_arguments(VCPKG_CXX_FLAGS_DEBUG ${PLATFORM_COMMAND} ${VCPKG_CXX_FLAGS_DEBUG}) + foreach(OPT IN LISTS VCPKG_CXX_FLAGS VCPKG_CXX_FLAGS_DEBUG) + list(APPEND CXXOPTS "--cxxopt='${OPT}'") + endforeach() + separate_arguments(VCPKG_LINKER_FLAGS ${PLATFORM_COMMAND} ${VCPKG_LINKER_FLAGS}) + separate_arguments(VCPKG_LINKER_FLAGS_DEBUG ${PLATFORM_COMMAND} ${VCPKG_LINKER_FLAGS_DEBUG}) + foreach(OPT IN LISTS VCPKG_LINKER_FLAGS VCPKG_LINKER_FLAGS_DEBUG) + list(APPEND LINKOPTS "--linkopt='${OPT}'") + endforeach() + else() + set(BUILD_OPTS "--compilation_mode=opt") + + separate_arguments(VCPKG_C_FLAGS ${PLATFORM_COMMAND} ${VCPKG_C_FLAGS}) + separate_arguments(VCPKG_C_FLAGS_RELEASE ${PLATFORM_COMMAND} ${VCPKG_C_FLAGS_RELEASE}) + foreach(OPT IN LISTS VCPKG_C_FLAGS VCPKG_C_FLAGS_RELEASE) + list(APPEND COPTS "--copt='${OPT}'") + endforeach() + separate_arguments(VCPKG_CXX_FLAGS ${PLATFORM_COMMAND} ${VCPKG_CXX_FLAGS}) + separate_arguments(VCPKG_CXX_FLAGS_RELEASE ${PLATFORM_COMMAND} ${VCPKG_CXX_FLAGS_RELEASE}) + foreach(OPT IN LISTS VCPKG_CXX_FLAGS VCPKG_CXX_FLAGS_RELEASE) + list(APPEND CXXOPTS "--cxxopt='${OPT}'") + endforeach() + separate_arguments(VCPKG_LINKER_FLAGS ${PLATFORM_COMMAND} ${VCPKG_LINKER_FLAGS}) + separate_arguments(VCPKG_LINKER_FLAGS_RELEASE ${PLATFORM_COMMAND} ${VCPKG_LINKER_FLAGS_RELEASE}) + foreach(OPT IN LISTS VCPKG_LINKER_FLAGS VCPKG_LINKER_FLAGS_RELEASE) + list(APPEND LINKOPTS "--linkopt='${OPT}'") + endforeach() + endif() + + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + if(VCPKG_TARGET_IS_WINDOWS) + list(JOIN COPTS " " COPTS) + list(JOIN CXXOPTS " " CXXOPTS) + list(JOIN LINKOPTS " " LINKOPTS) + vcpkg_execute_build_process( + COMMAND ${BASH} --noprofile --norc -c "'${BAZEL}' build --verbose_failures ${BUILD_OPTS} ${COPTS} ${CXXOPTS} ${LINKOPTS} --python_path='${PYTHON3}' --define=no_tensorflow_py_deps=true ///tensorflow:tensorflow_cc.dll ///tensorflow:install_headers" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + else() + vcpkg_execute_build_process( + COMMAND ${BAZEL} build --verbose_failures ${BUILD_OPTS} --python_path=${PYTHON3} ${COPTS} ${CXXOPTS} ${LINKOPTS} --define=no_tensorflow_py_deps=true //tensorflow:${BAZEL_LIB_NAME} //tensorflow:install_headers + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + endif() + else() + if(VCPKG_TARGET_IS_WINDOWS) + if(VCPKG_CRT_LINKAGE STREQUAL static) + if(BUILD_TYPE STREQUAL dbg) + list(APPEND COPTS "--copt=-MTd") + else() + list(APPEND COPTS "--copt=-MT") + endif() + endif() + list(JOIN COPTS " " COPTS) + list(JOIN CXXOPTS " " CXXOPTS) + list(JOIN LINKOPTS " " LINKOPTS) + vcpkg_execute_build_process( + COMMAND ${BASH} --noprofile --norc -c "${BAZEL} build -s --verbose_failures ${BUILD_OPTS} --features=fully_static_link ${COPTS} ${CXXOPTS} ${LINKOPTS} --python_path='${PYTHON3}' --define=no_tensorflow_py_deps=true ///tensorflow:tensorflow_cc.dll ///tensorflow:install_headers" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + else() + vcpkg_execute_build_process( + COMMAND ${BAZEL} build -s --verbose_failures ${BUILD_OPTS} ${COPTS} ${CXXOPTS} ${LINKOPTS} --python_path=${PYTHON3} --define=no_tensorflow_py_deps=true //tensorflow:${BAZEL_LIB_NAME} //tensorflow:install_headers + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + endif() + if(NOT VCPKG_TARGET_IS_OSX) + if(VCPKG_TARGET_IS_WINDOWS) + vcpkg_execute_build_process( + COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow + LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + else() + vcpkg_execute_build_process( + COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" "${TF_VERSION}" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow + LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + endif() + endif() + vcpkg_execute_build_process( + COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/generate_static_link_cmd_${PLATFORM_SUFFIX}.py" "${CURRENT_BUILDTREES_DIR}/build-${TARGET_TRIPLET}-${BUILD_TYPE}-err.log" # for some reason stdout of bazel ends up in stderr + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME postbuild2-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + vcpkg_execute_build_process( + COMMAND ${STATIC_LINK_CMD} + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-${TARGET_TRIPLET}-${BUILD_TYPE} + LOGNAME postbuild3-${TARGET_TRIPLET}-${BUILD_TYPE} + ) + endif() + + if(BUILD_TYPE STREQUAL "dbg") + set(DIR_PREFIX "/debug") + else() + set(DIR_PREFIX "") + endif() + + if(VCPKG_TARGET_IS_WINDOWS) + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc.dll DESTINATION ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/bin) + # rename before copy because after copy the file might be locked by anti-malware scanners for some time so that renaming fails + file(RENAME ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc.dll.if.lib ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc.lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc.lib DESTINATION ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib) + if(BUILD_TYPE STREQUAL dbg) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc.pdb DESTINATION ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/bin) + message(STATUS "Warning: debug information tensorflow_cc.pdb will be of limited use because only a reduced set could be produced due to the 4GB internal PDB file limit even on x64.") + endif() + else() + if(BUILD_TYPE STREQUAL dbg) + set(library_parts_variable TF_LIB_PARTS_DEBUG) + else() + set(library_parts_variable TF_LIB_PARTS_RELEASE) + endif() + set(${library_parts_variable}) + + # library might have been split because no more than 4GB are supported even on x64 Windows + foreach(PART_NO RANGE 1 100) + set(source "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/tensorflow_cc-part${PART_NO}.lib") + if(EXISTS "${source}") + file(COPY "${source}" DESTINATION "${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib") + list(APPEND ${library_parts_variable} "tensorflow_cc-part${PART_NO}.lib") + else() + break() + endif() + endforeach() + endif() + else() + file(COPY + ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/${TF_LIB_NAME_FULL} + DESTINATION ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib + ) + + # Note: these use relative links + file(CREATE_LINK ${TF_LIB_NAME_FULL} + ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib/${TF_LIB_NAME_SHORT} + SYMBOLIC + ) + file(CREATE_LINK ${TF_LIB_NAME_FULL} + ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib/${TF_LIB_NAME} + SYMBOLIC + ) + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + file(COPY + ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow/${TF_FRAMEWORK_NAME_FULL} + DESTINATION ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib + ) + file(CREATE_LINK + ${TF_FRAMEWORK_NAME_FULL} + ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib/${TF_FRAMEWORK_NAME_SHORT} + SYMBOLIC + ) + file(CREATE_LINK + ${TF_FRAMEWORK_NAME_FULL} + ${CURRENT_PACKAGES_DIR}${DIR_PREFIX}/lib/${TF_FRAMEWORK_NAME} + SYMBOLIC + ) + endif() + endif() +endforeach() + +file(COPY + ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/include/ + DESTINATION ${CURRENT_PACKAGES_DIR}/include/tensorflow-external +) + +if(VCPKG_LIBRARY_LINKAGE STREQUAL static) + message(STATUS "Warning: Static TensorFlow build contains several external dependencies that may cause linking conflicts (for example, one cannot use both openssl and TensorFlow in the same project, since TensorFlow contains boringssl).") + if(VCPKG_TARGET_IS_WINDOWS) + message(STATUS "Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to tell the linker to include the whole library, i.e., link using options '/WHOLEARCHIVE:tensorflow_cc-part1.lib /WHOLEARCHIVE:tensorflow_cc-part2.lib ...'") + else() + message(STATUS "Note: There is no separate libtensorflow_framework.a as it got merged into libtensorflow_cc.a to avoid linking conflicts.") + if(VCPKG_TARGET_IS_OSX) + message(STATUS "Note: Beside TensorFlow itself, you'll need to also pass its dependancies to the linker, for example '-ltensorflow_cc -framework CoreFoundation'") + message(STATUS "Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to tell the linker to include the whole library: '-Wl,-force_load,path/to/libtensorflow_cc.a -framework CoreFoundation -framework Security [rest of linker arguments]'") + else() + message(STATUS "Note: Beside TensorFlow itself, you'll need to also pass its dependancies to the linker, for example '-ltensorflow_cc -lm -ldl -lpthread'") + message(STATUS "Note: For some TensorFlow features (e.g. OpRegistry), it might be necessary to tell the linker to include the whole library: '-Wl,--whole-archive -ltensorflow_cc -Wl,--no-whole-archive [rest of linker arguments]'") + endif() + endif() + + configure_file( + ${CMAKE_CURRENT_LIST_DIR}/README-${PLATFORM_SUFFIX} + ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/README + COPYONLY) endif() file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc) file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright) -file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc) -file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake) + +# NOTE: if this port ever supports VCPKG_BUILD_TYPE, use that to set these. +set(TENSORFLOW_HAS_RELEASE ON) +set(TENSORFLOW_HAS_DEBUG ON) + +if(VCPKG_TARGET_IS_WINDOWS) + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + configure_file( + ${CMAKE_CURRENT_LIST_DIR}/tensorflow-cc-config-windows-dll.cmake.in + ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/tensorflow-cc-config.cmake + @ONLY) + else() + configure_file( + ${CMAKE_CURRENT_LIST_DIR}/tensorflow-cc-config-windows-lib.cmake.in + ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/tensorflow-cc-config.cmake + @ONLY) + + set(VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES enabled) + + set(prefix [[${TENSORFLOW_INSTALL_PREFIX}]]) + + set(libs_to_link) + foreach(lib IN LISTS TF_LIB_PARTS_RELEASE) + list(APPEND libs_to_link "$<$:${prefix}/lib/${lib}>") + endforeach() + foreach(lib IN LISTS TF_LIB_PARTS_DEBUG) + list(APPEND libs_to_link "$<$:${prefix}/debug/lib/${lib}>") + endforeach() + if(TENSORFLOW_HAS_RELEASE) + set(TF_LIB_PARTS_DEFAULT ${TF_LIB_PARTS_RELEASE}) + set(prefix_DEFAULT "${prefix}") + elseif(TENSORFLOW_HAS_DEBUG) + set(TF_LIB_PARTS_DEFAULT ${TF_LIB_PARTS_DEBUG}) + set(prefix_DEFAULT "${prefix}/debug") + endif() + + foreach(lib IN LISTS TF_LIB_PARTS_DEFAULT) + list(APPEND libs_to_link + "$<$,$>>:${prefix}/lib/${lib}>") + endforeach() + + string(REPLACE ";" "\n\t\t" libs_to_link "${libs_to_link}") + file(APPEND ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/tensorflow-cc-config.cmake " +target_link_libraries(tensorflow_cc::tensorflow_cc + INTERFACE + ${libs_to_link} +)" + ) + endif() +else() + if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + configure_file( + ${CMAKE_CURRENT_LIST_DIR}/tensorflow-cc-config-shared.cmake.in + ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/tensorflow-cc-config.cmake + @ONLY) + else() + configure_file( + ${CMAKE_CURRENT_LIST_DIR}/tensorflow-cc-config-static.cmake.in + ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/tensorflow-cc-config.cmake + @ONLY) + endif() +endif() + +message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} and ${BUILDTREES_DIR}/.bzl to free diskspace.") diff --git a/ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in b/ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in new file mode 100644 index 0000000000..98149e0004 --- /dev/null +++ b/ports/tensorflow-cc/tensorflow-cc-config-shared.cmake.in @@ -0,0 +1,88 @@ +if(TARGET tensorflow_cc::tensorflow_cc OR TARGET tensorflow_cc::tensorflow_framework) + if(NOT (TARGET tensorflow_cc::tensorflow_cc AND TARGET tensorflow_cc::tensorflow_framework)) + message(FATAL_ERROR "Some (but not all) targets in this config.cmake were already defined.") + endif() + + return() +endif() + +add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED GLOBAL) +add_library(tensorflow_cc::tensorflow_framework SHARED IMPORTED GLOBAL) + +# Compute the installation prefix from this tensorflow-cc-config.cmake file location. +# CMAKE_CURRENT_LIST_DIR = [vcpkg_installed_dir]/[target_triplet]/share/tensorflow-cc +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet]/share +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${TENSORFLOW_INSTALL_PREFIX}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet] + +target_include_directories(tensorflow_cc::tensorflow_cc + INTERFACE + "${TENSORFLOW_INSTALL_PREFIX}/include" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external/src" +) +target_include_directories(tensorflow_cc::tensorflow_framework + INTERFACE + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external/src" +) + +if(@TENSORFLOW_HAS_RELEASE@) + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_LIB_NAME@" + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_FRAMEWORK_NAME@" + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION_RELEASE + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_LIB_NAME@" + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + PROPERTY IMPORTED_LOCATION_RELEASE + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_FRAMEWORK_NAME@" + ) +endif() + +if(@TENSORFLOW_HAS_DEBUG) + if(NOT @TENSORFLOW_HAS_RELEASE) + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_LIB_NAME@" + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_FRAMEWORK_NAME@" + ) + endif() + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION_DEBUG + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_LIB_NAME@" + ) + set_property(TARGET tensorflow_cc::tensorflow_framework + PROPERTY IMPORTED_LOCATION_DEBUG + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_FRAMEWORK_NAME@" + ) +endif() + +set(tensorflow_cc_FOUND TRUE) +set(tensorflow_framework_FOUND TRUE) diff --git a/ports/tensorflow-cc/tensorflow-cc-config-static.cmake.in b/ports/tensorflow-cc/tensorflow-cc-config-static.cmake.in new file mode 100644 index 0000000000..7f01714706 --- /dev/null +++ b/ports/tensorflow-cc/tensorflow-cc-config-static.cmake.in @@ -0,0 +1,56 @@ +if(TARGET tensorflow_cc::tensorflow_cc) + return() +endif() + +add_library(tensorflow_cc::tensorflow_cc STATIC IMPORTED GLOBAL) + +# Compute the installation prefix from this tensorflow-cc-config.cmake file location. +# CMAKE_CURRENT_LIST_DIR = [vcpkg_installed_dir]/[target_triplet]/share/tensorflow-cc +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet]/share +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${TENSORFLOW_INSTALL_PREFIX}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet] + +target_include_directories(tensorflow_cc::tensorflow_cc + INTERFACE + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external/src" +) + +if(@TENSORFLOW_HAS_RELEASE@) + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_LIB_NAME@" + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS + RELEASE + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION_RELEASE + "${TENSORFLOW_INSTALL_PREFIX}/lib/@TF_LIB_NAME@" + ) +endif() + +if(@TENSORFLOW_HAS_DEBUG@) + if(NOT @TENSORFLOW_HAS_RELEASE@) + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_LIB_NAME@" + ) + endif() + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS + DEBUG + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + PROPERTY IMPORTED_LOCATION_DEBUG + "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/@TF_LIB_NAME@" + ) +endif() + +set(tensorflow_cc_FOUND TRUE) diff --git a/ports/tensorflow-cc/tensorflow-cc-config-windows-dll.cmake.in b/ports/tensorflow-cc/tensorflow-cc-config-windows-dll.cmake.in new file mode 100644 index 0000000000..ed67e5ad40 --- /dev/null +++ b/ports/tensorflow-cc/tensorflow-cc-config-windows-dll.cmake.in @@ -0,0 +1,56 @@ +if(TARGET tensorflow_cc::tensorflow_cc) + return() +endif() + +add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED GLOBAL) + +# Compute the installation prefix from this tensorflow-cc-config.cmake file location. +# CMAKE_CURRENT_LIST_DIR = [vcpkg_installed_dir]/[target_triplet]/share/tensorflow-cc +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet]/share +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${TENSORFLOW_INSTALL_PREFIX}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet] + +target_include_directories(tensorflow_cc::tensorflow_cc + INTERFACE + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external/src" +) + +if(@TENSORFLOW_HAS_RELEASE@) + set_target_properties(tensorflow_cc::tensorflow_cc + PROPERTIES + IMPORTED_LOCATION "${TENSORFLOW_INSTALL_PREFIX}/bin/tensorflow_cc.dll" + IMPORTED_IMPLIB "${TENSORFLOW_INSTALL_PREFIX}/lib/tensorflow_cc.lib" + ) + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE + ) + set_target_properties(tensorflow_cc::tensorflow_cc + PROPERTIES + IMPORTED_LOCATION_RELEASE "${TENSORFLOW_INSTALL_PREFIX}/bin/tensorflow_cc.dll" + IMPORTED_IMPLIB_RELEASE "${TENSORFLOW_INSTALL_PREFIX}/lib/tensorflow_cc.lib" + ) +endif() + +if(@TENSORFLOW_HAS_DEBUG@) + if(NOT @TENSORFLOW_HAS_RELEASE@) + set_target_properties(tensorflow_cc::tensorflow_cc + PROPERTIES + IMPORTED_LOCATION "${TENSORFLOW_INSTALL_PREFIX}/debug/bin/tensorflow_cc.dll" + IMPORTED_IMPLIB "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/tensorflow_cc.lib" + ) + endif() + + set_property(TARGET tensorflow_cc::tensorflow_cc + APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG + ) + set_target_properties(tensorflow_cc::tensorflow_cc + PROPERTIES + IMPORTED_LOCATION_DEBUG "${TENSORFLOW_INSTALL_PREFIX}/debug/bin/tensorflow_cc.dll" + IMPORTED_IMPLIB_DEBUG "${TENSORFLOW_INSTALL_PREFIX}/debug/lib/tensorflow_cc.lib" + ) +endif() + +set(tensorflow_cc_FOUND TRUE) diff --git a/ports/tensorflow-cc/tensorflow-cc-config-windows-lib.cmake.in b/ports/tensorflow-cc/tensorflow-cc-config-windows-lib.cmake.in new file mode 100644 index 0000000000..0eab84c7f0 --- /dev/null +++ b/ports/tensorflow-cc/tensorflow-cc-config-windows-lib.cmake.in @@ -0,0 +1,19 @@ +if(TARGET tensorflow_cc::tensorflow_cc) + return() +endif() + +set(tensorflow_cc_FOUND TRUE) + +# Compute the installation prefix from this tensorflow-cc-config.cmake file location. +# CMAKE_CURRENT_LIST_DIR = [vcpkg_installed_dir]/[target_triplet]/share/tensorflow-cc +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet]/share +get_filename_component(TENSORFLOW_INSTALL_PREFIX "${TENSORFLOW_INSTALL_PREFIX}" DIRECTORY) +# TENSORFLOW_INSTALL_PREFIX = [vcpkg_installed_dir]/[target_triplet] + +add_library(tensorflow_cc::tensorflow_cc INTERFACE IMPORTED GLOBAL) +target_include_directories(tensorflow_cc::tensorflow_cc + INTERFACE + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external" + "${TENSORFLOW_INSTALL_PREFIX}/include/tensorflow-external/src" +) diff --git a/scripts/build_info.cmake b/scripts/build_info.cmake index 45d81513f4..7470560c85 100644 --- a/scripts/build_info.cmake +++ b/scripts/build_info.cmake @@ -8,6 +8,9 @@ endif() if (DEFINED VCPKG_POLICY_DLLS_WITHOUT_EXPORTS) file(APPEND ${BUILD_INFO_FILE_PATH} "PolicyDLLsWithoutExports: ${VCPKG_POLICY_DLLS_WITHOUT_EXPORTS}\n") endif() +if (DEFINED VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES) + file(APPEND ${BUILD_INFO_FILE_PATH} "PolicyMismatchedNumberOfBinaries: ${VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES}\n") +endif() if (DEFINED VCPKG_POLICY_EMPTY_PACKAGE) file(APPEND ${BUILD_INFO_FILE_PATH} "PolicyEmptyPackage: ${VCPKG_POLICY_EMPTY_PACKAGE}\n") endif() @@ -31,4 +34,4 @@ if (DEFINED VCPKG_POLICY_SKIP_ARCHITECTURE_CHECK) endif() if (DEFINED VCPKG_HEAD_VERSION) file(APPEND ${BUILD_INFO_FILE_PATH} "Version: ${VCPKG_HEAD_VERSION}\n") -endif() \ No newline at end of file +endif() diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 32129df5f7..69f152589e 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1570,12 +1570,6 @@ teemo:arm64-windows=fail teemo:x64-osx=fail telnetpp:arm-uwp=fail telnetpp:x64-uwp=fail -tensorflow-cc:arm64-windows=fail -tensorflow-cc:x64-linux=skip -tensorflow-cc:x64-osx=skip -tensorflow-cc:x64-windows=fail -tensorflow-cc:x64-windows-static=fail -tensorflow-cc:x86-windows=fail tesseract:x64-windows=fail tesseract:x64-windows-static=fail tesseract:x86-windows=fail diff --git a/scripts/cmake/vcpkg_acquire_msys.cmake b/scripts/cmake/vcpkg_acquire_msys.cmake index f0a8b7e0ea..b502eab6a3 100644 --- a/scripts/cmake/vcpkg_acquire_msys.cmake +++ b/scripts/cmake/vcpkg_acquire_msys.cmake @@ -304,6 +304,155 @@ function(vcpkg_acquire_msys PATH_TO_ROOT_OUT) SHA512 f094a7f4926195ef7ba015f0c5c56587b1faa94d85530f07aaaa5557a1494c3bd75257d4687c8401cbf1328d23e5586a92b05f0a872caebb1a7e941a07829776 ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-python-numpy-1.19.0-1-any.pkg.tar.zst" + SHA512 15791fff23deda17a4452c9ca3f23210ed77ee20dcdd6e0c31d0e626a63aeb93d15ed814078729101f1cce96129b4b5e3c898396b003d794a52d7169dd027465 + DEPS mingw-w64-x86_64-openblas mingw-w64-x86_64-python + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-openblas-0.3.10-2-any.pkg.tar.zst" + SHA512 3cf15ef191ceb303a7e40ad98aca94c56211b245617c17682379b5606a1a76e12d04fa1a83c6109e89620200a74917bcd981380c7749dda12fa8e79f0b923877 + DEPS mingw-w64-x86_64-gcc-libgfortran mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-libwinpthread + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libgfortran-10.2.0-1-any.pkg.tar.zst" + SHA512 c2dee2957356fa51aae39d907d0cc07f966028b418f74a1ea7ea551ff001c175d86781f980c0cf994207794322dcd369fa122ab78b6c6d0f0ab01e39a754e780 + DEPS mingw-w64-x86_64-gcc-libs + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-python-3.8.5-1-any.pkg.tar.zst" + SHA512 49bbcaa9479ff95fd21b473a1bc286886b204ec3e2e0d9466322e96a9ee07ccd8116024b54b967a87e4752057004475cac5060605e87bd5057de45efe5122a25 + DEPS mingw-w64-x86_64-bzip2 mingw-w64-x86_64-expat mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-libffi mingw-w64-x86_64-mpdecimal mingw-w64-x86_64-ncurses mingw-w64-x86_64-openssl mingw-w64-x86_64-sqlite3 mingw-w64-x86_64-tcl mingw-w64-x86_64-tk mingw-w64-x86_64-xz mingw-w64-x86_64-zlib + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-bzip2-1.0.8-1-any.pkg.tar.xz" + SHA512 6e01b26a2144f99ca00406dbce5b8c3e928ec8a3ff77e0b741b26aaf9c927e9bea8cb1b5f38cd59118307e10dd4523a0ea2a1ea61f798f99e6d605ef1d100503 + DEPS mingw-w64-x86_64-gcc-libs + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-mpdecimal-2.5.0-1-any.pkg.tar.zst" + SHA512 48130ff676c0235bad4648527021e597ee00aa49a4443740a134005877e2ff2ca27b30a0ac86b923192a65348b36de4e8d3f9c57d76ab42b2e21d1a92dbf7ccf + DEPS mingw-w64-x86_64-gcc-libs + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-ncurses-6.2-1-any.pkg.tar.xz" + SHA512 1cbffe0e181a3d4ceaa8f39b2a649584b2c7d689e6a057d85cb9f84edece2cf60eddc220127c7fa4f29e4aa6e8fb4f568ef9d73582d08168607135af977407e0 + DEPS mingw-w64-x86_64-libsystre + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libsystre-1.0.1-4-any.pkg.tar.xz" + SHA512 6540e896636d00d1ea4782965b3fe4d4ef1e32e689a98d25e2987191295b319eb1de2e56be3a4b524ff94f522a6c3e55f8159c1a6f58c8739e90f8e24e2d40d8 + DEPS mingw-w64-x86_64-libtre + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libtre-git-r128.6fb7206-2-any.pkg.tar.xz" + NAME mingw-w64-x86_64-libtre + VERSION git-r128.6fb7206-2 + ANY + REPO mingw/x86_64 + SHA512 d595dbcf3a3b6ed098e46f370533ab86433efcd6b4d3dcf00bbe944ab8c17db7a20f6535b523da43b061f071a3b8aa651700b443ae14ec752ae87500ccc0332d + DEPS mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-gettext + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-openssl-1.1.1.g-1-any.pkg.tar.xz" + SHA512 81681089a19cae7dbdee1bc9d3148f03458fa7a1d2fd105be39299b3a0c91b34450bcfe2ad86622bc6819da1558d7217deb0807b4a7bed942a9a7a786fcd54a3 + DEPS mingw-w64-x86_64-ca-certificates mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-zlib + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-ca-certificates-20200601-1-any.pkg.tar.zst" + SHA512 21a81e1529a3ad4f6eceb3b7d4e36400712d3a690d3991131573d4aae8364965757f9b02054d93c853eb75fbb7f6173a278b122450c800b2c9a1e8017dd35e28 + DEPS mingw-w64-x86_64-p11-kit + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-p11-kit-0.23.20-2-any.pkg.tar.xz" + SHA512 c441c4928465a98aa53917df737b728275bc0f6e9b41e13de7c665a37d2111b46f057bb652a1d5a6c7cdf8a74ea15e365a727671b698f5bbb5a7cfd0b889935e + DEPS mingw-w64-x86_64-gettext mingw-w64-x86_64-libffi mingw-w64-x86_64-libtasn1 + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libtasn1-4.16.0-1-any.pkg.tar.xz" + SHA512 c450cd49391b46af552a89f2f6e2c21dd5da7d40e7456b380290c514a0f06bcbd63f0f972b3c173c4237bec7b652ff22d2d330e8fdf5c888558380bd2667be64 + DEPS mingw-w64-x86_64-gcc-libs + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-sqlite3-3.33.0-1-any.pkg.tar.zst" + SHA512 eae319f87c9849049347f132efc2ecc46e9ac1ead55542e31a3ea216932a4fa5c5bae8d468d2f050e1e22068ac9fbe9d8e1aa7612cc0110cafe6605032adeb0f + DEPS mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-readline mingw-w64-x86_64-tcl + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-readline-8.0.004-1-any.pkg.tar.xz" + SHA512 e3fb3030a50f677697bec0da39ba2eb979dc28991ad0e29012cbf1bda82723176148510bf924b7fce7a0b79e7b078232d69e07f3fbb7d657b8ee631841730120 + DEPS mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-termcap + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-termcap-1.3.1-6-any.pkg.tar.zst" + SHA512 602d182ba0f1e20c4c51ae09b327c345bd736e6e4f22cd7d58374ac68c705dd0af97663b9b94d41870457f46bb9110abb29186d182196133618fc460f71d1300 + DEPS mingw-w64-x86_64-gcc-libs + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-tk-8.6.10-1-any.pkg.tar.xz" + SHA512 3be88b87d5e77a875ea98f0bce4192242e550eeb1b0d44abfee9c8797135a45dd3219b89006de99458dd3f9ae47da77dccc63dab25cea93fbc285af756264eb8 + DEPS mingw-w64-x86_64-tcl + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-tcl-8.6.10-1-any.pkg.tar.xz" + SHA512 c3f21588e19725598878ef13145fbe7a995c2a0c678ef0a4782e28fd64d65fe3271178369bf0c54e92123eba82f2d3da6ae2fc34acd3b20150d1e173be1c0f8f + DEPS mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-zlib + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-xz-5.2.5-1-any.pkg.tar.xz" + SHA512 0e1336a1565cda6e78996d69ba973aaa3522392ab586f70b0b93dbe09be50baf3e14f8ba0afcc665bc885508f1a898b16f206f89eaa3cbc9985afeea6ff1c02b + DEPS mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-gettext + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gettext-0.19.8.1-9-any.pkg.tar.zst" + SHA512 571a36cf60e40172aaa7a5a40b1db60bbea145d9f399603a625a57ca106679f6feb53fda73d935ce8f0057935cad5b9a8770ae4f065e54e1554a1932b48eec97 + DEPS mingw-w64-x86_64-expat mingw-w64-x86_64-gcc-libs mingw-w64-x86_64-libiconv + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libs-10.2.0-1-any.pkg.tar.zst" + SHA512 d17eff08c83d08ef020d999a2ead0d25036ada1c1bf6ed7c02bad9b56840ee5a3304acd790d86f52b83b09c1e788f0cecdf7254dc6760c3c7e478f65882cd32d + DEPS mingw-w64-x86_64-gmp mingw-w64-x86_64-libwinpthread mingw-w64-x86_64-mpc mingw-w64-x86_64-mpfr + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-mpc-1.2.0-1-any.pkg.tar.zst" + SHA512 e2e561ef7c1bd85bbf021ecbe4df1cfd377a5b426ec0091f267111b9f18d476d5f95a40e0ffbd97aee5f331c49dc7a8dfc2111d54cc979643fae30e564d671aa + DEPS mingw-w64-x86_64-mpfr + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-mpfr-4.1.0-2-any.pkg.tar.zst" + SHA512 14739667242b8852f0d26547eb3297899a51fd1edafc7101b4e7489273e1efb9cb8422fc067361e3c3694c2afcc6c49fc89537f9f811ad5b9b595873112ee890 + DEPS mingw-w64-x86_64-gmp + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gmp-6.2.0-1-any.pkg.tar.xz" + SHA512 0b22b7363e27cec706eb79ee0c45b5fe7088a5ca69e0868e7366481ed2ea9b3f6623d340cebba0b5ed3d79e4dfc7cf15f53530eb260c6d4057bfc3d92eb8c7bc + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-expat-2.2.9-1-any.pkg.tar.xz" + SHA512 1f747b9c7e6ee680b6d8f76429e81a42e2d4ab72d5d930207c90f4513cca5158c08c8296889fd27fe07a275cdeff5d34b5de0e6d1cd982d2e1d05765d6c8c31a + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libffi-3.3-1-any.pkg.tar.xz" + SHA512 90451ac2dadcd3f1310b6af977d4c56d239500743a3d67e4f8df915e6e6f65f34d4244843d8bac5718642973be5312c17cb3fb0b4c64732cda06437e9f1ce86d + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libiconv-1.16-1-any.pkg.tar.xz" + SHA512 c8e2fda532c753e0b1004596bf737c3669355f32af9b45d96c23fcef14994ba21ddf4f75138bdecc94cbf8a8c449eff530d24b74a0da47793e24ce92d154f411 + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-zlib-1.2.11-7-any.pkg.tar.xz" + SHA512 bbd4a549efc2a5f4b1e9f1be00331e8726d80401a9c6117afa9d5dd92f4ac42a06cf2ce491a988e5c6ed7a6e536f8f1746081f4944bc6d473ccd16390fea27fe + ) + msys_package( + URL "https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libwinpthread-git-8.0.0.5906.c9a21571-1-any.pkg.tar.zst" + NAME mingw-w64-x86_64-libwinpthread + VERSION git-8.0.0.5906.c9a21571-1 + ANY + ZST + REPO mingw/x86_64 + SHA512 a6969a5db1c55ba458c1a047d0a2a9d2db6cc24266ea47f740598b149a601995d2de734a0984ac5e57ee611d5982cbc03fd6fc0f498435e8d6401bf15724caad + ) + if(PACKAGES) message(FATAL_ERROR "Unknown packages were required for vcpkg_acquire_msys(${_am_PACKAGES}): ${PACKAGES}\nThis can be resolved by explicitly passing URL/SHA pairs to DIRECT_PACKAGES.") endif() diff --git a/scripts/cmake/vcpkg_find_acquire_program.cmake b/scripts/cmake/vcpkg_find_acquire_program.cmake index 131e7b4bf0..7d12e80840 100644 --- a/scripts/cmake/vcpkg_find_acquire_program.cmake +++ b/scripts/cmake/vcpkg_find_acquire_program.cmake @@ -385,7 +385,7 @@ function(vcpkg_find_acquire_program VAR) set(SUBDIR 6bac47ec55-25c819fd77) elseif(VAR MATCHES "BAZEL") set(PROGNAME bazel) - set(BAZEL_VERSION 0.25.2) + set(BAZEL_VERSION 3.7.0) set(_vfa_RENAME "bazel") if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") set(_vfa_SUPPORTED ON) @@ -393,19 +393,19 @@ function(vcpkg_find_acquire_program VAR) set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${SUBDIR}-x86_64") set(ARCHIVE "bazel-${SUBDIR}-x86_64") set(NOEXTRACT ON) - set(HASH db4a583cf2996aeb29fd008261b12fe39a4a5faf0fbf96f7124e6d3ffeccf6d9655d391378e68dd0915bc91c9e146a51fd9661963743857ca25179547feceab1) + set(HASH 1118eb939627cc5570616f7bd41c72a90df9bb4a3c802eb8149b5b2eebf27090535c029590737557e270c5a8556267b8c1843eb0ff55dc9e4b82581a64e07ec1) elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(_vfa_SUPPORTED ON) set(SUBDIR ${BAZEL_VERSION}-darwin) set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${SUBDIR}-x86_64") set(ARCHIVE "bazel-${SUBDIR}-x86_64") set(NOEXTRACT ON) - set(HASH 420a37081e6ee76441b0d92ff26d1715ce647737ce888877980d0665197b5a619d6afe6102f2e7edfb5062c9b40630a10b2539585e35479b780074ada978d23c) + set(HASH e2d792f0fc03a4a57a4c2c8345141d86a2dc25a09757f26cb18534426f73d10b4de021e2a3d439956a92d2a712aae9ad75357db24d02f9b0890cc643615a997c) else() set(SUBDIR ${BAZEL_VERSION}-windows) set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${SUBDIR}-x86_64.zip") set(ARCHIVE "bazel-${SUBDIR}-x86_64.zip") - set(HASH 6482f99a0896f55ef65739e7b53452fd9c0adf597b599d0022a5e0c5fa4374f4a958d46f98e8ba25af4b065adacc578bfedced483d8c169ea5cb1777a99eea53) + set(HASH 410b6788f624b3b0b9f13f5b4d12c1b24447f133210a68e2f110aff8d95bb954e40ea1d863a8cc3473402d1c2f15c38042e6af0cb207056811e4cc7bd0b9ca00) endif() elseif(VAR MATCHES "ARIA2") set(PROGNAME aria2c) diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index 86c0faff15..4f832d8776 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2020.06.15" +"2020.11.09" diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 3e56101c9e..b05f79a05b 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -18,10 +18,17 @@ namespace fs { +#if defined(_WIN32) struct IsSlash { bool operator()(const wchar_t c) const noexcept { return c == L'/' || c == L'\\'; } }; +#else + struct IsSlash + { + bool operator()(const char c) const noexcept { return c == '/'; } + }; +#endif constexpr IsSlash is_slash; diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 4cbde1cb1a..f15a2f724a 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -243,6 +243,7 @@ namespace vcpkg::Build EMPTY_PACKAGE, DLLS_WITHOUT_LIBS, DLLS_WITHOUT_EXPORTS, + MISMATCHED_NUMBER_OF_BINARIES, ONLY_RELEASE_CRT, EMPTY_INCLUDE_FOLDER, ALLOW_OBSOLETE_MSVCRT, @@ -253,16 +254,8 @@ namespace vcpkg::Build COUNT, }; - constexpr std::array G_ALL_POLICIES = { - BuildPolicy::EMPTY_PACKAGE, - BuildPolicy::DLLS_WITHOUT_LIBS, - BuildPolicy::DLLS_WITHOUT_EXPORTS, - BuildPolicy::ONLY_RELEASE_CRT, - BuildPolicy::EMPTY_INCLUDE_FOLDER, - BuildPolicy::ALLOW_OBSOLETE_MSVCRT, - BuildPolicy::ALLOW_RESTRICTED_HEADERS, - BuildPolicy::SKIP_DUMPBIN_CHECKS, - BuildPolicy::SKIP_ARCHITECTURE_CHECK}; + // could be constexpr, but we want to generate this and that's not constexpr + extern const std::array ALL_POLICIES; const std::string& to_string(BuildPolicy policy); CStringView to_cmake_variable(BuildPolicy policy); diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index 4cccebd459..891207445f 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -26,20 +26,6 @@ namespace { -#if defined(_WIN32) - struct IsSlash - { - bool operator()(const wchar_t c) const noexcept { return c == L'/' || c == L'\\'; } - }; -#else - struct IsSlash - { - bool operator()(const char c) const noexcept { return c == '/'; } - }; -#endif - - constexpr IsSlash is_slash; - struct NativeStringView { const fs::path::value_type* first; diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index c20f48b361..bae9372775 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -203,6 +203,7 @@ namespace vcpkg::Build static const std::string NAME_EMPTY_PACKAGE = "PolicyEmptyPackage"; static const std::string NAME_DLLS_WITHOUT_LIBS = "PolicyDLLsWithoutLIBs"; static const std::string NAME_DLLS_WITHOUT_EXPORTS = "PolicyDLLsWithoutExports"; + static const std::string NAME_MISMATCHED_NUMBER_OF_BINARIES = "PolicyMismatchedNumberOfBinaries"; static const std::string NAME_ONLY_RELEASE_CRT = "PolicyOnlyReleaseCRT"; static const std::string NAME_EMPTY_INCLUDE_FOLDER = "PolicyEmptyIncludeFolder"; static const std::string NAME_ALLOW_OBSOLETE_MSVCRT = "PolicyAllowObsoleteMsvcrt"; @@ -210,6 +211,19 @@ namespace vcpkg::Build static const std::string NAME_SKIP_DUMPBIN_CHECKS = "PolicySkipDumpbinChecks"; static const std::string NAME_SKIP_ARCHITECTURE_CHECK = "PolicySkipArchitectureCheck"; + static std::remove_const_t generate_all_policies() + { + std::remove_const_t res; + for (size_t i = 0; i < res.size(); ++i) + { + res[i] = static_cast(i); + } + + return res; + } + + decltype(ALL_POLICIES) ALL_POLICIES = generate_all_policies(); + const std::string& to_string(BuildPolicy policy) { switch (policy) @@ -217,6 +231,7 @@ namespace vcpkg::Build case BuildPolicy::EMPTY_PACKAGE: return NAME_EMPTY_PACKAGE; case BuildPolicy::DLLS_WITHOUT_LIBS: return NAME_DLLS_WITHOUT_LIBS; case BuildPolicy::DLLS_WITHOUT_EXPORTS: return NAME_DLLS_WITHOUT_EXPORTS; + case BuildPolicy::MISMATCHED_NUMBER_OF_BINARIES: return NAME_MISMATCHED_NUMBER_OF_BINARIES; case BuildPolicy::ONLY_RELEASE_CRT: return NAME_ONLY_RELEASE_CRT; case BuildPolicy::EMPTY_INCLUDE_FOLDER: return NAME_EMPTY_INCLUDE_FOLDER; case BuildPolicy::ALLOW_OBSOLETE_MSVCRT: return NAME_ALLOW_OBSOLETE_MSVCRT; @@ -234,6 +249,7 @@ namespace vcpkg::Build case BuildPolicy::EMPTY_PACKAGE: return "VCPKG_POLICY_EMPTY_PACKAGE"; case BuildPolicy::DLLS_WITHOUT_LIBS: return "VCPKG_POLICY_DLLS_WITHOUT_LIBS"; case BuildPolicy::DLLS_WITHOUT_EXPORTS: return "VCPKG_POLICY_DLLS_WITHOUT_EXPORTS"; + case BuildPolicy::MISMATCHED_NUMBER_OF_BINARIES: return "VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES"; case BuildPolicy::ONLY_RELEASE_CRT: return "VCPKG_POLICY_ONLY_RELEASE_CRT"; case BuildPolicy::EMPTY_INCLUDE_FOLDER: return "VCPKG_POLICY_EMPTY_INCLUDE_FOLDER"; case BuildPolicy::ALLOW_OBSOLETE_MSVCRT: return "VCPKG_POLICY_ALLOW_OBSOLETE_MSVCRT"; @@ -1221,7 +1237,7 @@ namespace vcpkg::Build if (!version.empty()) build_info.version = std::move(version); std::map policies; - for (auto policy : G_ALL_POLICIES) + for (auto policy : ALL_POLICIES) { const auto setting = parser.optional_field(to_string(policy)); if (setting.empty()) continue; diff --git a/toolsrc/src/vcpkg/postbuildlint.cpp b/toolsrc/src/vcpkg/postbuildlint.cpp index 7b8e65ed6e..8490ac3649 100644 --- a/toolsrc/src/vcpkg/postbuildlint.cpp +++ b/toolsrc/src/vcpkg/postbuildlint.cpp @@ -872,7 +872,7 @@ namespace vcpkg::PostBuildLint std::vector release_libs = fs.get_files_recursive(release_lib_dir); Util::erase_remove_if(release_libs, not_extension_pred(fs, ".lib")); - if (!pre_build_info.build_type) + if (!pre_build_info.build_type && !build_info.policies.is_enabled(BuildPolicy::MISMATCHED_NUMBER_OF_BINARIES)) error_count += check_matching_debug_and_release_binaries(debug_libs, release_libs); if (!build_info.policies.is_enabled(BuildPolicy::SKIP_ARCHITECTURE_CHECK)) @@ -892,7 +892,8 @@ namespace vcpkg::PostBuildLint { case Build::LinkageType::DYNAMIC: { - if (!pre_build_info.build_type) + if (!pre_build_info.build_type && + !build_info.policies.is_enabled(BuildPolicy::MISMATCHED_NUMBER_OF_BINARIES)) error_count += check_matching_debug_and_release_binaries(debug_dlls, release_dlls); error_count += check_lib_files_are_available_if_dlls_are_available(