From 248169d8eea9bdcfe9b56f4f0dfaaedae0aaa760 Mon Sep 17 00:00:00 2001 From: Koby Kahane Date: Tue, 13 Oct 2020 03:13:33 +0300 Subject: [PATCH] [rocksdb] update to v6.11.4 (#13643) --- ports/rocksdb/0001-disable-gtest.patch | 16 +- .../rocksdb/0002-only-build-one-flavor.patch | 41 +---- ports/rocksdb/0003-use-find-package.patch | 16 +- .../0004-add-config-to-findpackage.patch | 13 ++ .../0004-static-linking-in-linux.patch | 18 -- .../0005-add-config-to-findpackage.patch | 34 ---- .../0005-backport-msvc-fixes-pr-7439.patch | 169 ++++++++++++++++++ ports/rocksdb/CONTROL | 2 +- ports/rocksdb/portfile.cmake | 17 +- 9 files changed, 216 insertions(+), 110 deletions(-) create mode 100644 ports/rocksdb/0004-add-config-to-findpackage.patch delete mode 100644 ports/rocksdb/0004-static-linking-in-linux.patch delete mode 100644 ports/rocksdb/0005-add-config-to-findpackage.patch create mode 100644 ports/rocksdb/0005-backport-msvc-fixes-pr-7439.patch diff --git a/ports/rocksdb/0001-disable-gtest.patch b/ports/rocksdb/0001-disable-gtest.patch index b3aa2d6294..b17a8b9634 100644 --- a/ports/rocksdb/0001-disable-gtest.patch +++ b/ports/rocksdb/0001-disable-gtest.patch @@ -1,13 +1,13 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index eebda35e9..c00e6b702 100644 +index f9c8f3346..248c0d2d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -888,7 +888,7 @@ option(WITH_TESTS "build with tests" ON) - # For test libraries, utilities, and exes that are build iff WITH_TESTS=ON and - # in Debug mode. Add test only code that is not #ifdefed for Release here. - if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug") +@@ -975,7 +975,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS) + endif() + + if(WITH_TESTS) - add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest) + #add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest) - set(TESTS - cache/cache_test.cc - cache/lru_cache_test.cc + add_library(testharness STATIC + test_util/testharness.cc) + target_link_libraries(testharness gtest) diff --git a/ports/rocksdb/0002-only-build-one-flavor.patch b/ports/rocksdb/0002-only-build-one-flavor.patch index 668a36caea..ec1051ac60 100644 --- a/ports/rocksdb/0002-only-build-one-flavor.patch +++ b/ports/rocksdb/0002-only-build-one-flavor.patch @@ -1,10 +1,10 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index eebda35e9..abaf2fd00 100644 +index f9c8f3346..74c2d263c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -850,23 +850,30 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS) +@@ -938,13 +938,17 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS) - install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination}) - install( - TARGETS ${ROCKSDB_STATIC_LIB} @@ -13,40 +13,17 @@ index eebda35e9..abaf2fd00 100644 - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" - ) -- -- install( -- TARGETS ${ROCKSDB_SHARED_LIB} -- EXPORT RocksDBTargets -- COMPONENT runtime -- ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" -- RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -- LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" -- INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" -- ) -+ if(NOT ROCKSDB_DISABLE_INSTALL_STATIC_LIB) -+ install( -+ TARGETS ${ROCKSDB_STATIC_LIB} -+ EXPORT RocksDBTargets -+ COMPONENT devel -+ ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" -+ INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" -+ ) -+ else() -+ set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES EXCLUDE_FROM_ALL TRUE) -+ endif() -+ if(NOT ROCKSDB_DISABLE_INSTALL_SHARED_LIB) ++ if(NOT ROCKSDB_BUILD_SHARED) + install( -+ TARGETS ${ROCKSDB_SHARED_LIB} ++ TARGETS ${ROCKSDB_STATIC_LIB} + EXPORT RocksDBTargets -+ COMPONENT runtime ++ COMPONENT devel + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" -+ RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + ) + else() -+ set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES EXCLUDE_FROM_ALL TRUE) ++ set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES EXCLUDE_FROM_ALL TRUE) + endif() - install( - EXPORT RocksDBTargets + if(ROCKSDB_BUILD_SHARED) + install( diff --git a/ports/rocksdb/0003-use-find-package.patch b/ports/rocksdb/0003-use-find-package.patch index 5fa21dacbc..093b9c889b 100644 --- a/ports/rocksdb/0003-use-find-package.patch +++ b/ports/rocksdb/0003-use-find-package.patch @@ -1,13 +1,13 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index eebda35e9..cf48c7ad5 100644 +index f9c8f3346..59747a684 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -67,7 +67,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows") - else() - option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF) - endif() +@@ -91,7 +91,7 @@ include(CMakeDependentOption) + CMAKE_DEPENDENT_OPTION(WITH_GFLAGS "build with GFlags" ON + "NOT MSVC;NOT MINGW" OFF) + -if(MSVC) +if(MSVC AND NOT VCPKG_TOOLCHAIN) - # Defaults currently different for GFLAGS. - # We will address find_package work a little later - option(WITH_GFLAGS "build with GFlags" OFF) + option(WITH_XPRESS "build with windows built in compression" OFF) + include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc) + else() diff --git a/ports/rocksdb/0004-add-config-to-findpackage.patch b/ports/rocksdb/0004-add-config-to-findpackage.patch new file mode 100644 index 0000000000..68df561e4c --- /dev/null +++ b/ports/rocksdb/0004-add-config-to-findpackage.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f9c8f3346..11aa70db9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -154,7 +154,7 @@ else() + endif() + + if(WITH_LZ4) +- find_package(lz4 REQUIRED) ++ find_package(lz4 CONFIG REQUIRED) + add_definitions(-DLZ4) + list(APPEND THIRDPARTY_LIBS lz4::lz4) + endif() diff --git a/ports/rocksdb/0004-static-linking-in-linux.patch b/ports/rocksdb/0004-static-linking-in-linux.patch deleted file mode 100644 index afa4fd326c..0000000000 --- a/ports/rocksdb/0004-static-linking-in-linux.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index eebda35e9..53ec5fced 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -781,7 +781,12 @@ if(WIN32) - set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) - else() - set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT}) -- set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) -+ -+ if(ROCKSDB_DISABLE_INSTALL_SHARED_LIB) -+ set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) -+ else() -+ set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) -+ endif() - - add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES}) - target_link_libraries(${ROCKSDB_SHARED_LIB} diff --git a/ports/rocksdb/0005-add-config-to-findpackage.patch b/ports/rocksdb/0005-add-config-to-findpackage.patch deleted file mode 100644 index 47cc22202f..0000000000 --- a/ports/rocksdb/0005-add-config-to-findpackage.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index eebda35e9..20bafd659 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -98,9 +98,9 @@ else() - endif() - - if(WITH_SNAPPY) -- find_package(snappy REQUIRED) -+ find_package(Snappy CONFIG REQUIRED) - add_definitions(-DSNAPPY) -- list(APPEND THIRDPARTY_LIBS snappy::snappy) -+ list(APPEND THIRDPARTY_LIBS Snappy::snappy) - endif() - - if(WITH_ZLIB) -@@ -122,7 +122,7 @@ else() - endif() - - if(WITH_LZ4) -- find_package(lz4 REQUIRED) -+ find_package(lz4 CONFIG REQUIRED) - add_definitions(-DLZ4) - list(APPEND THIRDPARTY_LIBS lz4::lz4) - endif() -@@ -131,7 +131,7 @@ else() - find_package(zstd REQUIRED) - add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) -- list(APPEND THIRDPARTY_LIBS zstd::zstd) -+ list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) - endif() - endif() - diff --git a/ports/rocksdb/0005-backport-msvc-fixes-pr-7439.patch b/ports/rocksdb/0005-backport-msvc-fixes-pr-7439.patch new file mode 100644 index 0000000000..57b889bb0b --- /dev/null +++ b/ports/rocksdb/0005-backport-msvc-fixes-pr-7439.patch @@ -0,0 +1,169 @@ +diff --git a/file/filename.cc b/file/filename.cc +index 968adbaa7..0ac4e6ef8 100644 +--- a/file/filename.cc ++++ b/file/filename.cc +@@ -461,8 +461,8 @@ Status GetInfoLogFiles(Env* env, const std::string& db_log_dir, + std::string NormalizePath(const std::string& path) { + std::string dst; + for (auto c : path) { +- if (!dst.empty() && c == kFilePathSeparator && +- dst.back() == kFilePathSeparator) { ++ if (!dst.empty() && (c == kFilePathSeparator || c == '/') && ++ (dst.back() == kFilePathSeparator || dst.back() == '/')) { + continue; + } + dst.push_back(c); +diff --git a/port/win/port_win.cc b/port/win/port_win.cc +index 2d99a7a9b..34f6b28de 100644 +--- a/port/win/port_win.cc ++++ b/port/win/port_win.cc +@@ -100,6 +100,14 @@ bool CondVar::TimedWait(uint64_t abs_time_us) { + + // Caller must ensure that mutex is held prior to calling this method + std::unique_lock lk(mu_->getLock(), std::adopt_lock); ++ ++ // Work around https://github.com/microsoft/STL/issues/369 ++#if defined(_MSC_VER) && (!defined(_MSVC_STL_UPDATE) || _MSVC_STL_UPDATE < 202008L) ++ if (relTimeUs == microseconds::zero()) { ++ lk.unlock(); ++ lk.lock(); ++ } ++#endif + #ifndef NDEBUG + mu_->locked_ = false; + #endif +diff --git a/port/win/port_win.h b/port/win/port_win.h +index a3ffd559c..2c5b8ff05 100644 +--- a/port/win/port_win.h ++++ b/port/win/port_win.h +@@ -283,7 +283,7 @@ extern const size_t kPageSize; + #endif + + static inline void AsmVolatilePause() { +-#if defined(_M_IX86) || defined(_M_X64) ++#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) + YieldProcessor(); + #endif + // it would be nice to get "wfe" on ARM here +diff --git a/third-party/folly/folly/chrono/Hardware.h b/third-party/folly/folly/chrono/Hardware.h +index ec7be82e8..6635b8717 100644 +--- a/third-party/folly/folly/chrono/Hardware.h ++++ b/third-party/folly/folly/chrono/Hardware.h +@@ -10,7 +10,7 @@ + #include + #include + +-#if _MSC_VER ++#if _MSC_VER && (defined(_M_IX86) || defined(_M_X64)) + extern "C" std::uint64_t __rdtsc(); + #pragma intrinsic(__rdtsc) + #endif +@@ -18,7 +18,7 @@ extern "C" std::uint64_t __rdtsc(); + namespace folly { + + inline std::uint64_t hardware_timestamp() { +-#if _MSC_VER ++#if _MSC_VER && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); + #elif __GNUC__ && (__i386__ || FOLLY_X64) + return __builtin_ia32_rdtsc(); +diff --git a/util/math.h b/util/math.h +index 64cdb2f44..514943164 100644 +--- a/util/math.h ++++ b/util/math.h +@@ -13,24 +13,75 @@ + + namespace ROCKSDB_NAMESPACE { + ++#if defined(_MSC_VER) && !defined(_M_X64) ++namespace detail { ++template ++int BitsSetToOneFallback(T v) { ++ const int kBits = static_cast(sizeof(T)) * 8; ++ static_assert((kBits & (kBits - 1)) == 0, "must be power of two bits"); ++ // we static_cast these bit patterns in order to truncate them to the correct size ++ v = static_cast(v - ((v >> 1) & static_cast(0x5555555555555555ull))); ++ v = static_cast((v & static_cast(0x3333333333333333ull)) + ++ ((v >> 2) & static_cast(0x3333333333333333ull))); ++ v = static_cast((v + (v >> 4)) & static_cast(0x0F0F0F0F0F0F0F0Full)); ++ for (int shift_bits = 8; shift_bits < kBits; shift_bits <<= 1) { ++ v += static_cast(v >> shift_bits); ++ } ++ // we want the bottom "slot" that's big enough to store kDigits ++ // (and including) kBits. ++ return static_cast(v & static_cast(kBits | (kBits - 1))); ++} ++ ++} // namespace detail ++#endif ++ ++// Number of bits set to 1. Also known as "population count". + template + inline int BitsSetToOne(T v) { + static_assert(std::is_integral::value, "non-integral type"); + #ifdef _MSC_VER + static_assert(sizeof(T) <= sizeof(uint64_t), "type too big"); +- if (sizeof(T) > sizeof(uint32_t)) { +- return static_cast(__popcnt64(static_cast(v))); +- } else { ++ if (sizeof(T) < sizeof(uint32_t)) { ++ // This bit mask is to avoid a compiler warning on unused path ++ constexpr auto mm = 8 * sizeof(uint32_t) - 1; ++ // The bit mask is to neutralize sign extension on small signed types ++ constexpr uint32_t m = (uint32_t{1} << ((8 * sizeof(T)) & mm)) - 1; ++#if defined(_M_X64) || defined(_M_IX86) ++ return static_cast(__popcnt(static_cast(v) & m)); ++#else ++ return static_cast(detail::BitsSetToOneFallback(v) & m); ++#endif ++ } else if (sizeof(T) == sizeof(uint32_t)) { ++#if defined(_M_X64) || defined(_M_IX86) + return static_cast(__popcnt(static_cast(v))); ++#else ++ return detail::BitsSetToOneFallback(static_cast(v)); ++#endif ++ } else { ++#ifdef _M_X64 ++ return static_cast(__popcnt64(static_cast(v))); ++#elif defined(_M_IX86) ++ return static_cast( ++ __popcnt(static_cast(static_cast(v) >> 32) + ++ __popcnt(static_cast(v)))); ++#else ++ return detail::BitsSetToOneFallback(static_cast(v)); ++#endif + } + #else + static_assert(sizeof(T) <= sizeof(unsigned long long), "type too big"); +- if (sizeof(T) > sizeof(unsigned long)) { +- return __builtin_popcountll(static_cast(v)); +- } else if (sizeof(T) > sizeof(unsigned int)) { ++ if (sizeof(T) < sizeof(unsigned int)) { ++ // This bit mask is to avoid a compiler warning on unused path ++ constexpr auto mm = 8 * sizeof(unsigned int) - 1; ++ // This bit mask is to neutralize sign extension on small signed types ++ constexpr unsigned int m = (1U << ((8 * sizeof(T)) & mm)) - 1; ++ return __builtin_popcount(static_cast(v) & m); ++ } else if (sizeof(T) == sizeof(unsigned int)) { ++ return __builtin_popcount(static_cast(v)); ++ } else if (sizeof(T) <= sizeof(unsigned long)) { + return __builtin_popcountl(static_cast(v)); + } else { +- return __builtin_popcount(static_cast(v)); ++ return __builtin_popcountll(static_cast(v)); + } + #endif + } +diff --git a/util/xxh3p.h b/util/xxh3p.h +index d1fc2bba2..c63f0e36d 100644 +--- a/util/xxh3p.h ++++ b/util/xxh3p.h +@@ -253,7 +253,7 @@ XXH_FORCE_INLINE U64x2 XXH_vec_mule(U32x4 a, U32x4 b) { + #if defined(XXH_NO_PREFETCH) + # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ + #else +-# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ ++# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() is not defined outside of x86/x64 */ + # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ + # define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) + # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) diff --git a/ports/rocksdb/CONTROL b/ports/rocksdb/CONTROL index 8e78cfc77e..bb68ae0986 100644 --- a/ports/rocksdb/CONTROL +++ b/ports/rocksdb/CONTROL @@ -1,5 +1,5 @@ Source: rocksdb -Version: 6.6.4-1 +Version: 6.11.4 Homepage: https://github.com/facebook/rocksdb Description: A library that provides an embeddable, persistent key-value store for fast storage Default-Features: zlib diff --git a/ports/rocksdb/portfile.cmake b/ports/rocksdb/portfile.cmake index 56afc1cf14..dcf4cd43e4 100644 --- a/ports/rocksdb/portfile.cmake +++ b/ports/rocksdb/portfile.cmake @@ -3,15 +3,15 @@ vcpkg_check_linkage(ONLY_STATIC_LIBRARY) vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO facebook/rocksdb - REF v6.6.4 - SHA512 22f5105244c55ca7030cc8d7ee06cb87f4afdfc85b3f44f7512a9c9c41c037c3b70fd291ec1b0df2553a93b770e11abc4ae2d7f984b2a0f1addd26d424c09539 + REF v6.11.4 + SHA512 46ee38930aee58541366c92ca5cda48c2856292e5e3d66830035b05bb114b01d23ea2c8ee5e2abd971699f5c789adb6038971c3f07c8d68039ddea67d1357d05 HEAD_REF master PATCHES 0001-disable-gtest.patch 0002-only-build-one-flavor.patch 0003-use-find-package.patch - 0004-static-linking-in-linux.patch - 0005-add-config-to-findpackage.patch + 0004-add-config-to-findpackage.patch + 0005-backport-msvc-fixes-pr-7439.patch ) file(REMOVE "${SOURCE_PATH}/cmake/modules/Findzlib.cmake") @@ -23,8 +23,7 @@ file(COPY ) string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" WITH_MD_LIBRARY) -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" ROCKSDB_DISABLE_INSTALL_SHARED_LIB) -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" ROCKSDB_DISABLE_INSTALL_STATIC_LIB) +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" ROCKSDB_BUILD_SHARED) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS FEATURES @@ -43,14 +42,14 @@ vcpkg_configure_cmake( OPTIONS -DWITH_GFLAGS=0 -DWITH_TESTS=OFF + -DWITH_BENCHMARK_TOOLS=OFF -DUSE_RTTI=1 -DROCKSDB_INSTALL_ON_WINDOWS=ON -DFAIL_ON_WARNINGS=OFF -DWITH_MD_LIBRARY=${WITH_MD_LIBRARY} -DPORTABLE=ON -DCMAKE_DEBUG_POSTFIX=d - -DROCKSDB_DISABLE_INSTALL_SHARED_LIB=${ROCKSDB_DISABLE_INSTALL_SHARED_LIB} - -DROCKSDB_DISABLE_INSTALL_STATIC_LIB=${ROCKSDB_DISABLE_INSTALL_STATIC_LIB} + -DROCKSDB_BUILD_SHARED=${ROCKSDB_BUILD_SHARED} -DCMAKE_DISABLE_FIND_PACKAGE_NUMA=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_gtest=TRUE -DCMAKE_DISABLE_FIND_PACKAGE_Git=TRUE @@ -65,5 +64,5 @@ file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) file(INSTALL ${SOURCE_PATH}/LICENSE.Apache DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) file(COPY ${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake ${SOURCE_PATH}/LICENSE.leveldb DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") vcpkg_copy_pdbs()