diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e69de29 diff --git a/3party/marl/.clang-format b/3party/marl/.clang-format new file mode 100644 index 0000000..08178fe --- /dev/null +++ b/3party/marl/.clang-format @@ -0,0 +1,5 @@ +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +BasedOnStyle: Chromium + +--- +Language: Cpp diff --git a/3party/marl/.gitignore b/3party/marl/.gitignore new file mode 100644 index 0000000..4c7dc22 --- /dev/null +++ b/3party/marl/.gitignore @@ -0,0 +1,8 @@ +/.vs/ +/.vscode/ +/build/ +/cmake-build-*/ +/out/ +bazel-* +CMakeSettings.json +/.idea/ diff --git a/3party/marl/.gitmodules b/3party/marl/.gitmodules new file mode 100644 index 0000000..335eee4 --- /dev/null +++ b/3party/marl/.gitmodules @@ -0,0 +1,6 @@ +[submodule "third_party/googletest"] + path = third_party/googletest + url = https://github.com/google/googletest.git +[submodule "third_party/benchmark"] + path = third_party/benchmark + url = https://github.com/google/benchmark.git diff --git a/3party/marl/AUTHORS b/3party/marl/AUTHORS new file mode 100644 index 0000000..8a5bf2a --- /dev/null +++ b/3party/marl/AUTHORS @@ -0,0 +1,9 @@ +# This is the list of the Marl authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. +Google LLC +Shawn Anastasio +A. Wilcox +Jiaxun Yang diff --git a/3party/marl/BUILD.bazel b/3party/marl/BUILD.bazel new file mode 100644 index 0000000..297fd44 --- /dev/null +++ b/3party/marl/BUILD.bazel @@ -0,0 +1,68 @@ +# Copyright 2019 The Marl Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +config_setting( + name = "linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "windows", + constraint_values = ["@platforms//os:windows"], +) + +cc_library( + name = "marl", + srcs = glob( + [ + "src/**/*.cpp", + "src/**/*.c", + "src/**/*.h", + ], + exclude = glob([ + "src/**/*_bench.cpp", + "src/**/*_test.cpp", + ]), + ) + select({ + ":windows": [], + "//conditions:default": glob(["src/**/*.S"]), + }), + hdrs = glob([ + "include/marl/**/*.h", + ]), + includes = [ + "include", + ], + linkopts = select({ + ":linux_x86_64": ["-pthread"], + "//conditions:default": [], + }), + visibility = [ + "//visibility:public", + ], +) + +cc_test( + name = "tests", + srcs = glob([ + "src/**/*_test.cpp", + ]), + deps = [ + "//:marl", + "@googletest//:gtest", + ], +) diff --git a/3party/marl/CHANGES.md b/3party/marl/CHANGES.md new file mode 100644 index 0000000..6076158 --- /dev/null +++ b/3party/marl/CHANGES.md @@ -0,0 +1,8 @@ +# Revision history for `marl` + +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](https://semver.org/). + +## 1.0.0-dev + +First versioned release of marl. diff --git a/3party/marl/CMakeLists.txt b/3party/marl/CMakeLists.txt new file mode 100644 index 0000000..33ff545 --- /dev/null +++ b/3party/marl/CMakeLists.txt @@ -0,0 +1,425 @@ +# Copyright 2019 The Marl Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.0) + +include(cmake/parse_version.cmake) +parse_version("${CMAKE_CURRENT_SOURCE_DIR}/CHANGES.md" MARL) + +set(CMAKE_CXX_STANDARD 11) + +project(Marl + VERSION "${MARL_VERSION_MAJOR}.${MARL_VERSION_MINOR}.${MARL_VERSION_PATCH}" + LANGUAGES C CXX ASM +) + +if (EMSCRIPTEN) + add_compile_options(-O3 -pthread) +endif() + +include(CheckCXXSourceCompiles) + +# MARL_IS_SUBPROJECT is 1 if added via add_subdirectory() from another project. +get_directory_property(MARL_IS_SUBPROJECT PARENT_DIRECTORY) +if(MARL_IS_SUBPROJECT) + set(MARL_IS_SUBPROJECT 1) +endif() + +########################################################### +# Options +########################################################### +function(option_if_not_defined name description default) + if(NOT DEFINED ${name}) + option(${name} ${description} ${default}) + endif() +endfunction() + +option_if_not_defined(MARL_WARNINGS_AS_ERRORS "Treat warnings as errors" OFF) +option_if_not_defined(MARL_BUILD_EXAMPLES "Build example applications" OFF) +option_if_not_defined(MARL_BUILD_TESTS "Build tests" OFF) +option_if_not_defined(MARL_BUILD_BENCHMARKS "Build benchmarks" OFF) +option_if_not_defined(MARL_BUILD_SHARED "Build marl as a shared / dynamic library (default static)" OFF) +option_if_not_defined(MARL_USE_PTHREAD_THREAD_LOCAL "Use pthreads for thread local storage" OFF) +option_if_not_defined(MARL_ASAN "Build marl with address sanitizer" OFF) +option_if_not_defined(MARL_MSAN "Build marl with memory sanitizer" OFF) +option_if_not_defined(MARL_TSAN "Build marl with thread sanitizer" OFF) +option_if_not_defined(MARL_UBSAN "Build marl with undefined-behavior sanitizer" OFF) +option_if_not_defined(MARL_INSTALL "Create marl install target" OFF) +option_if_not_defined(MARL_FULL_BENCHMARK "Run benchmarks for [0 .. numLogicalCPUs] with no stepping" OFF) +option_if_not_defined(MARL_FIBERS_USE_UCONTEXT "Use ucontext instead of assembly for fibers (ignored for platforms that do not support ucontext)" OFF) +option_if_not_defined(MARL_DEBUG_ENABLED "Enable debug checks even in release builds" OFF) + +########################################################### +# Directories +########################################################### +function(set_if_not_defined name value) + if(NOT DEFINED ${name}) + set(${name} ${value} PARENT_SCOPE) + endif() +endfunction() + +set(MARL_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) +set(MARL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) +set_if_not_defined(MARL_THIRD_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party) +set_if_not_defined(MARL_GOOGLETEST_DIR ${MARL_THIRD_PARTY_DIR}/googletest) +set_if_not_defined(MARL_BENCHMARK_DIR ${MARL_THIRD_PARTY_DIR}/benchmark) + +########################################################### +# Submodules +########################################################### +if(MARL_BUILD_TESTS) + if(NOT EXISTS ${MARL_GOOGLETEST_DIR}/.git) + message(WARNING "third_party/googletest submodule missing.") + message(WARNING "Run: `git submodule update --init` to build tests.") + set(MARL_BUILD_TESTS OFF) + endif() +endif(MARL_BUILD_TESTS) + +if(MARL_BUILD_BENCHMARKS) + if(NOT EXISTS ${MARL_BENCHMARK_DIR}/.git) + message(WARNING "third_party/benchmark submodule missing.") + message(WARNING "Run: `git submodule update --init` to build benchmarks.") + set(MARL_BUILD_BENCHMARKS OFF) + endif() +endif(MARL_BUILD_BENCHMARKS) + +if(MARL_BUILD_BENCHMARKS) + set(BENCHMARK_ENABLE_TESTING FALSE CACHE BOOL FALSE FORCE) + add_subdirectory(${MARL_BENCHMARK_DIR}) +endif(MARL_BUILD_BENCHMARKS) + +########################################################### +# Compiler feature tests +########################################################### +# Check that the Clang Thread Safety Analysis' try_acquire_capability behaves +# correctly. This is broken on some earlier versions of clang. +# See: https://bugs.llvm.org/show_bug.cgi?id=32954 +set(SAVE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) +set(CMAKE_REQUIRED_FLAGS "-Wthread-safety -Werror") +check_cxx_source_compiles( + "int main() { + struct __attribute__((capability(\"mutex\"))) Mutex { + void Unlock() __attribute__((release_capability)) {}; + bool TryLock() __attribute__((try_acquire_capability(true))) { return true; }; + }; + Mutex m; + if (m.TryLock()) { + m.Unlock(); // Should not warn. + } + return 0; + }" + MARL_THREAD_SAFETY_ANALYSIS_SUPPORTED) +set(CMAKE_REQUIRED_FLAGS ${SAVE_CMAKE_REQUIRED_FLAGS}) + +# Check whether ucontext is supported. +set(SAVE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) +set(CMAKE_REQUIRED_FLAGS "-Werror") +check_cxx_source_compiles( + "#include + int main() { + ucontext_t ctx; + getcontext(&ctx); + makecontext(&ctx, nullptr, 2, 1, 2); + swapcontext(&ctx, &ctx); + return 0; + }" + MARL_UCONTEXT_SUPPORTED) +set(CMAKE_REQUIRED_FLAGS ${SAVE_CMAKE_REQUIRED_FLAGS}) +if (MARL_FIBERS_USE_UCONTEXT AND NOT MARL_UCONTEXT_SUPPORTED) + # Disable MARL_FIBERS_USE_UCONTEXT and warn if MARL_UCONTEXT_SUPPORTED is 0. + message(WARNING "MARL_FIBERS_USE_UCONTEXT is enabled, but ucontext is not supported by the target. Disabling") + set(MARL_FIBERS_USE_UCONTEXT 0) +endif() + +if(MARL_IS_SUBPROJECT) + # Export supported flags as this may be useful to parent projects + set(MARL_THREAD_SAFETY_ANALYSIS_SUPPORTED PARENT_SCOPE ${MARL_THREAD_SAFETY_ANALYSIS_SUPPORTED}) + set(MARL_UCONTEXT_SUPPORTED PARENT_SCOPE ${MARL_UCONTEXT_SUPPORTED}) +endif() + +########################################################### +# File lists +########################################################### +set(MARL_LIST + ${MARL_SRC_DIR}/debug.cpp + ${MARL_SRC_DIR}/memory.cpp + ${MARL_SRC_DIR}/scheduler.cpp + ${MARL_SRC_DIR}/thread.cpp + ${MARL_SRC_DIR}/trace.cpp +) +if(NOT MSVC) + list(APPEND MARL_LIST + ${MARL_SRC_DIR}/osfiber_aarch64.c + ${MARL_SRC_DIR}/osfiber_arm.c + ${MARL_SRC_DIR}/osfiber_asm_aarch64.S + ${MARL_SRC_DIR}/osfiber_asm_arm.S + ${MARL_SRC_DIR}/osfiber_asm_loongarch64.S + ${MARL_SRC_DIR}/osfiber_asm_mips64.S + ${MARL_SRC_DIR}/osfiber_asm_ppc64.S + ${MARL_SRC_DIR}/osfiber_asm_rv64.S + ${MARL_SRC_DIR}/osfiber_asm_x64.S + ${MARL_SRC_DIR}/osfiber_asm_x86.S + ${MARL_SRC_DIR}/osfiber_loongarch64.c + ${MARL_SRC_DIR}/osfiber_mips64.c + ${MARL_SRC_DIR}/osfiber_ppc64.c + ${MARL_SRC_DIR}/osfiber_rv64.c + ${MARL_SRC_DIR}/osfiber_x64.c + ${MARL_SRC_DIR}/osfiber_x86.c + ${MARL_SRC_DIR}/osfiber_emscripten.cpp + ) + # CMAKE_OSX_ARCHITECTURES settings aren't propagated to assembly files when + # building for Apple platforms (https://gitlab.kitware.com/cmake/cmake/-/issues/20771), + # we treat assembly files as C files to work around this bug. + set_source_files_properties( + ${MARL_SRC_DIR}/osfiber_asm_aarch64.S + ${MARL_SRC_DIR}/osfiber_asm_arm.S + ${MARL_SRC_DIR}/osfiber_asm_loongarch64.S + ${MARL_SRC_DIR}/osfiber_asm_mips64.S + ${MARL_SRC_DIR}/osfiber_asm_ppc64.S + ${MARL_SRC_DIR}/osfiber_asm_x64.S + ${MARL_SRC_DIR}/osfiber_asm_x86.S + PROPERTIES LANGUAGE C + ) +endif(NOT MSVC) + +########################################################### +# OS libraries +########################################################### +find_package(Threads REQUIRED) + +########################################################### +# Functions +########################################################### +function(marl_set_target_options target) + if(MARL_THREAD_SAFETY_ANALYSIS_SUPPORTED) + target_compile_options(${target} PRIVATE "-Wthread-safety") + endif() + + # Enable all warnings + if(MSVC) + target_compile_options(${target} PRIVATE "-W4") + else() + target_compile_options(${target} PRIVATE "-Wall") + endif() + + # Disable specific, pedantic warnings + if(MSVC) + target_compile_options(${target} PRIVATE + "-D_CRT_SECURE_NO_WARNINGS" + "/wd4127" # conditional expression is constant + "/wd4324" # structure was padded due to alignment specifier + ) + endif() + + # Treat all warnings as errors + if(MARL_WARNINGS_AS_ERRORS) + if(MSVC) + target_compile_options(${target} PRIVATE "/WX") + else() + target_compile_options(${target} PRIVATE "-Werror") + endif() + endif(MARL_WARNINGS_AS_ERRORS) + + if(MARL_USE_PTHREAD_THREAD_LOCAL) + target_compile_definitions(${target} PRIVATE "MARL_USE_PTHREAD_THREAD_LOCAL=1") + target_link_libraries(${target} PUBLIC pthread) + endif() + + if(MARL_ASAN) + target_compile_options(${target} PUBLIC "-fsanitize=address") + target_link_libraries(${target} PUBLIC "-fsanitize=address") + elseif(MARL_MSAN) + target_compile_options(${target} PUBLIC "-fsanitize=memory") + target_link_libraries(${target} PUBLIC "-fsanitize=memory") + elseif(MARL_TSAN) + target_compile_options(${target} PUBLIC "-fsanitize=thread") + target_link_libraries(${target} PUBLIC "-fsanitize=thread") + elseif(MARL_UBSAN) + target_compile_options(${target} PUBLIC "-fsanitize=undefined") + target_link_libraries(${target} PUBLIC "-fsanitize=undefined") + endif() + + if(MARL_FIBERS_USE_UCONTEXT) + target_compile_definitions(${target} PRIVATE "MARL_FIBERS_USE_UCONTEXT=1") + endif() + + if(MARL_DEBUG_ENABLED) + target_compile_definitions(${target} PRIVATE "MARL_DEBUG_ENABLED=1") + endif() + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^rv.*") + target_link_libraries(${target} INTERFACE atomic) #explicitly use -latomic for RISC-V linking + endif() + + target_include_directories(${target} PUBLIC $) +endfunction(marl_set_target_options) + +########################################################### +# Targets +########################################################### + +# marl +if(MARL_BUILD_SHARED OR BUILD_SHARED_LIBS) + add_library(marl SHARED ${MARL_LIST}) + if(MSVC) + target_compile_definitions(marl + PRIVATE "MARL_BUILDING_DLL=1" + PUBLIC "MARL_DLL=1" + ) + endif() +else() + add_library(marl ${MARL_LIST}) +endif() + +if(NOT MSVC) + # Public API symbols are made visible with the MARL_EXPORT annotation. + target_compile_options(marl PRIVATE "-fvisibility=hidden") +endif() + +set_target_properties(marl PROPERTIES + POSITION_INDEPENDENT_CODE 1 + VERSION ${MARL_VERSION} + SOVERSION "${MARL_VERSION_MAJOR}" +) + +marl_set_target_options(marl) + +target_link_libraries(marl PUBLIC Threads::Threads) + +# install +if(MARL_INSTALL) + include(CMakePackageConfigHelpers) + include(GNUInstallDirs) + + configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/marl-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/marl-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/marl + ) + + install(DIRECTORY ${MARL_INCLUDE_DIR}/marl + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + USE_SOURCE_PERMISSIONS + ) + + install(TARGETS marl + EXPORT marl-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + + install(EXPORT marl-targets + FILE marl-targets.cmake + NAMESPACE marl:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/marl + ) + + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/marl-config.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/marl + ) +endif(MARL_INSTALL) + +# tests +if(MARL_BUILD_TESTS) + set(MARL_TEST_LIST + ${MARL_SRC_DIR}/blockingcall_test.cpp + ${MARL_SRC_DIR}/conditionvariable_test.cpp + ${MARL_SRC_DIR}/containers_test.cpp + ${MARL_SRC_DIR}/dag_test.cpp + ${MARL_SRC_DIR}/defer_test.cpp + ${MARL_SRC_DIR}/event_test.cpp + ${MARL_SRC_DIR}/marl_test.cpp + ${MARL_SRC_DIR}/marl_test.h + ${MARL_SRC_DIR}/memory_test.cpp + ${MARL_SRC_DIR}/osfiber_test.cpp + ${MARL_SRC_DIR}/parallelize_test.cpp + ${MARL_SRC_DIR}/pool_test.cpp + ${MARL_SRC_DIR}/scheduler_test.cpp + ${MARL_SRC_DIR}/thread_test.cpp + ${MARL_SRC_DIR}/ticket_test.cpp + ${MARL_SRC_DIR}/waitgroup_test.cpp + ${MARL_GOOGLETEST_DIR}/googletest/src/gtest-all.cc + ${MARL_GOOGLETEST_DIR}/googlemock/src/gmock-all.cc + ) + + set(MARL_TEST_INCLUDE_DIR + ${MARL_GOOGLETEST_DIR}/googletest/include/ + ${MARL_GOOGLETEST_DIR}/googlemock/include/ + ${MARL_GOOGLETEST_DIR}/googletest/ + ${MARL_GOOGLETEST_DIR}/googlemock/ + ) + + add_executable(marl-unittests ${MARL_TEST_LIST}) + + set_target_properties(marl-unittests PROPERTIES + INCLUDE_DIRECTORIES "${MARL_TEST_INCLUDE_DIR}" + FOLDER "Tests" + ) + + marl_set_target_options(marl-unittests) + + target_link_libraries(marl-unittests PRIVATE marl) +endif(MARL_BUILD_TESTS) + +# benchmarks +if(MARL_BUILD_BENCHMARKS) + set(MARL_BENCHMARK_LIST + ${MARL_SRC_DIR}/blockingcall_bench.cpp + ${MARL_SRC_DIR}/defer_bench.cpp + ${MARL_SRC_DIR}/event_bench.cpp + ${MARL_SRC_DIR}/marl_bench.cpp + ${MARL_SRC_DIR}/non_marl_bench.cpp + ${MARL_SRC_DIR}/scheduler_bench.cpp + ${MARL_SRC_DIR}/ticket_bench.cpp + ${MARL_SRC_DIR}/waitgroup_bench.cpp + ) + + add_executable(marl-benchmarks ${MARL_BENCHMARK_LIST}) + set_target_properties(${target} PROPERTIES FOLDER "Benchmarks") + + marl_set_target_options(marl-benchmarks) + + target_compile_definitions(marl-benchmarks PRIVATE + "MARL_FULL_BENCHMARK=${MARL_FULL_BENCHMARK}" + ) + + target_link_libraries(marl-benchmarks PRIVATE benchmark::benchmark marl) +endif(MARL_BUILD_BENCHMARKS) + +# examples +if(MARL_BUILD_EXAMPLES) + function(build_example target) + add_executable(${target} "${CMAKE_CURRENT_SOURCE_DIR}/examples/${target}.cpp") + set_target_properties(${target} PROPERTIES FOLDER "Examples") + marl_set_target_options(${target}) + target_link_libraries(${target} PRIVATE marl) + if (EMSCRIPTEN) + target_link_options(${target} PRIVATE + -O1 + -pthread -sPTHREAD_POOL_SIZE=2 -sPROXY_TO_PTHREAD + -sASYNCIFY # -sASYNCIFY_STACK_SIZE=1000000 + -sALLOW_MEMORY_GROWTH=1 -sASSERTIONS + -sENVIRONMENT=web,worker + "SHELL:--shell-file ${CMAKE_CURRENT_SOURCE_DIR}/examples/shell.emscripten.html") + set_target_properties(${target} PROPERTIES SUFFIX .html) + endif() + endfunction(build_example) + + build_example(fractal) + build_example(hello_task) + build_example(primes) + build_example(tasks_in_tasks) +endif(MARL_BUILD_EXAMPLES) diff --git a/3party/marl/CONTRIBUTING.md b/3party/marl/CONTRIBUTING.md new file mode 100644 index 0000000..ebbb59e --- /dev/null +++ b/3party/marl/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). \ No newline at end of file diff --git a/3party/marl/LICENSE b/3party/marl/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/3party/marl/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/3party/marl/README.md b/3party/marl/README.md new file mode 100644 index 0000000..8c57078 --- /dev/null +++ b/3party/marl/README.md @@ -0,0 +1,234 @@ +# Marl + +Marl is a hybrid thread / fiber task scheduler written in C++ 11. + +## About + +Marl is a C++ 11 library that provides a fluent interface for running tasks across a number of threads. + +Marl uses a combination of fibers and threads to allow efficient execution of tasks that can block, while keeping a fixed number of hardware threads. + +Marl supports Windows, macOS, Linux, FreeBSD, Fuchsia, Emscripten, Android and iOS (arm, aarch64, loongarch64, mips64, ppc64, rv64, x86 and x64). + +Marl has no dependencies on other libraries (with an exception on googletest for building the optional unit tests). + +Example: + +```cpp +#include "marl/defer.h" +#include "marl/event.h" +#include "marl/scheduler.h" +#include "marl/waitgroup.h" + +#include + +int main() { + // Create a marl scheduler using all the logical processors available to the process. + // Bind this scheduler to the main thread so we can call marl::schedule() + marl::Scheduler scheduler(marl::Scheduler::Config::allCores()); + scheduler.bind(); + defer(scheduler.unbind()); // Automatically unbind before returning. + + constexpr int numTasks = 10; + + // Create an event that is manually reset. + marl::Event sayHello(marl::Event::Mode::Manual); + + // Create a WaitGroup with an initial count of numTasks. + marl::WaitGroup saidHello(numTasks); + + // Schedule some tasks to run asynchronously. + for (int i = 0; i < numTasks; i++) { + // Each task will run on one of the 4 worker threads. + marl::schedule([=] { // All marl primitives are capture-by-value. + // Decrement the WaitGroup counter when the task has finished. + defer(saidHello.done()); + + printf("Task %d waiting to say hello...\n", i); + + // Blocking in a task? + // The scheduler will find something else for this thread to do. + sayHello.wait(); + + printf("Hello from task %d!\n", i); + }); + } + + sayHello.signal(); // Unblock all the tasks. + + saidHello.wait(); // Wait for all tasks to complete. + + printf("All tasks said hello.\n"); + + // All tasks are guaranteed to complete before the scheduler is destructed. +} +``` + +## Benchmarks + +Graphs of several microbenchmarks can be found [here](https://google.github.io/marl/benchmarks). + +## Building + +Marl contains many unit tests and examples that can be built using CMake. + +Unit tests require fetching the `googletest` external project, which can be done by typing the following in your terminal: + +```bash +cd +git submodule update --init +``` + +### Linux and macOS + +To build the unit tests and examples, type the following in your terminal: + +```bash +cd +mkdir build +cd build +cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 +make +``` + +The resulting binaries will be found in `/build` + +### Emscripten + +1. install and activate the emscripten sdk following [standard instructions for your platform](https://emscripten.org/docs/getting_started/downloads.html). +2. build an example from the examples folder using emscripten, say `hello_task`. +```bash +cd +mkdir build +cd build +emcmake cmake .. -DMARL_BUILD_EXAMPLES=1 +make hello_task -j 8 +``` +NOTE: you want to change the value of the linker flag `sPTHREAD_POOL_SIZE` that must be at least as large as the number of threads used by your application. +3. Test the emscripten output. +You can use the provided python script to create a local web server: +```bash +../run_webserver +``` +In your browser, navigate to the example URL: [http://127.0.0.1:8080/hello_task.html](http://127.0.0.1:8080/hello_task.html). +Voilà - you should see the log output appear on the web page. + +### Installing Marl (vcpkg) + +Alternatively, you can build and install Marl using [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + +```bash or powershell +git clone https://github.com/Microsoft/vcpkg.git +cd vcpkg +./bootstrap-vcpkg.sh +./vcpkg integrate install +./vcpkg install marl +``` + +The Marl port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +### Windows + +Marl can be built using [Visual Studio 2019's CMake integration](https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=vs-2019). + +### Using Marl in your CMake project + +You can build and link Marl using `add_subdirectory()` in your project's `CMakeLists.txt` file: + +```cmake +set(MARL_DIR ) # example : "${CMAKE_CURRENT_SOURCE_DIR}/third_party/marl" +add_subdirectory(${MARL_DIR}) +``` + +This will define the `marl` library target, which you can pass to `target_link_libraries()`: + +```cmake +target_link_libraries( marl) # replace with the name of your project's target +``` + +You may also wish to specify your own paths to the third party libraries used by `marl`. +You can do this by setting any of the following variables before the call to `add_subdirectory()`: + +```cmake +set(MARL_THIRD_PARTY_DIR ) # defaults to ${MARL_DIR}/third_party +set(MARL_GOOGLETEST_DIR ) # defaults to ${MARL_THIRD_PARTY_DIR}/googletest +add_subdirectory(${MARL_DIR}) +``` + +### Usage Recommendations + +#### Capture marl synchronization primitives by value + +All marl synchronization primitives aside from `marl::ConditionVariable` should be lambda-captured by **value**: + +```c++ +marl::Event event; +marl::schedule([=]{ // [=] Good, [&] Bad. + event.signal(); +}) +``` + +Internally, these primitives hold a shared pointer to the primitive state. By capturing by value we avoid common issues where the primitive may be destructed before the last reference is used. + +#### Create one instance of `marl::Scheduler`, use it for the lifetime of the process + +The `marl::Scheduler` constructor can be expensive as it may spawn a number of hardware threads. \ +Destructing the `marl::Scheduler` requires waiting on all tasks to complete. + +Multiple `marl::Scheduler`s may fight each other for hardware thread utilization. + +For these reasons, it is recommended to create a single `marl::Scheduler` for the lifetime of your process. + +For example: + +```c++ +int main() { + marl::Scheduler scheduler(marl::Scheduler::Config::allCores()); + scheduler.bind(); + defer(scheduler.unbind()); + + return do_program_stuff(); +} +``` + +#### Bind the scheduler to externally created threads + +In order to call `marl::schedule()` the scheduler must be bound to the calling thread. Failure to bind the scheduler to the thread before calling `marl::schedule()` will result in undefined behavior. + +`marl::Scheduler` may be simultaneously bound to any number of threads, and the scheduler can be retrieved from a bound thread with `marl::Scheduler::get()`. + +A typical way to pass the scheduler from one thread to another would be: + +```c++ +std::thread spawn_new_thread() { + // Grab the scheduler from the currently running thread. + marl::Scheduler* scheduler = marl::Scheduler::get(); + + // Spawn the new thread. + return std::thread([=] { + // Bind the scheduler to the new thread. + scheduler->bind(); + defer(scheduler->unbind()); + + // You can now safely call `marl::schedule()` + run_thread_logic(); + }); +} + +``` + +Always remember to unbind the scheduler before terminating the thread. Forgetting to unbind will result in the `marl::Scheduler` destructor blocking indefinitely. + +#### Don't use externally blocking calls in marl tasks + +The `marl::Scheduler` internally holds a number of worker threads which will execute the scheduled tasks. If a marl task becomes blocked on a marl synchronization primitive, marl can yield from the blocked task and continue execution of other scheduled tasks. + +Calling a non-marl blocking function on a marl worker thread will prevent that worker thread from being able to switch to execute other tasks until the blocking function has returned. Examples of these non-marl blocking functions include: [`std::mutex::lock()`](https://en.cppreference.com/w/cpp/thread/mutex/lock), [`std::condition_variable::wait()`](https://en.cppreference.com/w/cpp/thread/condition_variable/wait), [`accept()`](http://man7.org/linux/man-pages/man2/accept.2.html). + +Short blocking calls are acceptable, such as a mutex lock to access a data structure. However be careful that you do not use a marl blocking call with a `std::mutex` lock held - the marl task may yield with the lock held, and block other tasks from re-locking the mutex. This sort of situation may end up with a deadlock. + +If you need to make a blocking call from a marl worker thread, you may wish to use [`marl::blocking_call()`](https://github.com/google/marl/blob/main/include/marl/blockingcall.h), which will spawn a new thread for performing the call, allowing the marl worker to continue processing other scheduled tasks. + +--- + +Note: This is not an officially supported Google product diff --git a/3party/marl/WORKSPACE b/3party/marl/WORKSPACE new file mode 100644 index 0000000..b11314c --- /dev/null +++ b/3party/marl/WORKSPACE @@ -0,0 +1,31 @@ +# Copyright 2019 The Marl Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "googletest", # 2021-07-09 + sha256 = "353571c2440176ded91c2de6d6cd88ddd41401d14692ec1f99e35d013feda55a", + strip_prefix = "googletest-release-1.11.0", + urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.11.0.zip"], +) + +http_archive( + name = "platforms", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", + ], + sha256 = "5308fc1d8865406a49427ba24a9ab53087f17f5266a7aabbfc28823f3916e1ca", +) diff --git a/3party/marl/cmake/marl-config.cmake.in b/3party/marl/cmake/marl-config.cmake.in new file mode 100644 index 0000000..050eff5 --- /dev/null +++ b/3party/marl/cmake/marl-config.cmake.in @@ -0,0 +1,23 @@ +# Copyright 2020 The Marl Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +find_dependency(Threads) + +if(NOT TARGET marl::marl) + include(${CMAKE_CURRENT_LIST_DIR}/marl-targets.cmake) +endif() diff --git a/3party/marl/cmake/parse_version.cmake b/3party/marl/cmake/parse_version.cmake new file mode 100644 index 0000000..b826096 --- /dev/null +++ b/3party/marl/cmake/parse_version.cmake @@ -0,0 +1,41 @@ +# Copyright 2020 The Marl Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# parse_version() reads and parses the version string from FILE, assigning the +# version string to ${PROJECT}_VERSION and the parsed version to +# ${PROJECT}_VERSION_MAJOR, ${PROJECT}_VERSION_MINOR, ${PROJECT}_VERSION_PATCH, +# and the optional ${PROJECT}_VERSION_FLAVOR. +# +# The version string take one of the forms: +# .. +# ..- +function(parse_version FILE PROJECT) + configure_file(${FILE} "${CMAKE_CURRENT_BINARY_DIR}/CHANGES.md") # Required to re-run cmake on version change + file(READ ${FILE} CHANGES) + if(${CHANGES} MATCHES "#+ *([0-9]+)\\.([0-9]+)\\.([0-9]+)(-[a-zA-Z0-9]+)?") + set(FLAVOR "") + if(NOT "${CMAKE_MATCH_4}" STREQUAL "") + string(SUBSTRING ${CMAKE_MATCH_4} 1 -1 FLAVOR) + endif() + set("${PROJECT}_VERSION_MAJOR" ${CMAKE_MATCH_1} PARENT_SCOPE) + set("${PROJECT}_VERSION_MINOR" ${CMAKE_MATCH_2} PARENT_SCOPE) + set("${PROJECT}_VERSION_PATCH" ${CMAKE_MATCH_3} PARENT_SCOPE) + set("${PROJECT}_VERSION_FLAVOR" ${FLAVOR} PARENT_SCOPE) + set("${PROJECT}_VERSION" + "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}${CMAKE_MATCH_4}" + PARENT_SCOPE) + else() + message(FATAL_ERROR "Unable to parse version from '${FILE}'") + endif() +endfunction() diff --git a/3party/marl/go.mod b/3party/marl/go.mod new file mode 100644 index 0000000..a2f1b57 --- /dev/null +++ b/3party/marl/go.mod @@ -0,0 +1,3 @@ +module github.com/google/marl + +go 1.16 diff --git a/3party/marl/include/marl/blockingcall.h b/3party/marl/include/marl/blockingcall.h new file mode 100644 index 0000000..6e65434 --- /dev/null +++ b/3party/marl/include/marl/blockingcall.h @@ -0,0 +1,106 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_blocking_call_h +#define marl_blocking_call_h + +#include "export.h" +#include "scheduler.h" +#include "waitgroup.h" + +#include +#include +#include + +namespace marl { +namespace detail { + +template +class OnNewThread { + public: + template + MARL_NO_EXPORT inline static RETURN_TYPE call(F&& f, Args&&... args) { + RETURN_TYPE result; + WaitGroup wg(1); + auto scheduler = Scheduler::get(); + auto thread = std::thread( + [&, wg](Args&&... args) { + if (scheduler != nullptr) { + scheduler->bind(); + } + result = f(std::forward(args)...); + if (scheduler != nullptr) { + Scheduler::unbind(); + } + wg.done(); + }, + std::forward(args)...); + wg.wait(); + thread.join(); + return result; + } +}; + +template <> +class OnNewThread { + public: + template + MARL_NO_EXPORT inline static void call(F&& f, Args&&... args) { + WaitGroup wg(1); + auto scheduler = Scheduler::get(); + auto thread = std::thread( + [&, wg](Args&&... args) { + if (scheduler != nullptr) { + scheduler->bind(); + } + f(std::forward(args)...); + if (scheduler != nullptr) { + Scheduler::unbind(); + } + wg.done(); + }, + std::forward(args)...); + wg.wait(); + thread.join(); + } +}; + +} // namespace detail + +// blocking_call() calls the function F on a new thread, yielding this fiber +// to execute other tasks until F has returned. +// +// Example: +// +// void runABlockingFunctionOnATask() +// { +// // Schedule a task that calls a blocking, non-yielding function. +// marl::schedule([=] { +// // call_blocking_function() may block indefinitely. +// // Ensure this call does not block other tasks from running. +// auto result = marl::blocking_call(call_blocking_function); +// // call_blocking_function() has now returned. +// // result holds the return value of the blocking function call. +// }); +// } +template +MARL_NO_EXPORT auto inline blocking_call(F&& f, Args&&... args) + -> decltype(f(args...)) { + return detail::OnNewThread::call( + std::forward(f), std::forward(args)...); +} + +} // namespace marl + +#endif // marl_blocking_call_h diff --git a/3party/marl/include/marl/conditionvariable.h b/3party/marl/include/marl/conditionvariable.h new file mode 100644 index 0000000..bafb37e --- /dev/null +++ b/3party/marl/include/marl/conditionvariable.h @@ -0,0 +1,197 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_condition_variable_h +#define marl_condition_variable_h + +#include "containers.h" +#include "debug.h" +#include "memory.h" +#include "mutex.h" +#include "scheduler.h" +#include "tsa.h" + +#include +#include + +namespace marl { + +// ConditionVariable is a synchronization primitive that can be used to block +// one or more fibers or threads, until another fiber or thread modifies a +// shared variable (the condition) and notifies the ConditionVariable. +// +// If the ConditionVariable is blocked on a thread with a Scheduler bound, the +// thread will work on other tasks until the ConditionVariable is unblocked. +class ConditionVariable { + public: + MARL_NO_EXPORT inline ConditionVariable( + Allocator* allocator = Allocator::Default); + + // notify_one() notifies and potentially unblocks one waiting fiber or thread. + MARL_NO_EXPORT inline void notify_one(); + + // notify_all() notifies and potentially unblocks all waiting fibers and/or + // threads. + MARL_NO_EXPORT inline void notify_all(); + + // wait() blocks the current fiber or thread until the predicate is satisfied + // and the ConditionVariable is notified. + template + MARL_NO_EXPORT inline void wait(marl::lock& lock, Predicate&& pred); + + // wait_for() blocks the current fiber or thread until the predicate is + // satisfied, and the ConditionVariable is notified, or the timeout has been + // reached. Returns false if pred still evaluates to false after the timeout + // has been reached, otherwise true. + template + MARL_NO_EXPORT inline bool wait_for( + marl::lock& lock, + const std::chrono::duration& duration, + Predicate&& pred); + + // wait_until() blocks the current fiber or thread until the predicate is + // satisfied, and the ConditionVariable is notified, or the timeout has been + // reached. Returns false if pred still evaluates to false after the timeout + // has been reached, otherwise true. + template + MARL_NO_EXPORT inline bool wait_until( + marl::lock& lock, + const std::chrono::time_point& timeout, + Predicate&& pred); + + private: + ConditionVariable(const ConditionVariable&) = delete; + ConditionVariable(ConditionVariable&&) = delete; + ConditionVariable& operator=(const ConditionVariable&) = delete; + ConditionVariable& operator=(ConditionVariable&&) = delete; + + marl::mutex mutex; + containers::list waiting; + std::condition_variable condition; + std::atomic numWaiting = {0}; + std::atomic numWaitingOnCondition = {0}; +}; + +ConditionVariable::ConditionVariable( + Allocator* allocator /* = Allocator::Default */) + : waiting(allocator) {} + +void ConditionVariable::notify_one() { + if (numWaiting == 0) { + return; + } + { + marl::lock lock(mutex); + if (waiting.size() > 0) { + (*waiting.begin())->notify(); // Only wake one fiber. + return; + } + } + if (numWaitingOnCondition > 0) { + condition.notify_one(); + } +} + +void ConditionVariable::notify_all() { + if (numWaiting == 0) { + return; + } + { + marl::lock lock(mutex); + for (auto fiber : waiting) { + fiber->notify(); + } + } + if (numWaitingOnCondition > 0) { + condition.notify_all(); + } +} + +template +void ConditionVariable::wait(marl::lock& lock, Predicate&& pred) { + if (pred()) { + return; + } + numWaiting++; + if (auto fiber = Scheduler::Fiber::current()) { + // Currently executing on a scheduler fiber. + // Yield to let other tasks run that can unblock this fiber. + mutex.lock(); + auto it = waiting.emplace_front(fiber); + mutex.unlock(); + + fiber->wait(lock, pred); + + mutex.lock(); + waiting.erase(it); + mutex.unlock(); + } else { + // Currently running outside of the scheduler. + // Delegate to the std::condition_variable. + numWaitingOnCondition++; + lock.wait(condition, pred); + numWaitingOnCondition--; + } + numWaiting--; +} + +template +bool ConditionVariable::wait_for( + marl::lock& lock, + const std::chrono::duration& duration, + Predicate&& pred) { + return wait_until(lock, std::chrono::system_clock::now() + duration, pred); +} + +template +bool ConditionVariable::wait_until( + marl::lock& lock, + const std::chrono::time_point& timeout, + Predicate&& pred) { + if (pred()) { + return true; + } + + if (auto fiber = Scheduler::Fiber::current()) { + numWaiting++; + + // Currently executing on a scheduler fiber. + // Yield to let other tasks run that can unblock this fiber. + mutex.lock(); + auto it = waiting.emplace_front(fiber); + mutex.unlock(); + + auto res = fiber->wait(lock, timeout, pred); + + mutex.lock(); + waiting.erase(it); + mutex.unlock(); + + numWaiting--; + return res; + } + + // Currently running outside of the scheduler. + // Delegate to the std::condition_variable. + numWaiting++; + numWaitingOnCondition++; + auto res = lock.wait_until(condition, timeout, pred); + numWaitingOnCondition--; + numWaiting--; + return res; +} + +} // namespace marl + +#endif // marl_condition_variable_h diff --git a/3party/marl/include/marl/containers.h b/3party/marl/include/marl/containers.h new file mode 100644 index 0000000..5410a7f --- /dev/null +++ b/3party/marl/include/marl/containers.h @@ -0,0 +1,571 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_containers_h +#define marl_containers_h + +#include "debug.h" +#include "memory.h" + +#include // std::max +#include // size_t +#include // std::move + +#include +#include +#include +#include +#include + +namespace marl { +namespace containers { + +//////////////////////////////////////////////////////////////////////////////// +// STL wrappers +// STL containers that use a marl::StlAllocator backed by a marl::Allocator. +// Note: These may be re-implemented to optimize for marl's usage cases. +// See: https://github.com/google/marl/issues/129 +//////////////////////////////////////////////////////////////////////////////// +template +using deque = std::deque>; + +template > +using map = std::map>>; + +template > +using set = std::set>; + +template , + typename E = std::equal_to> +using unordered_map = + std::unordered_map>>; + +template , typename E = std::equal_to> +using unordered_set = std::unordered_set>; + +// take() takes and returns the front value from the deque. +template +MARL_NO_EXPORT inline T take(deque& queue) { + auto out = std::move(queue.front()); + queue.pop_front(); + return out; +} + +// take() takes and returns the first value from the unordered_set. +template +MARL_NO_EXPORT inline T take(unordered_set& set) { + auto it = set.begin(); + auto out = std::move(*it); + set.erase(it); + return out; +} + +//////////////////////////////////////////////////////////////////////////////// +// vector +//////////////////////////////////////////////////////////////////////////////// + +// vector is a container of contiguously stored elements. +// Unlike std::vector, marl::containers::vector keeps the first +// BASE_CAPACITY elements internally, which will avoid dynamic heap +// allocations. Once the vector exceeds BASE_CAPACITY elements, vector will +// allocate storage from the heap. +template +class vector { + public: + MARL_NO_EXPORT inline vector(Allocator* allocator = Allocator::Default); + + template + MARL_NO_EXPORT inline vector(const vector& other, + Allocator* allocator = Allocator::Default); + + template + MARL_NO_EXPORT inline vector(vector&& other, + Allocator* allocator = Allocator::Default); + + MARL_NO_EXPORT inline ~vector(); + + MARL_NO_EXPORT inline vector& operator=(const vector&); + + template + MARL_NO_EXPORT inline vector& operator=( + const vector&); + + template + MARL_NO_EXPORT inline vector& operator=( + vector&&); + + MARL_NO_EXPORT inline void push_back(const T& el); + MARL_NO_EXPORT inline void emplace_back(T&& el); + MARL_NO_EXPORT inline void pop_back(); + MARL_NO_EXPORT inline T& front(); + MARL_NO_EXPORT inline T& back(); + MARL_NO_EXPORT inline const T& front() const; + MARL_NO_EXPORT inline const T& back() const; + MARL_NO_EXPORT inline T* begin(); + MARL_NO_EXPORT inline T* end(); + MARL_NO_EXPORT inline const T* begin() const; + MARL_NO_EXPORT inline const T* end() const; + MARL_NO_EXPORT inline T& operator[](size_t i); + MARL_NO_EXPORT inline const T& operator[](size_t i) const; + MARL_NO_EXPORT inline size_t size() const; + MARL_NO_EXPORT inline size_t cap() const; + MARL_NO_EXPORT inline void resize(size_t n); + MARL_NO_EXPORT inline void reserve(size_t n); + MARL_NO_EXPORT inline T* data(); + MARL_NO_EXPORT inline const T* data() const; + + Allocator* const allocator; + + private: + using TStorage = typename marl::aligned_storage::type; + + vector(const vector&) = delete; + + MARL_NO_EXPORT inline void free(); + + size_t count = 0; + size_t capacity = BASE_CAPACITY; + TStorage buffer[BASE_CAPACITY]; + TStorage* elements = buffer; + Allocation allocation; +}; + +template +vector::vector( + Allocator* allocator_ /* = Allocator::Default */) + : allocator(allocator_) {} + +template +template +vector::vector( + const vector& other, + Allocator* allocator_ /* = Allocator::Default */) + : allocator(allocator_) { + *this = other; +} + +template +template +vector::vector( + vector&& other, + Allocator* allocator_ /* = Allocator::Default */) + : allocator(allocator_) { + *this = std::move(other); +} + +template +vector::~vector() { + free(); +} + +template +vector& vector::operator=( + const vector& other) { + free(); + reserve(other.size()); + count = other.size(); + for (size_t i = 0; i < count; i++) { + new (&reinterpret_cast(elements)[i]) T(other[i]); + } + return *this; +} + +template +template +vector& vector::operator=( + const vector& other) { + free(); + reserve(other.size()); + count = other.size(); + for (size_t i = 0; i < count; i++) { + new (&reinterpret_cast(elements)[i]) T(other[i]); + } + return *this; +} + +template +template +vector& vector::operator=( + vector&& other) { + free(); + reserve(other.size()); + count = other.size(); + for (size_t i = 0; i < count; i++) { + new (&reinterpret_cast(elements)[i]) T(std::move(other[i])); + } + other.resize(0); + return *this; +} + +template +void vector::push_back(const T& el) { + reserve(count + 1); + new (&reinterpret_cast(elements)[count]) T(el); + count++; +} + +template +void vector::emplace_back(T&& el) { + reserve(count + 1); + new (&reinterpret_cast(elements)[count]) T(std::move(el)); + count++; +} + +template +void vector::pop_back() { + MARL_ASSERT(count > 0, "pop_back() called on empty vector"); + count--; + reinterpret_cast(elements)[count].~T(); +} + +template +T& vector::front() { + MARL_ASSERT(count > 0, "front() called on empty vector"); + return reinterpret_cast(elements)[0]; +} + +template +T& vector::back() { + MARL_ASSERT(count > 0, "back() called on empty vector"); + return reinterpret_cast(elements)[count - 1]; +} + +template +const T& vector::front() const { + MARL_ASSERT(count > 0, "front() called on empty vector"); + return reinterpret_cast(elements)[0]; +} + +template +const T& vector::back() const { + MARL_ASSERT(count > 0, "back() called on empty vector"); + return reinterpret_cast(elements)[count - 1]; +} + +template +T* vector::begin() { + return reinterpret_cast(elements); +} + +template +T* vector::end() { + return reinterpret_cast(elements) + count; +} + +template +const T* vector::begin() const { + return reinterpret_cast(elements); +} + +template +const T* vector::end() const { + return reinterpret_cast(elements) + count; +} + +template +T& vector::operator[](size_t i) { + MARL_ASSERT(i < count, "index %d exceeds vector size %d", int(i), int(count)); + return reinterpret_cast(elements)[i]; +} + +template +const T& vector::operator[](size_t i) const { + MARL_ASSERT(i < count, "index %d exceeds vector size %d", int(i), int(count)); + return reinterpret_cast(elements)[i]; +} + +template +size_t vector::size() const { + return count; +} + +template +void vector::resize(size_t n) { + reserve(n); + while (count < n) { + new (&reinterpret_cast(elements)[count++]) T(); + } + while (n < count) { + reinterpret_cast(elements)[--count].~T(); + } +} + +template +void vector::reserve(size_t n) { + if (n > capacity) { + capacity = std::max(n * 2, 8); + + Allocation::Request request; + request.size = sizeof(T) * capacity; + request.alignment = alignof(T); + request.usage = Allocation::Usage::Vector; + + auto alloc = allocator->allocate(request); + auto grown = reinterpret_cast(alloc.ptr); + for (size_t i = 0; i < count; i++) { + new (&reinterpret_cast(grown)[i]) + T(std::move(reinterpret_cast(elements)[i])); + } + free(); + elements = grown; + allocation = alloc; + } +} + +template +T* vector::data() { + return elements; +} + +template +const T* vector::data() const { + return elements; +} + +template +void vector::free() { + for (size_t i = 0; i < count; i++) { + reinterpret_cast(elements)[i].~T(); + } + + if (allocation.ptr != nullptr) { + allocator->free(allocation); + allocation = {}; + elements = nullptr; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// list +//////////////////////////////////////////////////////////////////////////////// + +// list is a minimal std::list like container that supports constant time +// insertion and removal of elements. +// list keeps hold of allocations (it only releases allocations on destruction), +// to avoid repeated heap allocations and frees when frequently inserting and +// removing elements. +template +class list { + struct Entry { + T data; + Entry* next; + Entry* prev; + }; + + public: + class iterator { + public: + MARL_NO_EXPORT inline iterator(Entry*); + MARL_NO_EXPORT inline T* operator->(); + MARL_NO_EXPORT inline T& operator*(); + MARL_NO_EXPORT inline iterator& operator++(); + MARL_NO_EXPORT inline bool operator==(const iterator&) const; + MARL_NO_EXPORT inline bool operator!=(const iterator&) const; + + private: + friend list; + Entry* entry; + }; + + MARL_NO_EXPORT inline list(Allocator* allocator = Allocator::Default); + MARL_NO_EXPORT inline ~list(); + + MARL_NO_EXPORT inline iterator begin(); + MARL_NO_EXPORT inline iterator end(); + MARL_NO_EXPORT inline size_t size() const; + + template + MARL_NO_EXPORT inline iterator emplace_front(Args&&... args); + MARL_NO_EXPORT inline void erase(iterator); + + private: + // copy / move is currently unsupported. + list(const list&) = delete; + list(list&&) = delete; + list& operator=(const list&) = delete; + list& operator=(list&&) = delete; + + struct AllocationChain { + Allocation allocation; + AllocationChain* next; + }; + + MARL_NO_EXPORT inline void grow(size_t count); + + MARL_NO_EXPORT static inline void unlink(Entry* entry, Entry*& list); + MARL_NO_EXPORT static inline void link(Entry* entry, Entry*& list); + + Allocator* const allocator; + size_t size_ = 0; + size_t capacity = 0; + AllocationChain* allocations = nullptr; + Entry* free = nullptr; + Entry* head = nullptr; +}; + +template +list::iterator::iterator(Entry* entry_) : entry(entry_) {} + +template +T* list::iterator::operator->() { + return &entry->data; +} + +template +T& list::iterator::operator*() { + return entry->data; +} + +template +typename list::iterator& list::iterator::operator++() { + entry = entry->next; + return *this; +} + +template +bool list::iterator::operator==(const iterator& rhs) const { + return entry == rhs.entry; +} + +template +bool list::iterator::operator!=(const iterator& rhs) const { + return entry != rhs.entry; +} + +template +list::list(Allocator* allocator_ /* = Allocator::Default */) + : allocator(allocator_) {} + +template +list::~list() { + for (auto el = head; el != nullptr; el = el->next) { + el->data.~T(); + } + + auto curr = allocations; + while (curr != nullptr) { + auto next = curr->next; + allocator->free(curr->allocation); + curr = next; + } +} + +template +typename list::iterator list::begin() { + return {head}; +} + +template +typename list::iterator list::end() { + return {nullptr}; +} + +template +size_t list::size() const { + return size_; +} + +template +template +typename list::iterator list::emplace_front(Args&&... args) { + if (free == nullptr) { + grow(std::max(capacity, 8)); + } + + auto entry = free; + + unlink(entry, free); + link(entry, head); + + new (&entry->data) T(std::forward(args)...); + size_++; + + return entry; +} + +template +void list::erase(iterator it) { + auto entry = it.entry; + unlink(entry, head); + link(entry, free); + + entry->data.~T(); + size_--; +} + +template +void list::grow(size_t count) { + auto const entriesSize = sizeof(Entry) * count; + auto const allocChainOffset = alignUp(entriesSize, alignof(AllocationChain)); + auto const allocSize = allocChainOffset + sizeof(AllocationChain); + + Allocation::Request request; + request.size = allocSize; + request.alignment = std::max(alignof(Entry), alignof(AllocationChain)); + request.usage = Allocation::Usage::List; + auto alloc = allocator->allocate(request); + + auto entries = reinterpret_cast(alloc.ptr); + for (size_t i = 0; i < count; i++) { + auto entry = &entries[i]; + entry->prev = nullptr; + entry->next = free; + if (free) { + free->prev = entry; + } + free = entry; + } + + auto allocChain = reinterpret_cast( + reinterpret_cast(alloc.ptr) + allocChainOffset); + + allocChain->allocation = alloc; + allocChain->next = allocations; + allocations = allocChain; + + capacity += count; +} + +template +void list::unlink(Entry* entry, Entry*& list) { + if (list == entry) { + list = list->next; + } + if (entry->prev) { + entry->prev->next = entry->next; + } + if (entry->next) { + entry->next->prev = entry->prev; + } + entry->prev = nullptr; + entry->next = nullptr; +} + +template +void list::link(Entry* entry, Entry*& list) { + MARL_ASSERT(entry->next == nullptr, "link() called on entry already linked"); + MARL_ASSERT(entry->prev == nullptr, "link() called on entry already linked"); + if (list) { + entry->next = list; + list->prev = entry; + } + list = entry; +} + +} // namespace containers +} // namespace marl + +#endif // marl_containers_h diff --git a/3party/marl/include/marl/dag.h b/3party/marl/include/marl/dag.h new file mode 100644 index 0000000..a2631aa --- /dev/null +++ b/3party/marl/include/marl/dag.h @@ -0,0 +1,410 @@ +// Copyright 2020 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// marl::DAG<> provides an ahead of time, declarative, directed acyclic +// task graph. + +#ifndef marl_dag_h +#define marl_dag_h + +#include "containers.h" +#include "export.h" +#include "memory.h" +#include "scheduler.h" +#include "waitgroup.h" + +namespace marl { +namespace detail { +using DAGCounter = std::atomic; +template +struct DAGRunContext { + T data; + Allocator::unique_ptr counters; + + template + MARL_NO_EXPORT inline void invoke(F&& f) { + f(data); + } +}; +template <> +struct DAGRunContext { + Allocator::unique_ptr counters; + + template + MARL_NO_EXPORT inline void invoke(F&& f) { + f(); + } +}; +template +struct DAGWork { + using type = std::function; +}; +template <> +struct DAGWork { + using type = std::function; +}; +} // namespace detail + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations +/////////////////////////////////////////////////////////////////////////////// +template +class DAG; + +template +class DAGBuilder; + +template +class DAGNodeBuilder; + +/////////////////////////////////////////////////////////////////////////////// +// DAGBase +/////////////////////////////////////////////////////////////////////////////// + +// DAGBase is derived by DAG and DAG. It has no public API. +template +class DAGBase { + protected: + friend DAGBuilder; + friend DAGNodeBuilder; + + using RunContext = detail::DAGRunContext; + using Counter = detail::DAGCounter; + using NodeIndex = size_t; + using Work = typename detail::DAGWork::type; + static const constexpr size_t NumReservedNodes = 32; + static const constexpr size_t NumReservedNumOuts = 4; + static const constexpr size_t InvalidCounterIndex = ~static_cast(0); + static const constexpr NodeIndex RootIndex = 0; + static const constexpr NodeIndex InvalidNodeIndex = + ~static_cast(0); + + // DAG work node. + struct Node { + MARL_NO_EXPORT inline Node() = default; + MARL_NO_EXPORT inline Node(Work&& work); + MARL_NO_EXPORT inline Node(const Work& work); + + // The work to perform for this node in the graph. + Work work; + + // counterIndex if valid, is the index of the counter in the RunContext for + // this node. The counter is decremented for each completed dependency task + // (ins), and once it reaches 0, this node will be invoked. + size_t counterIndex = InvalidCounterIndex; + + // Indices for all downstream nodes. + containers::vector outs; + }; + + // initCounters() allocates and initializes the ctx->coutners from + // initialCounters. + MARL_NO_EXPORT inline void initCounters(RunContext* ctx, + Allocator* allocator); + + // notify() is called each time a dependency task (ins) has completed for the + // node with the given index. + // If all dependency tasks have completed (or this is the root node) then + // notify() returns true and the caller should then call invoke(). + MARL_NO_EXPORT inline bool notify(RunContext*, NodeIndex); + + // invoke() calls the work function for the node with the given index, then + // calls notify() and possibly invoke() for all the dependee nodes. + MARL_NO_EXPORT inline void invoke(RunContext*, NodeIndex, WaitGroup*); + + // nodes is the full list of the nodes in the graph. + // nodes[0] is always the root node, which has no dependencies (ins). + containers::vector nodes; + + // initialCounters is a list of initial counter values to be copied to + // RunContext::counters on DAG<>::run(). + // initialCounters is indexed by Node::counterIndex, and only contains counts + // for nodes that have at least 2 dependencies (ins) - because of this the + // number of entries in initialCounters may be fewer than nodes. + containers::vector initialCounters; +}; + +template +DAGBase::Node::Node(Work&& work) : work(std::move(work)) {} + +template +DAGBase::Node::Node(const Work& work) : work(work) {} + +template +void DAGBase::initCounters(RunContext* ctx, Allocator* allocator) { + auto numCounters = initialCounters.size(); + ctx->counters = allocator->make_unique_n(numCounters); + for (size_t i = 0; i < numCounters; i++) { + ctx->counters.get()[i] = {initialCounters[i]}; + } +} + +template +bool DAGBase::notify(RunContext* ctx, NodeIndex nodeIdx) { + Node* node = &nodes[nodeIdx]; + + // If we have multiple dependencies, decrement the counter and check whether + // we've reached 0. + if (node->counterIndex == InvalidCounterIndex) { + return true; + } + auto counters = ctx->counters.get(); + auto counter = --counters[node->counterIndex]; + return counter == 0; +} + +template +void DAGBase::invoke(RunContext* ctx, NodeIndex nodeIdx, WaitGroup* wg) { + Node* node = &nodes[nodeIdx]; + + // Run this node's work. + if (node->work) { + ctx->invoke(node->work); + } + + // Then call notify() on all dependees (outs), and invoke() those that + // returned true. + // We buffer the node to invoke (toInvoke) so we can schedule() all but the + // last node to invoke(), and directly call the last invoke() on this thread. + // This is done to avoid the overheads of scheduling when a direct call would + // suffice. + NodeIndex toInvoke = InvalidNodeIndex; + for (NodeIndex idx : node->outs) { + if (notify(ctx, idx)) { + if (toInvoke != InvalidNodeIndex) { + wg->add(1); + // Schedule while promoting the WaitGroup capture from a pointer + // reference to a value. This ensures that the WaitGroup isn't dropped + // while in use. + schedule( + [=](WaitGroup wg) { + invoke(ctx, toInvoke, &wg); + wg.done(); + }, + *wg); + } + toInvoke = idx; + } + } + if (toInvoke != InvalidNodeIndex) { + invoke(ctx, toInvoke, wg); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// DAGNodeBuilder +/////////////////////////////////////////////////////////////////////////////// + +// DAGNodeBuilder is the builder interface for a DAG node. +template +class DAGNodeBuilder { + using NodeIndex = typename DAGBase::NodeIndex; + + public: + // then() builds and returns a new DAG node that will be invoked after this + // node has completed. + // + // F is a function that will be called when the new DAG node is invoked, with + // the signature: + // void(T) when T is not void + // or + // void() when T is void + template + MARL_NO_EXPORT inline DAGNodeBuilder then(F&&); + + private: + friend DAGBuilder; + MARL_NO_EXPORT inline DAGNodeBuilder(DAGBuilder*, NodeIndex); + DAGBuilder* builder; + NodeIndex index; +}; + +template +DAGNodeBuilder::DAGNodeBuilder(DAGBuilder* builder, NodeIndex index) + : builder(builder), index(index) {} + +template +template +DAGNodeBuilder DAGNodeBuilder::then(F&& work) { + auto node = builder->node(std::forward(work)); + builder->addDependency(*this, node); + return node; +} + +/////////////////////////////////////////////////////////////////////////////// +// DAGBuilder +/////////////////////////////////////////////////////////////////////////////// +template +class DAGBuilder { + public: + // DAGBuilder constructor + MARL_NO_EXPORT inline DAGBuilder(Allocator* allocator = Allocator::Default); + + // root() returns the root DAG node. + MARL_NO_EXPORT inline DAGNodeBuilder root(); + + // node() builds and returns a new DAG node with no initial dependencies. + // The returned node must be attached to the graph in order to invoke F or any + // of the dependees of this returned node. + // + // F is a function that will be called when the new DAG node is invoked, with + // the signature: + // void(T) when T is not void + // or + // void() when T is void + template + MARL_NO_EXPORT inline DAGNodeBuilder node(F&& work); + + // node() builds and returns a new DAG node that depends on all the tasks in + // after to be completed before invoking F. + // + // F is a function that will be called when the new DAG node is invoked, with + // the signature: + // void(T) when T is not void + // or + // void() when T is void + template + MARL_NO_EXPORT inline DAGNodeBuilder node( + F&& work, + std::initializer_list> after); + + // addDependency() adds parent as dependency on child. All dependencies of + // child must have completed before child is invoked. + MARL_NO_EXPORT inline void addDependency(DAGNodeBuilder parent, + DAGNodeBuilder child); + + // build() constructs and returns the DAG. No other methods of this class may + // be called after calling build(). + MARL_NO_EXPORT inline Allocator::unique_ptr> build(); + + private: + static const constexpr size_t NumReservedNumIns = 4; + using Node = typename DAG::Node; + + // The DAG being built. + Allocator::unique_ptr> dag; + + // Number of dependencies (ins) for each node in dag->nodes. + containers::vector numIns; +}; + +template +DAGBuilder::DAGBuilder(Allocator* allocator /* = Allocator::Default */) + : dag(allocator->make_unique>()), numIns(allocator) { + // Add root + dag->nodes.emplace_back(Node{}); + numIns.emplace_back(0); +} + +template +DAGNodeBuilder DAGBuilder::root() { + return DAGNodeBuilder{this, DAGBase::RootIndex}; +} + +template +template +DAGNodeBuilder DAGBuilder::node(F&& work) { + return node(std::forward(work), {}); +} + +template +template +DAGNodeBuilder DAGBuilder::node( + F&& work, + std::initializer_list> after) { + MARL_ASSERT(numIns.size() == dag->nodes.size(), + "NodeBuilder vectors out of sync"); + auto index = dag->nodes.size(); + numIns.emplace_back(0); + dag->nodes.emplace_back(Node{std::forward(work)}); + auto node = DAGNodeBuilder{this, index}; + for (auto in : after) { + addDependency(in, node); + } + return node; +} + +template +void DAGBuilder::addDependency(DAGNodeBuilder parent, + DAGNodeBuilder child) { + numIns[child.index]++; + dag->nodes[parent.index].outs.push_back(child.index); +} + +template +Allocator::unique_ptr> DAGBuilder::build() { + auto numNodes = dag->nodes.size(); + MARL_ASSERT(numIns.size() == dag->nodes.size(), + "NodeBuilder vectors out of sync"); + for (size_t i = 0; i < numNodes; i++) { + if (numIns[i] > 1) { + auto& node = dag->nodes[i]; + node.counterIndex = dag->initialCounters.size(); + dag->initialCounters.push_back(numIns[i]); + } + } + return std::move(dag); +} + +/////////////////////////////////////////////////////////////////////////////// +// DAG +/////////////////////////////////////////////////////////////////////////////// +template +class DAG : public DAGBase { + public: + using Builder = DAGBuilder; + using NodeBuilder = DAGNodeBuilder; + + // run() invokes the function of each node in the graph of the DAG, passing + // data to each, starting with the root node. All dependencies need to have + // completed their function before dependees will be invoked. + MARL_NO_EXPORT inline void run(T& data, + Allocator* allocator = Allocator::Default); +}; + +template +void DAG::run(T& arg, Allocator* allocator /* = Allocator::Default */) { + typename DAGBase::RunContext ctx{arg}; + this->initCounters(&ctx, allocator); + WaitGroup wg; + this->invoke(&ctx, this->RootIndex, &wg); + wg.wait(); +} + +/////////////////////////////////////////////////////////////////////////////// +// DAG +/////////////////////////////////////////////////////////////////////////////// +template <> +class DAG : public DAGBase { + public: + using Builder = DAGBuilder; + using NodeBuilder = DAGNodeBuilder; + + // run() invokes the function of each node in the graph of the DAG, starting + // with the root node. All dependencies need to have completed their function + // before dependees will be invoked. + MARL_NO_EXPORT inline void run(Allocator* allocator = Allocator::Default); +}; + +void DAG::run(Allocator* allocator /* = Allocator::Default */) { + typename DAGBase::RunContext ctx{}; + this->initCounters(&ctx, allocator); + WaitGroup wg; + this->invoke(&ctx, this->RootIndex, &wg); + wg.wait(); +} + +} // namespace marl + +#endif // marl_dag_h diff --git a/3party/marl/include/marl/debug.h b/3party/marl/include/marl/debug.h new file mode 100644 index 0000000..5d9d665 --- /dev/null +++ b/3party/marl/include/marl/debug.h @@ -0,0 +1,61 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_debug_h +#define marl_debug_h + +#include "export.h" + +#if !defined(MARL_DEBUG_ENABLED) +#if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON) +#define MARL_DEBUG_ENABLED 1 +#else +#define MARL_DEBUG_ENABLED 0 +#endif +#endif + +namespace marl { + +MARL_EXPORT +void fatal(const char* msg, ...); + +MARL_EXPORT +void warn(const char* msg, ...); + +MARL_EXPORT +void assert_has_bound_scheduler(const char* feature); + +#if MARL_DEBUG_ENABLED +#define MARL_FATAL(msg, ...) marl::fatal(msg "\n", ##__VA_ARGS__); +#define MARL_ASSERT(cond, msg, ...) \ + do { \ + if (!(cond)) { \ + MARL_FATAL("ASSERT: " msg, ##__VA_ARGS__); \ + } \ + } while (false); +#define MARL_ASSERT_HAS_BOUND_SCHEDULER(feature) \ + marl::assert_has_bound_scheduler(feature); +#define MARL_UNREACHABLE() MARL_FATAL("UNREACHABLE"); +#define MARL_WARN(msg, ...) marl::warn("WARNING: " msg "\n", ##__VA_ARGS__); +#else +#define MARL_FATAL(msg, ...) +#define MARL_ASSERT(cond, msg, ...) +#define MARL_ASSERT_HAS_BOUND_SCHEDULER(feature) +#define MARL_UNREACHABLE() +#define MARL_WARN(msg, ...) +#endif + +} // namespace marl + +#endif // marl_debug_h diff --git a/3party/marl/include/marl/defer.h b/3party/marl/include/marl/defer.h new file mode 100644 index 0000000..6b78aa2 --- /dev/null +++ b/3party/marl/include/marl/defer.h @@ -0,0 +1,45 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_defer_h +#define marl_defer_h + +#include "finally.h" + +namespace marl { + +#define MARL_CONCAT_(a, b) a##b +#define MARL_CONCAT(a, b) MARL_CONCAT_(a, b) + +// defer() is a macro to defer execution of a statement until the surrounding +// scope is closed and is typically used to perform cleanup logic once a +// function returns. +// +// Note: Unlike golang's defer(), the defer statement is executed when the +// surrounding *scope* is closed, not necessarily the function. +// +// Example usage: +// +// void sayHelloWorld() +// { +// defer(printf("world\n")); +// printf("hello "); +// } +// +#define defer(x) \ + auto MARL_CONCAT(defer_, __LINE__) = marl::make_finally([&] { x; }) + +} // namespace marl + +#endif // marl_defer_h diff --git a/3party/marl/include/marl/deprecated.h b/3party/marl/include/marl/deprecated.h new file mode 100644 index 0000000..6713151 --- /dev/null +++ b/3party/marl/include/marl/deprecated.h @@ -0,0 +1,38 @@ +// Copyright 2020 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_deprecated_h +#define marl_deprecated_h + +#ifndef MARL_WARN_DEPRECATED +#define MARL_WARN_DEPRECATED 1 +#endif // MARL_WARN_DEPRECATED + +#if MARL_WARN_DEPRECATED +#if defined(_WIN32) +#define MARL_DEPRECATED(issue_num, message) \ + __declspec(deprecated( \ + message "\nSee: https://github.com/google/marl/issues/" #issue_num \ + " for more information")) +#else +#define MARL_DEPRECATED(issue_num, message) \ + __attribute__((deprecated( \ + message "\nSee: https://github.com/google/marl/issues/" #issue_num \ + " for more information"))) +#endif +#else +#define MARL_DEPRECATED(issue_num, message) +#endif + +#endif // marl_deprecated_h diff --git a/3party/marl/include/marl/event.h b/3party/marl/include/marl/event.h new file mode 100644 index 0000000..92e1ef8 --- /dev/null +++ b/3party/marl/include/marl/event.h @@ -0,0 +1,250 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_event_h +#define marl_event_h + +#include "conditionvariable.h" +#include "containers.h" +#include "export.h" +#include "memory.h" + +#include + +namespace marl { + +// Event is a synchronization primitive used to block until a signal is raised. +class Event { + public: + enum class Mode : uint8_t { + // The event signal will be automatically reset when a call to wait() + // returns. + // A single call to signal() will only unblock a single (possibly + // future) call to wait(). + Auto, + + // While the event is in the signaled state, any calls to wait() will + // unblock without automatically reseting the signaled state. + // The signaled state can be reset with a call to clear(). + Manual + }; + + MARL_NO_EXPORT inline Event(Mode mode = Mode::Auto, + bool initialState = false, + Allocator* allocator = Allocator::Default); + + // signal() signals the event, possibly unblocking a call to wait(). + MARL_NO_EXPORT inline void signal() const; + + // clear() clears the signaled state. + MARL_NO_EXPORT inline void clear() const; + + // wait() blocks until the event is signaled. + // If the event was constructed with the Auto Mode, then only one + // call to wait() will unblock before returning, upon which the signalled + // state will be automatically cleared. + MARL_NO_EXPORT inline void wait() const; + + // wait_for() blocks until the event is signaled, or the timeout has been + // reached. + // If the timeout was reached, then wait_for() return false. + // If the event is signalled and event was constructed with the Auto Mode, + // then only one call to wait() will unblock before returning, upon which the + // signalled state will be automatically cleared. + template + MARL_NO_EXPORT inline bool wait_for( + const std::chrono::duration& duration) const; + + // wait_until() blocks until the event is signaled, or the timeout has been + // reached. + // If the timeout was reached, then wait_for() return false. + // If the event is signalled and event was constructed with the Auto Mode, + // then only one call to wait() will unblock before returning, upon which the + // signalled state will be automatically cleared. + template + MARL_NO_EXPORT inline bool wait_until( + const std::chrono::time_point& timeout) const; + + // test() returns true if the event is signaled, otherwise false. + // If the event is signalled and was constructed with the Auto Mode + // then the signalled state will be automatically cleared upon returning. + MARL_NO_EXPORT inline bool test() const; + + // isSignalled() returns true if the event is signaled, otherwise false. + // Unlike test() the signal is not automatically cleared when the event was + // constructed with the Auto Mode. + // Note: No lock is held after bool() returns, so the event state may + // immediately change after returning. Use with caution. + MARL_NO_EXPORT inline bool isSignalled() const; + + // any returns an event that is automatically signalled whenever any of the + // events in the list are signalled. + template + MARL_NO_EXPORT inline static Event any(Mode mode, + const Iterator& begin, + const Iterator& end); + + // any returns an event that is automatically signalled whenever any of the + // events in the list are signalled. + // This overload defaults to using the Auto mode. + template + MARL_NO_EXPORT inline static Event any(const Iterator& begin, + const Iterator& end); + + private: + struct Shared { + MARL_NO_EXPORT inline Shared(Allocator* allocator, + Mode mode, + bool initialState); + MARL_NO_EXPORT inline void signal(); + MARL_NO_EXPORT inline void wait(); + + template + MARL_NO_EXPORT inline bool wait_for( + const std::chrono::duration& duration); + + template + MARL_NO_EXPORT inline bool wait_until( + const std::chrono::time_point& timeout); + + marl::mutex mutex; + ConditionVariable cv; + containers::vector, 1> deps; + const Mode mode; + bool signalled; + }; + + const std::shared_ptr shared; +}; + +Event::Shared::Shared(Allocator* allocator, Mode mode_, bool initialState) + : cv(allocator), mode(mode_), signalled(initialState) {} + +void Event::Shared::signal() { + marl::lock lock(mutex); + if (signalled) { + return; + } + signalled = true; + if (mode == Mode::Auto) { + cv.notify_one(); + } else { + cv.notify_all(); + } + for (auto dep : deps) { + dep->signal(); + } +} + +void Event::Shared::wait() { + marl::lock lock(mutex); + cv.wait(lock, [&] { return signalled; }); + if (mode == Mode::Auto) { + signalled = false; + } +} + +template +bool Event::Shared::wait_for( + const std::chrono::duration& duration) { + marl::lock lock(mutex); + if (!cv.wait_for(lock, duration, [&] { return signalled; })) { + return false; + } + if (mode == Mode::Auto) { + signalled = false; + } + return true; +} + +template +bool Event::Shared::wait_until( + const std::chrono::time_point& timeout) { + marl::lock lock(mutex); + if (!cv.wait_until(lock, timeout, [&] { return signalled; })) { + return false; + } + if (mode == Mode::Auto) { + signalled = false; + } + return true; +} + +Event::Event(Mode mode /* = Mode::Auto */, + bool initialState /* = false */, + Allocator* allocator /* = Allocator::Default */) + : shared(allocator->make_shared(allocator, mode, initialState)) {} + +void Event::signal() const { + shared->signal(); +} + +void Event::clear() const { + marl::lock lock(shared->mutex); + shared->signalled = false; +} + +void Event::wait() const { + shared->wait(); +} + +template +bool Event::wait_for(const std::chrono::duration& duration) const { + return shared->wait_for(duration); +} + +template +bool Event::wait_until( + const std::chrono::time_point& timeout) const { + return shared->wait_until(timeout); +} + +bool Event::test() const { + marl::lock lock(shared->mutex); + if (!shared->signalled) { + return false; + } + if (shared->mode == Mode::Auto) { + shared->signalled = false; + } + return true; +} + +bool Event::isSignalled() const { + marl::lock lock(shared->mutex); + return shared->signalled; +} + +template +Event Event::any(Mode mode, const Iterator& begin, const Iterator& end) { + Event any(mode, false); + for (auto it = begin; it != end; it++) { + auto s = it->shared; + marl::lock lock(s->mutex); + if (s->signalled) { + any.signal(); + } + s->deps.push_back(any.shared); + } + return any; +} + +template +Event Event::any(const Iterator& begin, const Iterator& end) { + return any(Mode::Auto, begin, end); +} + +} // namespace marl + +#endif // marl_event_h diff --git a/3party/marl/include/marl/export.h b/3party/marl/include/marl/export.h new file mode 100644 index 0000000..0e4a9f3 --- /dev/null +++ b/3party/marl/include/marl/export.h @@ -0,0 +1,43 @@ +// Copyright 2020 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_export_h +#define marl_export_h + +#ifdef MARL_DLL + +#if MARL_BUILDING_DLL +#define MARL_EXPORT __declspec(dllexport) +#else +#define MARL_EXPORT __declspec(dllimport) +#endif + +#else // #ifdef MARL_DLL + +#if __GNUC__ >= 4 +#define MARL_EXPORT __attribute__((visibility("default"))) +#define MARL_NO_EXPORT __attribute__((visibility("hidden"))) +#endif + +#endif + +#ifndef MARL_EXPORT +#define MARL_EXPORT +#endif + +#ifndef MARL_NO_EXPORT +#define MARL_NO_EXPORT +#endif + +#endif // marl_export_h diff --git a/3party/marl/include/marl/finally.h b/3party/marl/include/marl/finally.h new file mode 100644 index 0000000..17c9b2b --- /dev/null +++ b/3party/marl/include/marl/finally.h @@ -0,0 +1,92 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Finally can be used to execute a lambda or function when the final reference +// to the Finally is dropped. +// +// The purpose of a finally is to perform cleanup or termination logic and is +// especially useful when there are multiple early returns within a function. +// +// A moveable Finally can be constructed with marl::make_finally(). +// A sharable Finally can be constructed with marl::make_shared_finally(). + +#ifndef marl_finally_h +#define marl_finally_h + +#include "export.h" + +#include +#include +#include + +namespace marl { + +// Finally is a pure virtual base class, implemented by the templated +// FinallyImpl. +class Finally { + public: + virtual ~Finally() = default; +}; + +// FinallyImpl implements a Finally. +// The template parameter F is the function type to be called when the finally +// is destructed. F must have the signature void(). +template +class FinallyImpl : public Finally { + public: + MARL_NO_EXPORT inline FinallyImpl(const F& func); + MARL_NO_EXPORT inline FinallyImpl(F&& func); + MARL_NO_EXPORT inline FinallyImpl(FinallyImpl&& other); + MARL_NO_EXPORT inline ~FinallyImpl(); + + private: + FinallyImpl(const FinallyImpl& other) = delete; + FinallyImpl& operator=(const FinallyImpl& other) = delete; + FinallyImpl& operator=(FinallyImpl&&) = delete; + F func; + bool valid = true; +}; + +template +FinallyImpl::FinallyImpl(const F& func_) : func(func_) {} + +template +FinallyImpl::FinallyImpl(F&& func_) : func(std::move(func_)) {} + +template +FinallyImpl::FinallyImpl(FinallyImpl&& other) + : func(std::move(other.func)) { + other.valid = false; +} + +template +FinallyImpl::~FinallyImpl() { + if (valid) { + func(); + } +} + +template +inline FinallyImpl make_finally(F&& f) { + return FinallyImpl(std::forward(f)); +} + +template +inline std::shared_ptr make_shared_finally(F&& f) { + return std::make_shared>(std::forward(f)); +} + +} // namespace marl + +#endif // marl_finally_h diff --git a/3party/marl/include/marl/memory.h b/3party/marl/include/marl/memory.h new file mode 100644 index 0000000..46c67ff --- /dev/null +++ b/3party/marl/include/marl/memory.h @@ -0,0 +1,461 @@ +// Copyright 2019 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef marl_memory_h +#define marl_memory_h + +#include "debug.h" +#include "export.h" + +#include + +#include +#include +#include +#include +#include // std::forward + +namespace marl { + +template +struct StlAllocator; + +// pageSize() returns the size in bytes of a virtual memory page for the host +// system. +MARL_EXPORT +size_t pageSize(); + +template +MARL_NO_EXPORT inline T alignUp(T val, T alignment) { + return alignment * ((val + alignment - 1) / alignment); +} + +// aligned_storage() is a replacement for std::aligned_storage that isn't busted +// on older versions of MSVC. +template +struct aligned_storage { + struct alignas(ALIGNMENT) type { + unsigned char data[SIZE]; + }; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Allocation +/////////////////////////////////////////////////////////////////////////////// + +// Allocation holds the result of a memory allocation from an Allocator. +struct Allocation { + // Intended usage of the allocation. Used for allocation trackers. + enum class Usage : uint8_t { + Undefined = 0, + Stack, // Fiber stack + Create, // Allocator::create(), make_unique(), make_shared() + Vector, // marl::containers::vector + List, // marl::containers::list + Stl, // marl::StlAllocator + Count, // Not intended to be used as a usage type - used for upper bound. + }; + + // Request holds all the information required to make an allocation. + struct Request { + size_t size = 0; // The size of the allocation in bytes. + size_t alignment = 0; // The minimum alignment of the allocation. + bool useGuards = false; // Whether the allocation is guarded. + Usage usage = Usage::Undefined; // Intended usage of the allocation. + }; + + void* ptr = nullptr; // The pointer to the allocated memory. + Request request; // Request used for the allocation. +}; + +/////////////////////////////////////////////////////////////////////////////// +// Allocator +/////////////////////////////////////////////////////////////////////////////// + +// Allocator is an interface to a memory allocator. +// Marl provides a default implementation with Allocator::Default. +class Allocator { + public: + // The default allocator. Initialized with an implementation that allocates + // from the OS. Can be assigned a custom implementation. + MARL_EXPORT static Allocator* Default; + + // Deleter is a smart-pointer compatible deleter that can be used to delete + // objects created by Allocator::create(). Deleter is used by the smart + // pointers returned by make_shared() and make_unique(). + struct MARL_EXPORT Deleter { + MARL_NO_EXPORT inline Deleter(); + MARL_NO_EXPORT inline Deleter(Allocator* allocator, size_t count); + + template + MARL_NO_EXPORT inline void operator()(T* object); + + Allocator* allocator = nullptr; + size_t count = 0; + }; + + // unique_ptr is an alias to std::unique_ptr. + template + using unique_ptr = std::unique_ptr; + + virtual ~Allocator() = default; + + // allocate() allocates memory from the allocator. + // The returned Allocation::request field must be equal to the Request + // parameter. + virtual Allocation allocate(const Allocation::Request&) = 0; + + // free() frees the memory returned by allocate(). + // The Allocation must have all fields equal to those returned by allocate(). + virtual void free(const Allocation&) = 0; + + // create() allocates and constructs an object of type T, respecting the + // alignment of the type. + // The pointer returned by create() must be deleted with destroy(). + template + inline T* create(ARGS&&... args); + + // destroy() destructs and frees the object allocated with create(). + template + inline void destroy(T* object); + + // make_unique() returns a new object allocated from the allocator wrapped + // in a unique_ptr that respects the alignment of the type. + template + inline unique_ptr make_unique(ARGS&&... args); + + // make_unique_n() returns an array of n new objects allocated from the + // allocator wrapped in a unique_ptr that respects the alignment of the + // type. + template + inline unique_ptr make_unique_n(size_t n, ARGS&&... args); + + // make_shared() returns a new object allocated from the allocator + // wrapped in a std::shared_ptr that respects the alignment of the type. + template + inline std::shared_ptr make_shared(ARGS&&... args); + + protected: + Allocator() = default; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Allocator::Deleter +/////////////////////////////////////////////////////////////////////////////// +Allocator::Deleter::Deleter() : allocator(nullptr) {} +Allocator::Deleter::Deleter(Allocator* allocator_, size_t count_) + : allocator(allocator_), count(count_) {} + +template +void Allocator::Deleter::operator()(T* object) { + object->~T(); + + Allocation allocation; + allocation.ptr = object; + allocation.request.size = sizeof(T) * count; + allocation.request.alignment = alignof(T); + allocation.request.usage = Allocation::Usage::Create; + allocator->free(allocation); +} + +/////////////////////////////////////////////////////////////////////////////// +// Allocator +/////////////////////////////////////////////////////////////////////////////// +template +T* Allocator::create(ARGS&&... args) { + Allocation::Request request; + request.size = sizeof(T); + request.alignment = alignof(T); + request.usage = Allocation::Usage::Create; + + auto alloc = allocate(request); + new (alloc.ptr) T(std::forward(args)...); + return reinterpret_cast(alloc.ptr); +} + +template +void Allocator::destroy(T* object) { + object->~T(); + + Allocation alloc; + alloc.ptr = object; + alloc.request.size = sizeof(T); + alloc.request.alignment = alignof(T); + alloc.request.usage = Allocation::Usage::Create; + free(alloc); +} + +template +Allocator::unique_ptr Allocator::make_unique(ARGS&&... args) { + return make_unique_n(1, std::forward(args)...); +} + +template +Allocator::unique_ptr Allocator::make_unique_n(size_t n, ARGS&&... args) { + if (n == 0) { + return nullptr; + } + + Allocation::Request request; + request.size = sizeof(T) * n; + request.alignment = alignof(T); + request.usage = Allocation::Usage::Create; + + auto alloc = allocate(request); + new (alloc.ptr) T(std::forward(args)...); + return unique_ptr(reinterpret_cast(alloc.ptr), Deleter{this, n}); +} + +template +std::shared_ptr Allocator::make_shared(ARGS&&... args) { + Allocation::Request request; + request.size = sizeof(T); + request.alignment = alignof(T); + request.usage = Allocation::Usage::Create; + + auto alloc = allocate(request); + new (alloc.ptr) T(std::forward(args)...); + return std::shared_ptr(reinterpret_cast(alloc.ptr), Deleter{this, 1}); +} + +/////////////////////////////////////////////////////////////////////////////// +// TrackedAllocator +/////////////////////////////////////////////////////////////////////////////// + +// TrackedAllocator wraps an Allocator to track the allocations made. +class TrackedAllocator : public Allocator { + public: + struct UsageStats { + // Total number of allocations. + size_t count = 0; + // total allocation size in bytes (as requested, may be higher due to + // alignment or guards). + size_t bytes = 0; + }; + + struct Stats { + // numAllocations() returns the total number of allocations across all + // usages for the allocator. + inline size_t numAllocations() const; + + // bytesAllocated() returns the total number of bytes allocated across all + // usages for the allocator. + inline size_t bytesAllocated() const; + + // Statistics per usage. + std::array byUsage; + }; + + // Constructor that wraps an existing allocator. + inline TrackedAllocator(Allocator* allocator); + + // stats() returns the current allocator statistics. + inline Stats stats(); + + // Allocator compliance + inline Allocation allocate(const Allocation::Request&) override; + inline void free(const Allocation&) override; + + private: + Allocator* const allocator; + std::mutex mutex; + Stats stats_; +}; + +size_t TrackedAllocator::Stats::numAllocations() const { + size_t out = 0; + for (auto& stats : byUsage) { + out += stats.count; + } + return out; +} + +size_t TrackedAllocator::Stats::bytesAllocated() const { + size_t out = 0; + for (auto& stats : byUsage) { + out += stats.bytes; + } + return out; +} + +TrackedAllocator::TrackedAllocator(Allocator* allocator_) + : allocator(allocator_) {} + +TrackedAllocator::Stats TrackedAllocator::stats() { + std::unique_lock lock(mutex); + return stats_; +} + +Allocation TrackedAllocator::allocate(const Allocation::Request& request) { + { + std::unique_lock lock(mutex); + auto& usageStats = stats_.byUsage[int(request.usage)]; + ++usageStats.count; + usageStats.bytes += request.size; + } + return allocator->allocate(request); +} + +void TrackedAllocator::free(const Allocation& allocation) { + { + std::unique_lock lock(mutex); + auto& usageStats = stats_.byUsage[int(allocation.request.usage)]; + MARL_ASSERT(usageStats.count > 0, + "TrackedAllocator detected abnormal free()"); + MARL_ASSERT(usageStats.bytes >= allocation.request.size, + "TrackedAllocator detected abnormal free()"); + --usageStats.count; + usageStats.bytes -= allocation.request.size; + } + return allocator->free(allocation); +} + +/////////////////////////////////////////////////////////////////////////////// +// StlAllocator +/////////////////////////////////////////////////////////////////////////////// + +// StlAllocator exposes an STL-compatible allocator wrapping a marl::Allocator. +template +struct StlAllocator { + using value_type = T; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = size_t; + using difference_type = size_t; + + // An equivalent STL allocator for a different type. + template + struct rebind { + typedef StlAllocator other; + }; + + // Constructs an StlAllocator that will allocate using allocator. + // allocator must remain valid until this StlAllocator has been destroyed. + inline StlAllocator(Allocator* allocator); + + template + inline StlAllocator(const StlAllocator& other); + + // Returns the actual address of x even in presence of overloaded operator&. + inline pointer address(reference x) const; + inline const_pointer address(const_reference x) const; + + // Allocates the memory for n objects of type T. + // Does not actually construct the objects. + inline T* allocate(std::size_t n); + + // Deallocates the memory for n objects of type T. + inline void deallocate(T* p, std::size_t n); + + // Returns the maximum theoretically possible number of T stored in this + // allocator. + inline size_type max_size() const; + + // Copy constructs an object of type T at the address p. + inline void construct(pointer p, const_reference val); + + // Constructs an object of type U at the address P forwarning all other + // arguments to the constructor. + template + inline void construct(U* p, Args&&... args); + + // Deconstructs the object at p. It does not free the memory. + inline void destroy(pointer p); + + // Deconstructs the object at p. It does not free the memory. + template + inline void destroy(U* p); + + private: + inline Allocation::Request request(size_t n) const; + + template + friend struct StlAllocator; + Allocator* allocator; +}; + +template +StlAllocator::StlAllocator(Allocator* allocator_) : allocator(allocator_) {} + +template +template +StlAllocator::StlAllocator(const StlAllocator& other) { + allocator = other.allocator; +} + +template +typename StlAllocator::pointer StlAllocator::address(reference x) const { + return &x; +} +template +typename StlAllocator::const_pointer StlAllocator::address( + const_reference x) const { + return &x; +} + +template +T* StlAllocator::allocate(std::size_t n) { + auto alloc = allocator->allocate(request(n)); + return reinterpret_cast(alloc.ptr); +} + +template +void StlAllocator::deallocate(T* p, std::size_t n) { + Allocation alloc; + alloc.ptr = p; + alloc.request = request(n); + allocator->free(alloc); +} + +template +typename StlAllocator::size_type StlAllocator::max_size() const { + return std::numeric_limits::max() / sizeof(value_type); +} + +template +void StlAllocator::construct(pointer p, const_reference val) { + new (p) T(val); +} + +template +template +void StlAllocator::construct(U* p, Args&&... args) { + ::new ((void*)p) U(std::forward(args)...); +} + +template +void StlAllocator::destroy(pointer p) { + ((T*)p)->~T(); +} + +template +template +void StlAllocator::destroy(U* p) { + p->~U(); +} + +template +Allocation::Request StlAllocator::request(size_t n) const { + Allocation::Request req = {}; + req.size = sizeof(T) * n; + req.alignment = alignof(T); + req.usage = Allocation::Usage::Stl; + return req; +} + +} // namespace marl + +#endif // marl_memory_h diff --git a/3party/marl/include/marl/mutex.h b/3party/marl/include/marl/mutex.h new file mode 100644 index 0000000..674ec33 --- /dev/null +++ b/3party/marl/include/marl/mutex.h @@ -0,0 +1,109 @@ +// Copyright 2020 The Marl Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wrappers around std::mutex and std::unique_lock that provide clang's +// Thread Safety Analysis annotations. +// See: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html + +#ifndef marl_mutex_h +#define marl_mutex_h + +#include "export.h" +#include "tsa.h" + +#include +#include + +namespace marl { + +// mutex is a wrapper around std::mutex that offers Thread Safety Analysis +// annotations. +// mutex also holds methods for performing std::condition_variable::wait() calls +// as these require a std::unique_lock<> which are unsupported by the TSA. +class CAPABILITY("mutex") mutex { + public: + MARL_NO_EXPORT inline void lock() ACQUIRE() { _.lock(); } + + MARL_NO_EXPORT inline void unlock() RELEASE() { _.unlock(); } + + MARL_NO_EXPORT inline bool try_lock() TRY_ACQUIRE(true) { + return _.try_lock(); + } + + // wait_locked calls cv.wait() on this already locked mutex. + template + MARL_NO_EXPORT inline void wait_locked(std::condition_variable& cv, + Predicate&& p) REQUIRES(this) { + std::unique_lock lock(_, std::adopt_lock); + cv.wait(lock, std::forward(p)); + lock.release(); // Keep lock held. + } + + // wait_until_locked calls cv.wait() on this already locked mutex. + template + MARL_NO_EXPORT inline bool wait_until_locked(std::condition_variable& cv, + Time&& time, + Predicate&& p) REQUIRES(this) { + std::unique_lock lock(_, std::adopt_lock); + auto res = cv.wait_until(lock, std::forward