Compare commits

...

2 Commits

Author SHA1 Message Date
tqcq
1cc2382bdf feat add rate_limiter
Some checks reported errors
test/pipeline/head Something is wrong with the build of this commit
2024-08-05 19:21:43 +08:00
tqcq
0e93d0b1f3 feat remove mustache 2024-08-05 16:42:16 +08:00
15 changed files with 344 additions and 14677 deletions

View File

@ -193,6 +193,7 @@ set(TILE_SRCS
"tile/io/detail/eintr_safe.cc"
"tile/io/native/acceptor.cc"
"tile/io/descriptor.cc"
"tile/io/util/rate_limiter.cc"
"tile/io/event_loop.cc"
"tile/init.cc"
"tile/init/on_init.cc"
@ -291,6 +292,7 @@ if(TILE_BUILD_TESTS)
add_test(NAME ${test_name} COMMAND ${test_name})
endmacro()
tile_add_test(io_util_rate_limiter_test "tile/io/util/rate_limiter_test.cc")
tile_add_test(base_exposed_var_test "tile/base/exposed_var_test.cc")
# tile_add_test(fiber_detail_scheduler_test "tile/fiber/detail/scheduler_test.cc")
tile_add_test(base_internal_meta_test "tile/base/internal/meta_test.cc")

View File

@ -1,5 +0,0 @@
/build
/build_xcode
/build64
mustache
mustache14

View File

@ -1,26 +0,0 @@
language: cpp
sudo: required
dist: trusty
matrix:
include:
- os: linux
compiler: gcc
- os: linux
compiler: clang
- os: osx
compiler: clang
script:
- make coverage
after_success:
# Use "-X gcov" to disable Codecov from running gcov. We already run this
# command in "make coverage" and then delete unwanted coverage files that
# contribute falsely to the coverage report, such as catch.hpp
# Only send Linux (GCC) code coverage as the clang ones seem to be buggy and
# report curly braces as unexecuted lines.
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then bash <(curl -s https://codecov.io/bash) -X gcov; fi

View File

@ -1,19 +0,0 @@
cmake_minimum_required(VERSION 2.8)
project(mustache)
if (UNIX)
add_definitions(
-Wall
-Wextra
-Werror
-std=c++11
)
elseif (MSVC)
add_definitions(
/W3
/WX
)
endif()
add_executable(mustache
mustache.hpp # to show in IDE
tests.cpp
)

View File

@ -1,25 +0,0 @@
Boost Software License - Version 1.0
Copyright 2015-2020 Kevin Wojniak
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

View File

@ -1,31 +0,0 @@
default:
g++ -O3 -Wall -Wextra -Werror -std=c++11 -o mustache tests.cpp
./mustache
mac:
clang++ -O3 -Wall -Wextra -Werror -std=c++11 -stdlib=libc++ -o mustache tests.cpp
./mustache
mac14:
clang++ -O3 -Wall -Wextra -Werror -std=c++14 -stdlib=libc++ -o mustache14 tests.cpp
./mustache14
clang:
clang++ -O3 -Wall -Wextra -Werror -std=c++11 -o mustache tests.cpp
# https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html
coverage:
g++ -std=c++11 -coverage -O0 -o mustache tests.cpp
./mustache
gcov -l tests.cpp
# We only want coverage for mustache.hpp and tests.cpp, so delete all the other *.gcov files
find . -type f -name 'tests.cpp*.gcov' ! -name 'tests.cpp.gcov' ! -name 'tests.cpp##mustache.hpp.gcov' -delete
xcode:
mkdir -p build_xcode
cd build_xcode && cmake -GXcode ..
open build_xcode/*.xcodeproj
clean:
rm -rf mustache mustache14 build build_xcode
rm -rf *.gcov *.gcda *.gcno # coverage artifacts

View File

@ -1,125 +0,0 @@
# About
* [Mustache](http://mustache.github.io) implementation for modern C++ (requires C++11)
* Header only
* Zero dependencies
* Templated string type for compatibility with any STL-like string (std::string, std::wstring, etc)
* Boost license
[![travis](https://travis-ci.org/kainjow/Mustache.svg?branch=master)](https://travis-ci.org/kainjow/Mustache) [![appveyor](https://ci.appveyor.com/api/projects/status/6uh5d5weajrffkyw?svg=true)](https://ci.appveyor.com/project/kainjow/mustache) [![codecov](https://codecov.io/gh/kainjow/Mustache/branch/master/graph/badge.svg)](https://codecov.io/gh/kainjow/Mustache)
## Example usage
All examples assume `using namespace kainjow::mustache`. Additional examples and usage can be found in the `tests.cpp` file.
### Example 1 - Hello World
````cpp
mustache tmpl{"Hello {{what}}!"};
std::cout << tmpl.render({"what", "World"}) << std::endl;
// Hello World!
````
### Example 2 - Lists
````cpp
mustache tmpl{"{{#employees}}{{name}}, {{/employees}}"};
data employees{data::type::list};
employees << data{"name", "Steve"} << data{"name", "Bill"};
tmpl.render({"employees", employees}, std::cout);
// Steve, Bill,
````
### Example 3 - Custom Render Handler
````cpp
mustache tmpl("Hello {{what}}!");
std::stringstream ss;
tmpl.render({"what", "World"}, [&ss](const std::string& str) {
ss << str;
});
// ss.str() == "Hello World!"
````
## Supported Features
This library supports all current Mustache features:
- Variables
- HTML escaping
- Sections
- Inverted Sections
- True/False
- Lists
- Lambdas
- Partials
- Comments
- Set Delimiter
Additional features:
- Custom escape function for use outside of HTML
## Run Tests
For *nix:
make
For macOS:
make mac
For Visual Studio 2013 (CMake 2.8+ required):
build.bat
For Visual Studio 2015 (CMake 3.1+ required):
build.bat 14
## Release Notes
#### 4.1 - April 18, 2020
* Fixed incorrect results when using lambda renderers
#### 4.0 - October 28, 2019
* Lines with sections that result in an empty line are removed, per the Mustache spec.
#### 3.2.1 - July 22, 2018
* Add an overload to render() that accepts a context and a stream (thanks Kitsune Ral)
* Added checks for empty objects (thanks Snafuuz)
* Refactored parser in preparation for future changes
#### 3.2 - February 24, 2018
* Added ability to provide a custom escape function (thanks to Kitsune Ral)
* Allow `data.set()` to override an existing value
#### 3.1 - July 22, 2017
* Added a new lambda type (innovatively called `lambda2`) that takes an additional render function. It will not render its result but allows the user to call the `render` argument to render the section text, or any other text.
#### 3.0 - July 8, 2017
* Performance improvements - about 45% faster than version 2
* Even simpler API. Not backwards compatible but upgrading should be straightforward:
* Namespace, classes, and methods are now in snake case to match the STL. For example, `Kainjow::Mustache` is now `kainjow::mustache`
* Classes and aliases are now under a `mustache` namespace, instead of being under the mustache class
* Removed `Data::List()` - use `data{data::type::list}` instead
* Removed `Data::type()` - use the various `is_xxx` methods to identity the type
#### 2.0 - June 11, 2016
* New simpler API (not backwards compatible)
* std::wstring support
* Bug fixes (thanks to Shen-Ta Hsieh)
* Automated tests on OS X
* 100% test coverage
#### 1.0 - April 19, 2015
* All current Mustache features are implemented.

View File

@ -1,18 +0,0 @@
branches:
only:
- master
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
CMAKE_GENERATOR: Visual Studio 12 Win64
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: Visual Studio 14 Win64
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: Visual Studio 15 Win64
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
platform: x64
build_script:
- cmake -G "%CMAKE_GENERATOR%" .
- cmake --build . --config Release
- Release\mustache.exe

View File

@ -1,50 +0,0 @@
:: don't output commands
@echo off
:: get version from input args
:: default value is 12
:: only 12 and 14 are supported
set vers=%1
if "%vers%" == "" set vers=12
if "%vers%" neq "12" (
if "%vers%" neq "14" (
echo Invalid version "%vers%" - expected 12 or 14.
exit /b 1
)
)
:: make build directory
if not exist build mkdir build
pushd build
:: run CMake
cmake -G "Visual Studio %vers%" ..
if %errorlevel% neq 0 popd & exit /b %errorlevel%
:: build with CMake
cmake --build . --config Release
if %errorlevel% neq 0 popd & exit /b %errorlevel%
:: run tests
Release\mustache.exe
if %errorlevel% neq 0 popd & exit /b %errorlevel%
popd
:: make build64 directory
if not exist build64 mkdir build64
pushd build64
:: run CMake
cmake -G "Visual Studio %vers% Win64" ..
if %errorlevel% neq 0 popd & exit /b %errorlevel%
:: build with CMake
cmake --build . --config Release
if %errorlevel% neq 0 popd & exit /b %errorlevel%
:: run tests
Release\mustache.exe
if %errorlevel% neq 0 popd & exit /b %errorlevel%
popd

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,84 @@
#include "tile/io/util/rate_limiter.h"
#include "tile/base/chrono.h"
#include "tile/base/logging.h"
namespace tile {
namespace {
class NullLimiter : public RateLimiter {
public:
std::size_t GetQuota() override { return 0; }
void ConsumeBytes(std::size_t consumed) override {}
};
} // namespace
RateLimiter *RateLimiter::GetDefaultRxRateLimiter() {
static NullLimiter null_limiter;
return &null_limiter;
}
RateLimiter *RateLimiter::GetDefaultTxRateLimiter() {
static NullLimiter null_limiter;
return &null_limiter;
}
TokenBucketRateLimiter::TokenBucketRateLimiter(std::size_t bucket_quota,
std::size_t quota_per_tick,
std::chrono::nanoseconds tick,
bool over_consumption_allowed)
: max_quota_(bucket_quota), quota_per_tick_(quota_per_tick), tick_(tick),
over_consumption_allowed_(over_consumption_allowed) {
TILE_CHECK_GT(bucket_quota, 0);
TILE_CHECK_GT(quota_per_tick, 0);
last_refill_ = ReadSteadyClock().time_since_epoch() / tick_;
curr_quota_ = max_quota_;
}
std::size_t TokenBucketRateLimiter::GetQuota() {
auto now = ReadSteadyClock().time_since_epoch() / tick_;
std::uint64_t last_refill = internal::Exchange(last_refill_, now);
curr_quota_ += quota_per_tick_ * (now - last_refill);
if (curr_quota_ > 0) {
curr_quota_ = std::min<std::size_t>(curr_quota_, max_quota_);
return curr_quota_;
} else {
return 0;
}
}
void TokenBucketRateLimiter::ConsumeBytes(std::size_t consumed) {
TILE_CHECK(over_consumption_allowed_ || consumed <= curr_quota_);
curr_quota_ -= consumed;
}
ThreadSafeRateLimiter::ThreadSafeRateLimiter(MaybeOwning<RateLimiter> limiter,
std::size_t burst_limit)
: burst_limit_(burst_limit), impl_(std::move(limiter)) {
TILE_CHECK_GT(burst_limit_, 0);
}
std::size_t ThreadSafeRateLimiter::GetQuota() {
std::lock_guard<std::mutex> lock(lock_);
return impl_->GetQuota();
}
void ThreadSafeRateLimiter::ConsumeBytes(std::size_t consumed) {
std::lock_guard<std::mutex> lock(lock_);
impl_->ConsumeBytes(consumed);
}
LayeredRateLimiter::LayeredRateLimiter(RateLimiter *upper,
MaybeOwning<RateLimiter> ours)
: upper_(upper), ours_(std::move(ours)) {}
std::size_t LayeredRateLimiter::GetQuota() {
return std::min(upper_->GetQuota(), ours_->GetQuota());
}
void LayeredRateLimiter::ConsumeBytes(std::size_t consumed) {
upper_->ConsumeBytes(consumed);
ours_->ConsumeBytes(consumed);
}
} // namespace tile

View File

@ -0,0 +1,70 @@
#ifndef TILE_IO_UTIL_RATE_LIMITER_H
#define TILE_IO_UTIL_RATE_LIMITER_H
#pragma once
#include "tile/base/maybe_owning.h"
#include <limits>
#include <mutex>
namespace tile {
class RateLimiter {
public:
virtual ~RateLimiter() = default;
virtual std::size_t GetQuota() = 0;
virtual void ConsumeBytes(std::size_t consumed) = 0;
static RateLimiter *GetDefaultRxRateLimiter();
static RateLimiter *GetDefaultTxRateLimiter();
};
class TokenBucketRateLimiter : public RateLimiter {
public:
TokenBucketRateLimiter(
std::size_t bucket_quota, std::size_t quota_per_tick,
std::chrono::nanoseconds tick = std::chrono::milliseconds(1),
bool over_consumption_allowed = true);
std::size_t GetQuota() override;
void ConsumeBytes(std::size_t consumed) override;
private:
std::size_t max_quota_;
std::size_t quota_per_tick_;
std::chrono::nanoseconds tick_;
bool over_consumption_allowed_;
std::uint64_t last_refill_;
std::int64_t curr_quota_{0};
};
class ThreadSafeRateLimiter : public RateLimiter {
public:
explicit ThreadSafeRateLimiter(
MaybeOwning<RateLimiter> limiter,
std::size_t burst_limit = std::numeric_limits<std::size_t>::max());
std::size_t GetQuota() override;
void ConsumeBytes(std::size_t consumed) override;
private:
std::size_t burst_limit_;
std::mutex lock_;
MaybeOwning<RateLimiter> impl_;
};
class LayeredRateLimiter : public RateLimiter {
public:
LayeredRateLimiter(RateLimiter *upper, MaybeOwning<RateLimiter> ours);
std::size_t GetQuota() override;
void ConsumeBytes(std::size_t consumed) override;
private:
RateLimiter *upper_;
MaybeOwning<RateLimiter> ours_;
};
} // namespace tile
#endif // TILE_IO_UTIL_RATE_LIMITER_H

View File

@ -0,0 +1,188 @@
#include "rate_limiter.h"
#include "gtest/gtest.h"
#include "tile/base/chrono.h"
#include "tile/base/make_unique.h"
#include "tile/base/random.h"
namespace tile {
TEST(RateLimiter, TokenBucketRateLimiter) {
TokenBucketRateLimiter limiter(1000, 1);
std::size_t total = 0;
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = limiter.GetQuota();
total += current;
limiter.ConsumeBytes(current);
// with sleep
std::this_thread::sleep_for(std::chrono::milliseconds(1) * Random(100));
}
ASSERT_NEAR(6000, total, 200);
}
TEST(RateLimiter, TokenBucketRateLimiter2) {
TokenBucketRateLimiter limiter(1000, 1);
std::size_t total = 0;
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = limiter.GetQuota();
total += current;
limiter.ConsumeBytes(current);
}
ASSERT_NEAR(6000, total, 200);
}
TEST(RateLimiter, TokenBucketRateLimiterCapBurst) {
TokenBucketRateLimiter limiter(25, 500);
for (int i = 0; i != 10; ++i) {
ASSERT_EQ(25, limiter.GetQuota());
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
TEST(RateLimiter, TokenBucketRateLimiterCapBurst2) {
TokenBucketRateLimiter limiter(1000, 500);
for (int i = 0; i != 10; ++i) {
ASSERT_EQ(1000, limiter.GetQuota());
std::this_thread::sleep_for(
std::chrono::milliseconds(10)); // Enough to fully fill the bucket.
}
}
TEST(RateLimiter, MultithreadedRateLimiter) {
ThreadSafeRateLimiter limiter(make_unique<TokenBucketRateLimiter>(1000, 1));
std::atomic<std::size_t> total{0};
std::vector<std::thread> ts;
for (int i = 0; i != 10; ++i) {
ts.emplace_back(std::thread([&] {
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = limiter.GetQuota();
total += current;
limiter.ConsumeBytes(current);
std::this_thread::sleep_for(std::chrono::milliseconds(10) * Random(10));
}
}));
}
for (auto &&t : ts) {
t.join();
}
ASSERT_NEAR(6000, total.load(), 500);
}
TEST(RateLimiter, LayeredRateLimiter) {
ThreadSafeRateLimiter base_limiter(
make_unique<TokenBucketRateLimiter>(1000, 1));
auto our_limiter = make_unique<ThreadSafeRateLimiter>(
make_unique<TokenBucketRateLimiter>(1000, 100));
LayeredRateLimiter layered_limiter(&base_limiter, std::move(our_limiter));
std::atomic<std::size_t> total{0};
std::vector<std::thread> ts;
for (int i = 0; i != 10; ++i) {
ts.emplace_back(std::thread([&] {
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = layered_limiter.GetQuota();
total += current;
layered_limiter.ConsumeBytes(current);
std::this_thread::sleep_for(std::chrono::milliseconds(10) * Random(10));
}
}));
}
for (auto &&t : ts) {
t.join();
}
ASSERT_NEAR(6000, total.load(), 500); // `msrl` takes effect.
}
TEST(RateLimiter, LayeredRateLimiter2) {
ThreadSafeRateLimiter base_limiter(
make_unique<TokenBucketRateLimiter>(1000, 100));
auto our_limiter = make_unique<ThreadSafeRateLimiter>(
make_unique<TokenBucketRateLimiter>(1000, 1));
LayeredRateLimiter layered_limiter(&base_limiter, std::move(our_limiter));
std::atomic<std::size_t> total{0};
std::vector<std::thread> ts;
for (int i = 0; i != 10; ++i) {
ts.emplace_back(std::thread([&] {
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = layered_limiter.GetQuota();
total += current;
layered_limiter.ConsumeBytes(current);
std::this_thread::sleep_for(std::chrono::milliseconds(1) * Random(10));
}
}));
}
for (auto &&t : ts) {
t.join();
}
ASSERT_NEAR(6000, total.load(), 500); // `tbsrl` takes effect.
}
TEST(RateLimiter, LayeredRateLimiter3) {
ThreadSafeRateLimiter base_limiter(
make_unique<TokenBucketRateLimiter>(1000, 1));
std::atomic<std::size_t> total{0};
std::vector<std::thread> ts;
for (int i = 0; i != 10; ++i) {
ts.emplace_back(std::thread([&] {
auto our_limiter = make_unique<TokenBucketRateLimiter>(1000, 100);
LayeredRateLimiter layered_limiter(&base_limiter, std::move(our_limiter));
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = layered_limiter.GetQuota();
total += current;
layered_limiter.ConsumeBytes(current);
}
}));
}
for (auto &&t : ts) {
t.join();
}
ASSERT_NEAR(6000, total.load(), 500); // `msrl` takes effect.
}
TEST(RateLimiter, LayeredRateLimiter4) {
ThreadSafeRateLimiter base_limiter(
make_unique<TokenBucketRateLimiter>(1000, 100));
std::atomic<std::size_t> total{0};
std::vector<std::thread> ts;
for (int i = 0; i != 10; ++i) {
ts.emplace_back(std::thread([&] {
auto our_limiter = make_unique<ThreadSafeRateLimiter>(
make_unique<TokenBucketRateLimiter>(1000, 1));
LayeredRateLimiter layered_limiter(&base_limiter, std::move(our_limiter));
auto start = ReadSteadyClock();
while (ReadSteadyClock() - start < std::chrono::seconds(5)) {
auto current = layered_limiter.GetQuota();
total += current;
layered_limiter.ConsumeBytes(current);
}
}));
}
for (auto &&t : ts) {
t.join();
}
ASSERT_NEAR(60000, total.load(), 5000); // `tbsrl` takes effect.
}
} // namespace tile