tile/tile/net/http/http_client.cc

#include "tile/net/http/http_client.h"

#include "tile/base/enum.h"
#include "tile/base/internal/background_task_host.h"
#include "tile/base/logging.h"
#include "tile/base/thread/latch.h"
#include "tile/net/internal/http_engine.h"
#include "tile/net/internal/http_task.h"

#include "curl/curl.h"
#include "gflags/gflags.h"

DEFINE_int32(tile_http_client_default_timeout_ms, 1000, "Default timeout of tile::HttpClient.");

namespace tile {
namespace {
HttpClient::ErrorCode
GetErrorCodeFromCurlCode(int c)
{
    switch (c) {
    case CURLE_UNSUPPORTED_PROTOCOL:
        return HttpClient::ERROR_PROTOCOL_NOT_SUPPORTED;
    case CURLE_URL_MALFORMAT:
        return HttpClient::ERROR_INVALID_URI_ADDRESS;
    case CURLE_COULDNT_RESOLVE_PROXY:
        return HttpClient::ERROR_PROXY;
    case CURLE_COULDNT_RESOLVE_HOST:
        return HttpClient::ERROR_FAIL_TO_RESOLVE_ADDRESS;
    case CURLE_COULDNT_CONNECT:
        return HttpClient::ERROR_CONNECTION;
    case CURLE_HTTP2:
    case CURLE_HTTP2_STREAM:
        return HttpClient::ERROR_HTTP2;
    case CURLE_HTTP_RETURNED_ERROR:
    case CURLE_HTTP_POST_ERROR:
        TILE_LOG_WARNING_EVERY_SECOND("ERROR_CURL_HTTP_ERROR CURLcode {}", c);
        return HttpClient::ERROR_INTERNAL_ERROR;
    case CURLE_OPERATION_TIMEDOUT:
        return HttpClient::ERROR_TIMEOUT;
    case CURLE_SSL_CONNECT_ERROR:
    case CURLE_SSL_ENGINE_NOTFOUND:
    case CURLE_SSL_ENGINE_SETFAILED:
    case CURLE_SSL_CERTPROBLEM:
    case CURLE_SSL_CIPHER:
    case CURLE_PEER_FAILED_VERIFICATION:
        TILE_LOG_WARNING_EVERY_SECOND("ERROR_SSL CURLcode {}", c);
        return HttpClient::ERROR_SSL;
    case CURLE_SEND_ERROR:
    case CURLE_RECV_ERROR:
        return HttpClient::ERROR_IO;
    case CURLE_TOO_MANY_REDIRECTS:
        return HttpClient::ERROR_TOO_MANY_REDIRECTS;
    case CURLE_GOT_NOTHING:
        return HttpClient::ERROR_GET_NOTHING;

    default: {
        TILE_LOG_WARNING_EVERY_SECOND(
            "ERROR_UNKNOWN CURLcode {}, curl msg: ", c, curl_easy_strerror(static_cast<CURLcode>(c)));
        return HttpClient::ERROR_UNKNOWN;
    }
    }
}

long
TileHttpVersionToCurlHttpVersion(HttpVersion v, bool no_automatic_upgrade)
{
    if (no_automatic_upgrade) { return CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE; }
    switch (v) {
    case HttpVersion::Unspecified:
        return CURL_HTTP_VERSION_NONE;
    case HttpVersion::V_1_0:
        return CURL_HTTP_VERSION_1_0;
    case HttpVersion::V_1_1:
        return CURL_HTTP_VERSION_1_1;
    case HttpVersion::V_2:
        return CURL_HTTP_VERSION_2_0;
    case HttpVersion::V_3:
        return CURL_HTTP_VERSION_3;
    default:
        TILE_CHECK(0, "Unknown flare http version");
    }
    return CURL_HTTP_VERSION_NONE;
}

HttpVersion
CurlHttpVersionToHttpVersion(long v)
{
    switch (v) {
    case CURL_HTTP_VERSION_NONE:
        return HttpVersion::Unspecified;
    case CURL_HTTP_VERSION_1_0:
        return HttpVersion::V_1_0;
    case CURL_HTTP_VERSION_1_1:
        return HttpVersion::V_1_1;
    case CURL_HTTP_VERSION_2_0:
        return HttpVersion::V_2;
    case CURL_HTTP_VERSION_3:
        return HttpVersion::V_3;
    default:
        TILE_CHECK(0, "Unknown flare http version {}", v);
    }
    return HttpVersion::Unspecified;
}

void
FillResponseInfo(CURL *easy_handler, HttpClient::ResponseInfo *response_info)
{
    TILE_CHECK(response_info);
    TILE_CHECK(easy_handler);

    {
        char *effective_url = nullptr;
        curl_easy_getinfo(easy_handler, CURLINFO_EFFECTIVE_URL, &effective_url);
        response_info->effective_url = effective_url;
    }

    {
        double transfer_time_in_seconds = 0;
        curl_easy_getinfo(easy_handler, CURLINFO_TOTAL_TIME, &transfer_time_in_seconds);
        response_info->total_time_transfer = std::chrono::duration_cast<std::chrono::nanoseconds>(
            std::chrono::duration<double>(transfer_time_in_seconds));
    }

    {
        long http_version;
        curl_easy_getinfo(easy_handler, CURLINFO_HTTP_VERSION, &http_version);
        response_info->http_version = CurlHttpVersionToHttpVersion(http_version);
    }
}

std::optional<std::string>
GetHttpRequestUriFromUrl(const std::string &url)
{
    // http://example.com/xxx
    // http://example.com

    auto start_pos = url.find("://");
    if (start_pos != std::string::npos) {
        start_pos += 3;

        auto end_pos = url.find("/", start_pos);
        if (end_pos != std::string::npos) { return url.substr(end_pos); }
    }

    return std::nullopt;
}

std::optional<std::pair<std::string, std::function<void(bool)>>>
OverrideHost(const std::string &url, const std::string &override_host_nslb)
{
    return std::make_pair(url, nullptr);
}

class HttpEngineWrapper : public detail::HttpChannel {
public:
    static HttpEngineWrapper *Instance()
    {
        static NeverDestroyedSingleton<HttpEngineWrapper> engine;
        return engine.Get();
    }

    internal::HttpTask GetHttpTask(const std::string &url,
                                   const HttpClient::Options opts,
                                   const HttpClient::RequestOptions &request_options)
    {
        internal::HttpTask task;
        task.SetUrl(url);
        task.SetTimeout(request_options.timeout);
        CURL *h = task.GetNativeHandle();
        if (opts.follow_redirects) {
            TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_FOLLOWLOCATION, 1L));
            TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_MAXREDIRS, request_options.max_redirection_count));
        }

        if (request_options.verbose) { TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_VERBOSE, 1L)); }

        if (opts.use_bintin_compression) {
            TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_ACCEPT_ENCODING, "identity, gzip, deflate"));
        }

        TILE_CHECK_EQ(
            CURLE_OK,
            curl_easy_setopt(
                h, CURLOPT_HTTP_VERSION,
                TileHttpVersionToCurlHttpVersion(request_options.http_version, request_options.no_automatic_upgrade)));

        for (auto &&s : request_options.headers) { task.AddHeader(s); }

        if (!request_options.content_type.empty()) { task.AddHeader("Content-Type: " + request_options.content_type); }

        if (!opts.verify_server_certificate) {
            TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_SSL_VERIFYPEER, 0));
            TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_SSL_VERIFYHOST, 0));
        }

        if (!opts.proxy_from_env) { TILE_CHECK_EQ(CURLE_OK, curl_easy_setopt(h, CURLOPT_PROXY, opts.proxy.c_str())); }

        return task;
    }

    void AsyncCallCallback(std::function<void(std::expected<HttpResponse, HttpClient::ErrorCode>)> done,
                           std::function<void(bool)> report_function,
                           std::expected<internal::HttpTaskCompletion, Status> completion,
                           HttpClient::ResponseInfo *response_info)
    {
        auto moved_report_function = MakeMoveOnCopy(report_function);
        auto moved_done            = MakeMoveOnCopy(done);
        auto moved_completion      = MakeMoveOnCopy(completion);

        internal::BackgroundTaskHost::Instance()->Queue(
            [moved_report_function, moved_completion, moved_done, response_info] {
                auto &&report_function = moved_report_function.Ref();
                auto &&completion      = moved_completion.Ref();
                auto &&done            = moved_done.Ref();
                if (report_function) { report_function(!!completion); }

                std::optional<HttpResponse> opt_response;
                if (completion) {
                    HttpResponse response;
                    response.set_version(completion->version());
                    response.set_status(completion->status());
                    response.set_body(std::move(*completion->body()));
                    CopyHeaders(*completion->headers(), response.headers());
                    opt_response = std::move(response);
                }

                if (completion && response_info) { FillResponseInfo(completion->GetNativeHandle(), response_info); }

                if (!completion) {
                    done(std::make_unexpected(GetErrorCodeFromCurlCode(completion.error().code())));
                } else {
                    TILE_CHECK(opt_response.has_value());
                    done(std::move(*opt_response));
                }
            });
    }

    template<class F>
    void AsyncCall(HttpMethod method,
                   const std::string &url,
                   const HttpClient::Options &opts,
                   const HttpClient::RequestOptions &request_options,
                   HttpClient::ResponseInfo *response_info,
                   std::function<void(std::expected<HttpResponse, HttpClient::ErrorCode>)> &&done,
                   F &&callback)
    {
        auto &&override_host = OverrideHost(url, request_options.override_host_nslb);
        if (!override_host) {
            done(std::make_unexpected(HttpClient::ERROR_FAIL_TO_RESOLVE_ADDRESS));
            return;
        }

        auto &&effective_url   = override_host->first;
        auto &&report_function = override_host->second;
        auto task              = GetHttpTask(effective_url, opts, request_options);
        task.SetMethod(method);
        std::unique_ptr<HttpRequest> request = make_unique<HttpRequest>();
        request->set_version(request_options.http_version);
        for (auto &&header : request_options.headers) {
            auto pos = header.find(':');
            if (pos != std::string::npos) {
                request->headers()->Append(header.substr(0, pos), Trim(header.substr(pos + 1)));
            }
        }

        callback(&task, request.get());
        auto moved_report_function = MakeMoveOnCopy(report_function);
        auto moved_done            = MakeMoveOnCopy(done);
        internal::HttpEngine::Instance()->StartTask(
            std::move(task),
            [this, moved_report_function, moved_done,
             response_info](std::expected<internal::HttpTaskCompletion, Status> &&completion) mutable {
                AsyncCallCallback(moved_done.Move(), moved_report_function.Move(), std::move(completion),
                                  response_info);
            });
    }

    void AsyncGet(const std::string &url,
                  const HttpClient::Options &opts,
                  const HttpClient::RequestOptions &request_options,
                  HttpClient::ResponseInfo *response_info,
                  std::function<void(std::expected<HttpResponse, HttpClient::ErrorCode>)> &&done) override
    {

        auto cb = [url](internal::HttpTask *, HttpRequest *p_request) {
            if (p_request) {
                p_request->set_method(HttpMethod::Get);
                auto uri = GetHttpRequestUriFromUrl(url);
                if (uri) { p_request->set_uri(*uri); }
            }
        };
        return AsyncCall(HttpMethod::Get, url, opts, request_options, response_info, std::move(done), std::move(cb));
    }

    void AsyncPost(const std::string &url,
                   const HttpClient::Options &opts,
                   std::string data,
                   const HttpClient::RequestOptions &request_options,
                   HttpClient::ResponseInfo *response_info,
                   std::function<void(std::expected<HttpResponse, HttpClient::ErrorCode>)> &&done) override
    {
        auto moved_data = MakeMoveOnCopy(data);

        auto cb         = [url, moved_data](internal::HttpTask *p_task, HttpRequest *p_request) {
            if (p_request) {
                p_request->set_method(HttpMethod::Post);
                auto uri = GetHttpRequestUriFromUrl(url);
                if (uri) { p_request->set_uri(*uri); }
                p_request->set_body(moved_data.Ref());
            }
            p_task->SetBody(moved_data.Move());
        };
        return AsyncCall(HttpMethod::Post, url, opts, request_options, response_info, std::move(done), std::move(cb));
    }

    void AsyncRequest(const std::string &protocol,
                      const std::string &host,
                      const HttpClient::Options &opts,
                      const HttpRequest &request,
                      const HttpClient::RequestOptions &request_options,
                      HttpClient::ResponseInfo *response_info,
                      std::function<void(std::expected<HttpResponse, HttpClient::ErrorCode>)> &&done) override
    {
        auto url = protocol + "://" + host + request.uri();
        auto cb  = [request](internal::HttpTask *p_task, HttpRequest *p_request) {
            if (p_request) { *p_request = request; }

            if (request.body_size() > 0) {
                if (request.noncontiguous_body()) {
                    p_task->SetBody(*request.noncontiguous_body());
                } else {
                    p_task->SetBody(*request.body());
                }
            }

            for (auto header : *request.headers()) { p_task->AddHeader(header.first + ": " + header.second); }
            TILE_CHECK(request.method() != HttpMethod::Unspecified, "You should specify http method.");
            TILE_CHECK_EQ(
                CURLE_OK,
                curl_easy_setopt(p_task->GetNativeHandle(), CURLOPT_CUSTOMREQUEST, ToSlice(request.method()).data()));
        };
        return AsyncCall(request.method(), url, opts, request_options, response_info, std::move(done), std::move(cb));
    }

private:
    static void CopyHeaders(const std::vector<std::string> &from, HttpHeaders *to)
    {
        for (auto &&s : from) {
            auto pos = s.find(":");
            if (pos != std::string::npos) {
                auto name  = s.substr(0, pos);
                auto value = s.substr(pos + 1);
                TILE_CHECK(!EndsWith(value, "\r\n"));
                to->Append(std::string(name), std::string(Trim(value)));
            }
        }
    }
};

detail::HttpChannel *mock_channel    = nullptr;
detail::HttpChannel *dry_run_channel = nullptr;

bool
IsMockAddress(const std::string &url_or_protocol, bool is_url)
{
    if (is_url) { return StartsWith(url_or_protocol, "mock://"); }
    return url_or_protocol == "mock";
}

detail::HttpChannel *
GetHttpChannel(const std::string &url_or_protocol, bool is_url = true)
{
    if (TILE_UNLIKELY(IsMockAddress(url_or_protocol, is_url))) {
        TILE_CHECK(mock_channel, "Mock channel has not been registered yet.");
        return mock_channel;
    }

    return HttpEngineWrapper::Instance();
}

}// namespace

namespace detail {}// namespace detail

HttpClient::HttpClient(const Options &options) : options_(options) {}

const char *
HttpClient::ErrorCodeToString(int error_code)
{
    switch (static_cast<ErrorCode>(error_code)) {
    case ERROR_INVALID:
        return "Invalid";
    case ERROR_INVALID_URI_ADDRESS:
        return "Invalid URI address";
    case ERROR_FAIL_TO_RESOLVE_ADDRESS:
        return "Failed to resolve address";
    case ERROR_FAIL_TO_SEND_REQUEST:
        return "Failed to send request";
    case ERROR_FAIL_TO_GET_RESPONSE:
        return "Failed to get response";
    case ERROR_CONNECTION:
        return "Connection io error";
    case ERROR_TIMEOUT:
        return "Response timeout";
    case ERROR_PROXY:
        return "ERROR_PROXY";
    case ERROR_PARSE_RESPONSE:
        return "Failed to parse response";
    case ERROR_FAIL_TO_CONNECT_SERVER:
        return "Failed to connect to server";
    case ERROR_PROTOCOL_NOT_SUPPORTED:
        return "Protocol is not supported";
    case ERROR_TOO_MANY_REDIRECTS:
        return "Too many redirections";
    case ERROR_REDIRECT_LOCATION_NOT_FOUND:
        return "redict location not found";
    case ERROR_DECOMPRESS_RESPONSE:
        return "Failed to decompress response";
    case ERROR_HTTP2:
        return "ERROR_HTTP2";
    case ERROR_SSL:
        return "ERROR_SSL";
    case ERROR_IO:
        return "ERROR_IO";
    case ERROR_INTERNAL_ERROR:
        return "ERROR_INTERNAL_ERROR";
    case ERROR_DRY_RUN:
        return "ERROR_DRY_RUN";
    case ERROR_GET_NOTHING:
        return "ERROR_GET_NOTHING";
    case ERROR_UNKNOWN:
        return "<Unknown>";
    }
    return "<Unknown>";
}

std::expected<HttpResponse, HttpClient::ErrorCode>
HttpClient::Get(const std::string &url, const RequestOptions &request_options, ResponseInfo *response_info)
{
    Latch l(1);
    std::expected<HttpResponse, HttpClient::ErrorCode> e;
    GetHttpChannel(url)->AsyncGet(url, options_, request_options, response_info,
                                  [&](std::expected<HttpResponse, HttpClient::ErrorCode> res) mutable {
                                      e = std::move(res);
                                      l.CountDown();
                                  });
    l.Wait();
    return e;
}

std::expected<HttpResponse, HttpClient::ErrorCode>
HttpClient::Post(const std::string &url,
                 const std::string &data,
                 const RequestOptions &request_options,
                 ResponseInfo *response_info)
{
    Latch l(1);
    std::expected<HttpResponse, HttpClient::ErrorCode> e;
    GetHttpChannel(url)->AsyncPost(url, options_, data, request_options, response_info,
                                   [&](std::expected<HttpResponse, HttpClient::ErrorCode> res) mutable {
                                       e = std::move(res);
                                       l.CountDown();
                                   });
    l.Wait();
    return e;
}

std::expected<HttpResponse, HttpClient::ErrorCode>
HttpClient::Request(const std::string &protocol,
                    const std::string &host,
                    const HttpRequest &request,
                    const RequestOptions &request_options,
                    ResponseInfo *response_info)
{
    Latch l(1);
    std::expected<HttpResponse, HttpClient::ErrorCode> e;
    GetHttpChannel(protocol)->AsyncRequest(protocol, host, options_, request, request_options, response_info,
                                           [&](std::expected<HttpResponse, HttpClient::ErrorCode> res) mutable {
                                               e = std::move(res);
                                               l.CountDown();
                                           });
    l.Wait();
    return e;
}

Future<std::expected<HttpResponse, HttpClient::ErrorCode>>
HttpClient::AsyncGet(const std::string &url, const RequestOptions &request_options, ResponseInfo *response_info)
{
    Promise<std::expected<HttpResponse, ErrorCode>> promise;
    auto future        = promise.GetFuture();
    auto moved_promise = MakeMoveOnCopy(promise);
    GetHttpChannel(url)->AsyncGet(url, options_, request_options, response_info,
                                  [moved_promise](std::expected<HttpResponse, ErrorCode> res) mutable {
                                      moved_promise->SetValue(std::move(res));
                                  });
    return future;
}

Future<std::expected<HttpResponse, HttpClient::ErrorCode>>
HttpClient::AsyncPost(const std::string &url,
                      const std::string &data,
                      const RequestOptions &request_options,
                      ResponseInfo *response_info)
{
    Promise<std::expected<HttpResponse, ErrorCode>> promise;
    auto future        = promise.GetFuture();
    auto moved_promise = MakeMoveOnCopy(promise);
    GetHttpChannel(url)->AsyncPost(url, options_, data, request_options, response_info,
                                   [moved_promise](std::expected<HttpResponse, ErrorCode> res) mutable {
                                       moved_promise->SetValue(std::move(res));
                                   });
    return future;
}

Future<std::expected<HttpResponse, HttpClient::ErrorCode>>
HttpClient::AsyncRequest(const std::string &protocol,
                         const std::string &host,
                         const HttpRequest &request,
                         const RequestOptions &request_options,
                         ResponseInfo *response_info)
{
    Promise<std::expected<HttpResponse, ErrorCode>> promise;
    auto future        = promise.GetFuture();
    auto moved_promise = MakeMoveOnCopy(promise);
    GetHttpChannel(protocol)->AsyncRequest(protocol, host, options_, request, request_options, response_info,
                                           [moved_promise](std::expected<HttpResponse, ErrorCode> res) mutable {
                                               moved_promise->SetValue(std::move(res));
                                           });
    return future;
}

}// namespace tile