Implementation in C++ of HTTPTransport test server

- Pulls in cpp-httplib for test-only usage in third_party/.
- Replaces http_transport_test_server.py with .cc server.
- Remove unnecessary Go toolchain pull. This was planned to be used for
  the test server, but the toolchain integration was too messy when
  covering all target platforms/configs.

Bug: crashpad:196, crashpad:227, crashpad:30
Change-Id: I5990781473dcadfcc036fbe711c02928638ff851
Reviewed-on: https://chromium-review.googlesource.com/1013293
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Commit-Queue: Scott Graham <scottmg@chromium.org>

.gitignore

@@ -18,7 +18,6 @@
 /third_party/linux/.cipd
 /third_party/linux/clang
 /third_party/linux/sysroot
-/third_party/go
 /third_party/gyp/gyp
 /third_party/mini_chromium/mini_chromium
 /third_party/zlib/zlib

DEPS

@@ -114,45 +114,6 @@ hooks = [
       'buildtools/win/gn.exe.sha1',
     ],
   },
-  {
-    'name': 'go_toolchain_mac',
-    'condition': 'host_os == "mac"',
-    'pattern': '.',
-    'action': [
-      'cipd',
-      'install',
-      'infra/go/mac-amd64',
-      'latest',
-      '-root', 'crashpad/third_party/go/mac-amd64',
-      '-log-level', 'info',
-    ],
-  },
-  {
-    'name': 'go_toolchain_linux',
-    'condition': 'host_os == "linux"',
-    'pattern': '.',
-    'action': [
-      'cipd',
-      'install',
-      'infra/go/linux-amd64',
-      'latest',
-      '-root', 'crashpad/third_party/go/linux-amd64',
-      '-log-level', 'info',
-    ],
-  },
-  {
-    'name': 'go_toolchain_win',
-    'condition': 'host_os == "win"',
-    'pattern': '.',
-    'action': [
-      'cipd',
-      'install',
-      'infra/go/windows-amd64',
-      'latest',
-      '-root', 'crashpad/third_party/go/windows-amd64',
-      '-log-level', 'info',
-    ],
-  },
   {
     # This uses “cipd install” so that mac-amd64 and linux-amd64 can coexist
     # peacefully. “cipd ensure” would remove the macOS package when running on a

third_party/cpp-httplib/BUILD.gn

@@ -0,0 +1,21 @@
+# Copyright 2018 The Crashpad Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("cpp-httplib") {
+  testonly = true
+  include_dirs = [ "cpp-httplib" ]
+  sources = [
+    "cpp-httplib/httplib.h",
+  ]
+}

third_party/cpp-httplib/README.crashpad

@@ -0,0 +1,15 @@
+Name: cpp-httplib
+Short Name: cpp-httplib
+URL: https://github.com/yhirose/cpp-httplib
+Revision: 4320d7ba3e8b7388e1443eb54c039a1304cf7a6b
+License: MIT
+License File: cpp-httplib/LICENSE
+Security Critical: no (test only)
+
+Description:
+A C++11 header-only HTTP library.
+
+Local Modifications:
+- Exclude test/ and example/ subdirs.
+- Patch httplib.h to use #include "third_party/zlib/zlib_crashpad.h" instead of
+  <zlib.h>.

third_party/cpp-httplib/cpp-httplib/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 yhirose
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

third_party/cpp-httplib/cpp-httplib/README.md

@@ -0,0 +1,188 @@
+cpp-httplib
+===========
+
+A C++11 header-only HTTP library.
+
+It's extremely easy to setup. Just include **httplib.h** file in your code!
+
+Inspired by [Sinatra](http://www.sinatrarb.com/) and [express](https://github.com/visionmedia/express).
+
+Server Example
+--------------
+
+```c++
+#include <httplib.h>
+
+int main(void)
+{
+    using namespace httplib;
+
+    Server svr;
+
+    svr.get("/hi", [](const Request& req, Response& res) {
+        res.set_content("Hello World!", "text/plain");
+    });
+
+    svr.get(R"(/numbers/(\d+))", [&](const Request& req, Response& res) {
+        auto numbers = req.matches[1];
+        res.set_content(numbers, "text/plain");
+    });
+
+    svr.listen("localhost", 1234);
+}
+```
+
+### Method Chain
+
+```cpp
+svr.get("/get", [](const auto& req, auto& res) {
+        res.set_content("get", "text/plain");
+    })
+    .post("/post", [](const auto& req, auto& res) {
+        res.set_content(req.body(), "text/plain");
+    })
+    .listen("localhost", 1234);
+```
+
+### Static File Server
+
+```cpp
+svr.set_base_dir("./www");
+```
+
+### Logging
+
+```cpp
+svr.set_logger([](const auto& req, const auto& res) {
+    your_logger(req, res);
+});
+```
+
+### Error Handler
+
+```cpp
+svr.set_error_handler([](const auto& req, auto& res) {
+    const char* fmt = "<p>Error Status: <span style='color:red;'>%d</span></p>";
+    char buf[BUFSIZ];
+    snprintf(buf, sizeof(buf), fmt, res.status);
+    res.set_content(buf, "text/html");
+});
+```
+
+### 'multipart/form-data' POST data
+
+```cpp
+svr.post("/multipart", [&](const auto& req, auto& res) {
+    auto size = req.files.size();
+    auto ret = req.has_file("name1"));
+    const auto& file = req.get_file_value("name1");
+    // file.filename;
+    // file.content_type;
+    auto body = req.body.substr(file.offset, file.length));
+})
+```
+
+Client Example
+--------------
+
+### GET
+
+```c++
+#include <httplib.h>
+#include <iostream>
+
+int main(void)
+{
+    httplib::Client cli("localhost", 1234);
+
+    auto res = cli.get("/hi");
+    if (res && res->status == 200) {
+        std::cout << res->body << std::endl;
+    }
+}
+```
+
+### POST
+
+```c++
+res = cli.post("/post", "text", "text/plain");
+res = cli.post("/person", "name=john1&note=coder", "application/x-www-form-urlencoded");
+```
+
+### POST with parameters
+
+```c++
+httplib::Map params;
+params["name"] = "john";
+params["note"] = "coder";
+auto res = cli.post("/post", params);
+```
+
+### Connection Timeout
+
+```c++
+httplib::Client cli("localhost", 8080, 5); // timeouts in 5 seconds
+```
+### With Progress Callback
+
+```cpp
+httplib::Client client(url, port);
+
+// prints: 0 / 000 bytes => 50% complete
+std::shared_ptr<httplib::Response> res =
+    cli.get("/", [](uint64_t len, uint64_t total) {
+        printf("%lld / %lld bytes => %d%% complete\n",
+            len, total,
+            (int)((len/total)*100));
+    }
+);
+```
+
+![progress](https://user-images.githubusercontent.com/236374/33138910-495c4ecc-cf86-11e7-8693-2fc6d09615c4.gif)
+
+This feature was contributed by [underscorediscovery](https://github.com/yhirose/cpp-httplib/pull/23).
+
+### Range
+
+```cpp
+httplib::Client cli("httpbin.org", 80);
+
+// 'Range: bytes=1-10'
+httplib::Headers headers = { httplib::make_range_header(1, 10) };
+
+auto res = cli.get("/range/32", headers);
+// res->status should be 206.
+// res->body should be "bcdefghijk".
+```
+
+OpenSSL Support
+---------------
+
+SSL support is available with `CPPHTTPLIB_OPENSSL_SUPPORT`. `libssl` and `libcrypto` should be linked.
+
+```c++
+#define CPPHTTPLIB_OPENSSL_SUPPORT
+
+SSLServer svr("./cert.pem", "./key.pem");
+
+SSLClient cli("localhost", 8080);
+```
+
+Zlib Support
+------------
+
+'gzip' compression is available with `CPPHTTPLIB_ZLIB_SUPPORT`.
+
+The server applies gzip compression to the following MIME type contents:
+
+  * all text types
+  * image/svg+xml
+  * application/javascript
+  * application/json
+  * application/xml
+  * application/xhtml+xml
+
+License
+-------
+
+MIT license (© 2017 Yuji Hirose)

util/BUILD.gn

@@ -448,6 +448,29 @@ static_library("util") {
   }
 }
 
+crashpad_executable("http_transport_test_server") {
+  testonly = true
+  sources = [
+    "net/http_transport_test_server.cc",
+  ]
+
+  deps = [
+    ":util",
+    "../third_party/cpp-httplib",
+    "../third_party/mini_chromium:base",
+    "../third_party/zlib",
+    "../tools:tool_support",
+  ]
+
+  remove_configs = [
+    "//third_party/mini_chromium/mini_chromium/build:Wexit_time_destructors",
+  ]
+
+  if (crashpad_is_win) {
+    libs = [ "ws2_32.lib" ]
+  }
+}
+
 source_set("util_test") {
   testonly = true
 
@@ -589,7 +612,6 @@ source_set("util_test") {
   }
 
   data = [
-    "net/http_transport_test_server.py",
     "net/testdata/",
   ]
 
@@ -604,6 +626,10 @@ source_set("util_test") {
     "../third_party/zlib",
   ]
 
+  data_deps = [
+    ":http_transport_test_server",
+  ]
+
   if (crashpad_is_mac) {
     libs = [ "Foundation.framework" ]
   }
@@ -613,7 +639,7 @@ source_set("util_test") {
       "rpcrt4.lib",
       "dbghelp.lib",
     ]
-    data_deps = [
+    data_deps += [
       ":crashpad_util_test_process_info_test_child",
       ":crashpad_util_test_safe_terminate_process_test_child",
     ]

util/net/http_transport_test.cc

@@ -56,19 +56,13 @@ class HTTPTransportTestFixture : public MultiprocessExec {
         body_stream_(std::move(body_stream)),
         response_code_(http_response_code),
         request_validator_(request_validator) {
-    base::FilePath server_path = TestPaths::TestDataRoot().Append(
+    base::FilePath server_path = TestPaths::Executable().DirName().Append(
-        FILE_PATH_LITERAL("util/net/http_transport_test_server.py"));
+        FILE_PATH_LITERAL("http_transport_test_server")
-#if defined(OS_POSIX)
+#if defined(OS_WIN)
+            FILE_PATH_LITERAL(".exe")
+#endif
+        );
     SetChildCommand(server_path, nullptr);
-#elif defined(OS_WIN)
-    // Explicitly invoke a shell and python so that python can be found in the
-    // path, and run the test script.
-    std::vector<std::string> args;
-    args.push_back("/c");
-    args.push_back("python");
-    args.push_back(base::UTF16ToUTF8(server_path.value()));
-    SetChildCommand(base::FilePath(_wgetenv(L"COMSPEC")), &args);
-#endif  // OS_POSIX
   }
 
   const HTTPHeaders& headers() { return headers_; }

util/net/http_transport_test_server.cc

@@ -0,0 +1,114 @@
+// Copyright 2018 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A one-shot testing webserver.
+//
+// When invoked, this server will write a short integer to stdout, indicating on
+// which port the server is listening. It will then read one integer from stdin,
+// indicating the response code to be sent in response to a request. It also
+// reads 16 characters from stdin, which, after having "\r\n" appended, will
+// form the response body in a successful response (one with code 200). The
+// server will process one HTTP request, deliver the prearranged response to the
+// client, and write the entire request to stdout. It will then terminate.
+
+#include "base/logging.h"
+#include "base/strings/stringprintf.h"
+#include "build/build_config.h"
+#include "tools/tool_support.h"
+#include "util/file/file_io.h"
+
+#if COMPILER_MSVC
+#pragma warning(push)
+#pragma warning(disable: 4244 4245 4267 4702)
+#endif
+
+#define CPPHTTPLIB_ZLIB_SUPPORT
+#include "third_party/cpp-httplib/cpp-httplib/httplib.h"
+
+#if COMPILER_MSVC
+#pragma warning(pop)
+#endif
+
+namespace crashpad {
+namespace {
+
+int HttpTransportTestServerMain(int argc, char* argv[]) {
+  httplib::Server svr(httplib::HttpVersion::v1_0);
+
+  if (!svr.is_valid()) {
+    LOG(ERROR) << "server creation failed";
+    return 1;
+  }
+
+  uint16_t response_code;
+  char response[16];
+
+  std::string to_stdout;
+
+  svr.post("/upload",
+           [&response, &response_code, &svr, &to_stdout](
+               const httplib::Request& req, httplib::Response& res) {
+             res.status = response_code;
+             if (response_code == 200) {
+               res.set_content(std::string(response, 16) + "\r\n",
+                               "text/plain");
+             } else {
+               res.set_content("error", "text/plain");
+             }
+
+             for (const auto& h : req.headers) {
+               to_stdout += base::StringPrintf(
+                   "%s: %s\r\n", h.first.c_str(), h.second.c_str());
+             }
+             to_stdout += "\r\n";
+             to_stdout += req.body;
+
+             svr.stop();
+           });
+
+  int port = svr.bind_to_any_port("127.0.0.1");
+
+  CheckedWriteFile(
+      StdioFileHandle(StdioStream::kStandardOutput), &port, sizeof(port));
+
+  CheckedReadFileExactly(StdioFileHandle(StdioStream::kStandardInput),
+                         &response_code,
+                         sizeof(response_code));
+
+  CheckedReadFileExactly(StdioFileHandle(StdioStream::kStandardInput),
+                         &response,
+                         sizeof(response));
+
+  svr.listen_after_bind();
+
+  LoggingWriteFile(StdioFileHandle(StdioStream::kStandardOutput),
+                   to_stdout.data(),
+                   to_stdout.size());
+
+  return 0;
+}
+
+} // namespace
+} // namespace crashpad
+
+#if defined(OS_POSIX) || defined(OS_FUCHSIA)
+int main(int argc, char* argv[]) {
+  return crashpad::HttpTransportTestServerMain(argc, argv);
+}
+#elif defined(OS_WIN)
+int wmain(int argc, wchar_t* argv[]) {
+  return crashpad::ToolSupport::Wmain(
+      argc, argv, crashpad::HttpTransportTestServerMain);
+}
+#endif  // OS_POSIX

util/net/http_transport_test_server.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# Copyright 2014 The Crashpad Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""A one-shot testing webserver.
-
-When invoked, this server will write a short integer to stdout, indiciating on
-which port the server is listening. It will then read one integer from stdin,
-indiciating the response code to be sent in response to a request. It also reads
-16 characters from stdin, which, after having "\r\n" appended, will form the
-response body in a successful response (one with code 200). The server will
-process one HTTP request, deliver the prearranged response to the client, and
-write the entire request to stdout. It will then terminate.
-
-This server is written in Python since it provides a simple HTTP stack, and
-because parsing chunked encoding is safer and easier in a memory-safe language.
-This could easily have been written in C++ instead.
-"""
-
-import os
-import struct
-import sys
-import zlib
-
-if sys.platform == 'win32':
-  import msvcrt
-
-if sys.version_info[0] < 3:
-  import BaseHTTPServer as http_server
-else:
-  import http.server as http_server
-
-
-class BufferedReadFile(object):
-  """A File-like object that stores all read contents into a buffer."""
-
-  def __init__(self, real_file):
-    self.file = real_file
-    self.buffer = b''
-
-  def read(self, size=-1):
-    buf = self.file.read(size)
-    self.buffer += buf
-    return buf
-
-  def readline(self, size=-1):
-    buf = self.file.readline(size)
-    self.buffer += buf
-    return buf
-
-  def flush(self):
-    self.file.flush()
-
-  def close(self):
-    self.file.close()
-
-
-class RequestHandler(http_server.BaseHTTPRequestHandler):
-  # Everything to be written to stdout is collected into this string. It can’t
-  # be written to stdout until after the HTTP transaction is complete, because
-  # stdout is a pipe being read by a test program that’s also the HTTP client.
-  # The test program expects to complete the entire HTTP transaction before it
-  # even starts reading this script’s stdout. If the stdout pipe buffer fills up
-  # during an HTTP transaction, deadlock would result.
-  raw_request = b''
-
-  response_code = 500
-  response_body = b''
-
-  def handle_one_request(self):
-    # Wrap the rfile in the buffering file object so that the raw header block
-    # can be written to stdout after it is parsed.
-    self.rfile = BufferedReadFile(self.rfile)
-    http_server.BaseHTTPRequestHandler.handle_one_request(self)
-
-  def do_POST(self):
-    RequestHandler.raw_request = self.rfile.buffer
-    self.rfile.buffer = b''
-
-    if self.headers.get('Transfer-Encoding', '').lower() == 'chunked':
-      if 'Content-Length' in self.headers:
-        raise AssertionError
-      body = self.handle_chunked_encoding()
-    else:
-      length = int(self.headers.get('Content-Length', -1))
-      body = self.rfile.read(length)
-
-    if self.headers.get('Content-Encoding', '').lower() == 'gzip':
-      # 15 is the value of |wbits|, which should be at the maximum possible
-      # value to ensure that any gzip stream can be decoded. The offset of 16
-      # specifies that the stream to decompress will be formatted with a gzip
-      # wrapper.
-      body = zlib.decompress(body, 16 + 15)
-
-    RequestHandler.raw_request += body
-
-    self.send_response(self.response_code)
-    self.end_headers()
-    if self.response_code == 200:
-      self.wfile.write(self.response_body)
-      self.wfile.write(b'\r\n')
-
-  def handle_chunked_encoding(self):
-    """This parses a "Transfer-Encoding: Chunked" body in accordance with RFC
-    7230 §4.1. This returns the result as a string.
-    """
-    body = b''
-    chunk_size = self.read_chunk_size()
-    while chunk_size > 0:
-      # Read the body.
-      data = self.rfile.read(chunk_size)
-      chunk_size -= len(data)
-      body += data
-
-      # Finished reading this chunk.
-      if chunk_size == 0:
-        # Read through any trailer fields.
-        trailer_line = self.rfile.readline()
-        while trailer_line.strip() != b'':
-          trailer_line = self.rfile.readline()
-
-        # Read the chunk size.
-        chunk_size = self.read_chunk_size()
-    return body
-
-  def read_chunk_size(self):
-    # Read the whole line, including the \r\n.
-    chunk_size_and_ext_line = self.rfile.readline()
-    # Look for a chunk extension.
-    chunk_size_end = chunk_size_and_ext_line.find(b';')
-    if chunk_size_end == -1:
-      # No chunk extensions; just encounter the end of line.
-      chunk_size_end = chunk_size_and_ext_line.find(b'\r')
-    if chunk_size_end == -1:
-      self.send_response(400)  # Bad request.
-      return -1
-    return int(chunk_size_and_ext_line[:chunk_size_end], base=16)
-
-  def log_request(self, code='-', size='-'):
-    # The default implementation logs these to sys.stderr, which is just noise.
-    pass
-
-
-def StdioBinaryEquivalent(file):
-  """Return a file object equivalent to sys.stdin or sys.stdout capable of
-  reading or writing binary “bytes”.
-
-  struct.unpack consumes “bytes”, and struct.pack produces “bytes”. These are
-  distinct from “str” in Python 3 (but not 2). In order to read and write these
-  from stdin and stdout, the underlying binary buffer must be used in place of
-  the upper-layer text wrapper. This function returns a suitable file.
-
-  There is no underlying buffer in Python 2, but on Windows, the file mode must
-  still be set to binary in order to cleanly pass binary data. Note that in this
-  case, the mode of |file| itself is changed, as it’s not distinct from the
-  returned file.
-  """
-  if hasattr(file, 'buffer'):
-    file = file.buffer
-  elif sys.platform == 'win32':
-    msvcrt.setmode(file.fileno(), os.O_BINARY)
-  return file
-
-
-def Main():
-  in_file = StdioBinaryEquivalent(sys.stdin)
-  out_file = StdioBinaryEquivalent(sys.stdout)
-
-  # Start the server.
-  server = http_server.HTTPServer(('127.0.0.1', 0), RequestHandler)
-
-  # Write the port as an unsigned short to the parent process.
-  out_file.write(struct.pack('=H', server.server_address[1]))
-  out_file.flush()
-
-  # Read the desired test response code as an unsigned short and the desired
-  # response body as a 16-byte string from the parent process.
-  RequestHandler.response_code, RequestHandler.response_body = \
-      struct.unpack('=H16s', in_file.read(struct.calcsize('=H16s')))
-
-  # Handle the request.
-  server.handle_request()
-
-  # Share the entire request with the test program, which will validate it.
-  out_file.write(RequestHandler.raw_request)
-  out_file.flush()
-
-if __name__ == '__main__':
-  Main()