fuchsia: Add runner, get crashpad_test_test building and running

- Implement build/run_tests.py to run on Fuchsia device
- Implement paths_fuchsia.cc using standard Fuchsia namespace layout
- Exclude multiprocess tests, currently unimplemented
- Don't use unnecessary O_ flags on Fuchsia in open() call.

Bug: crashpad:196, chromium:726124, ZX-797
Change-Id: Ie59dce685b4c3fe54f3e36f357c1101d402ee8b7
Reviewed-on: https://chromium-review.googlesource.com/802180
Commit-Queue: Scott Graham <scottmg@chromium.org>
Reviewed-by: Mark Mentovai <mark@chromium.org>

build/run_tests.py

@@ -18,8 +18,129 @@
 from __future__ import print_function
 
 import os
+import pipes
 import subprocess
 import sys
+import uuid
+
+CRASHPAD_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                            os.pardir)
+IS_WINDOWS_HOST = sys.platform.startswith('win')
+
+
+def _GetFuchsiaSDKRoot():
+  arch = 'mac-amd64' if sys.platform == 'darwin' else 'linux-amd64'
+  return os.path.join(CRASHPAD_DIR, 'third_party', 'fuchsia', 'sdk', arch)
+
+
+def _BinaryDirLooksLikeFuchsiaBuild(binary_dir):
+  """Checks whether the provided output directory targets Fuchsia."""
+  popen = subprocess.Popen(
+      ['gn', 'args', binary_dir, '--list=target_os', '--short'],
+      shell=IS_WINDOWS_HOST, stdout=subprocess.PIPE, stderr=open(os.devnull))
+  value = popen.communicate()[0]
+  return popen.returncode == 0 and 'target_os = "fuchsia"' in value
+
+
+def _GenerateFuchsiaRuntimeDepsFiles(binary_dir, tests):
+  """Ensures a <binary_dir>/<test>.runtime_deps file exists for each test."""
+  targets_file = os.path.abspath(os.path.join(binary_dir, 'targets.txt'))
+  with open(targets_file, 'wb') as f:
+    f.write('//:' + '\n//:'.join(tests) + '\n')
+  subprocess.check_call(
+      ['gn', 'gen', binary_dir, '--runtime-deps-list-file=' + targets_file])
+
+
+def _HandleOutputFromFuchsiaLogListener(process, done_message):
+  """Pass through the output from |process| (which should be an instance of
+  Fuchsia's loglistener) until a special termination |done_message| is
+  encountered.
+
+  Also attempts to determine if any tests failed by inspecting the log output,
+  and returns False if there were failures.
+  """
+  success = True
+  while True:
+    line = process.stdout.readline().rstrip()
+    if 'FAILED TEST' in line:
+      success = False
+    elif done_message in line and 'echo ' not in line:
+      break
+    print(line)
+  return success
+
+
+def _RuntimeDepsPathToFuchsiaTargetPath(runtime_dep):
+  """Determines the target location for a given Fuchsia runtime dependency file.
+
+  If the file is in the build directory, then it's stored in /bin, otherwise
+  in /assets. This is only a rough heuristic, but is sufficient for the current
+  data set.
+  """
+  norm = os.path.normpath(runtime_dep)
+  in_build_dir = not norm.startswith('../')
+  no_prefix = norm.lstrip('/.')
+  return ('/bin/' if in_build_dir else '/assets/') + no_prefix
+
+
+def _RunOnFuchsiaTarget(binary_dir, test, device_name):
+  """Runs the given Fuchsia |test| executable on the given |device_name|. The
+  device must already be booted.
+
+  Copies the executable and its runtime dependencies as specified by GN to the
+  target in /tmp using `netcp`, runs the binary on the target, and logs output
+  back to stdout on this machine via `loglistener`.
+  """
+  sdk_root = _GetFuchsiaSDKRoot()
+
+  # Run loglistener and filter the output to know when the test is done.
+  loglistener_process = subprocess.Popen(
+      [os.path.join(sdk_root, 'tools', 'loglistener'), device_name],
+      stdout=subprocess.PIPE, stdin=open(os.devnull), stderr=open(os.devnull))
+
+  runtime_deps_file = os.path.join(binary_dir, test + '.runtime_deps')
+  with open(runtime_deps_file, 'rb') as f:
+    runtime_deps = f.read().splitlines()
+
+  def netruncmd(*args):
+    """Runs a list of commands on the target device. Each command is escaped
+    by using pipes.quote(), and then each command is chained by shell ';'.
+    """
+    local_binary = os.path.join(sdk_root, 'tools', 'netruncmd')
+    final_args = ' ; '.join(' '.join(pipes.quote(x) for x in command)
+                            for command in args)
+    subprocess.check_call([local_binary, device_name, final_args])
+
+  try:
+    unique_id = uuid.uuid4().hex
+    tmp_root = '/tmp/%s_%s/tmp' % (test, unique_id)
+    staging_root = '/tmp/%s_%s/pkg' % (test, unique_id)
+
+    # Make a staging directory tree on the target.
+    directories_to_create = [tmp_root, '%s/bin' % staging_root,
+                             '%s/assets' % staging_root]
+    netruncmd(['mkdir', '-p'] + directories_to_create)
+
+    # Copy runtime deps into the staging tree.
+    netcp = os.path.join(sdk_root, 'tools', 'netcp')
+    for dep in runtime_deps:
+      target_path = staging_root + _RuntimeDepsPathToFuchsiaTargetPath(dep)
+      subprocess.check_call([netcp, os.path.join(binary_dir, dep),
+                            device_name + ':' + target_path],
+                            stderr=open(os.devnull))
+
+    done_message = 'TERMINATED: ' + unique_id
+    namespace_command = [
+        'namespace', '/pkg=' + staging_root, '/tmp=' + tmp_root, '--',
+        staging_root + '/bin/' + test]
+    netruncmd(namespace_command, ['echo', done_message])
+
+    success = _HandleOutputFromFuchsiaLogListener(
+        loglistener_process, done_message)
+    if not success:
+      raise subprocess.CalledProcessError(1, test)
+  finally:
+    netruncmd(['rm', '-rf', tmp_root, staging_root])
 
 
 # This script is primarily used from the waterfall so that the list of tests
@@ -30,8 +151,6 @@ def main(args):
     print('usage: run_tests.py <binary_dir>', file=sys.stderr)
     return 1
 
-  crashpad_dir = \
-      os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
   binary_dir = args[0]
 
   # Tell 64-bit Windows tests where to find 32-bit test executables, for
@@ -46,20 +165,45 @@ def main(args):
     if os.path.isdir(binary_dir_32):
       os.environ['CRASHPAD_TEST_32_BIT_OUTPUT'] = binary_dir_32
 
+  is_fuchsia = _BinaryDirLooksLikeFuchsiaBuild(binary_dir)
+
   tests = [
+      'crashpad_minidump_test',
+      'crashpad_test_test',
+  ]
+
+  if not is_fuchsia:
+    tests.extend([
+      # TODO(scottmg): Move the rest of these to the common section once they
+      # are building and running successfully.
       'crashpad_client_test',
       'crashpad_handler_test',
-      'crashpad_minidump_test',
       'crashpad_snapshot_test',
-      'crashpad_test_test',
       'crashpad_util_test',
-  ]
+      ])
+
+  if is_fuchsia:
+    zircon_nodename = os.environ.get('ZIRCON_NODENAME')
+    if not zircon_nodename:
+      netls = os.path.join(_GetFuchsiaSDKRoot(), 'tools', 'netls')
+      popen = subprocess.Popen([netls, '--nowait'], stdout=subprocess.PIPE)
+      devices = popen.communicate()[0].splitlines()
+      if popen.returncode != 0 or len(devices) != 1:
+        print("Please set ZIRCON_NODENAME to your device's hostname",
+              file=sys.stderr)
+        return 2
+      zircon_nodename = devices[0].strip().split()[1]
+      print('Using autodetected Fuchsia device:', zircon_nodename)
+    _GenerateFuchsiaRuntimeDepsFiles(binary_dir, tests)
 
   for test in tests:
     print('-' * 80)
     print(test)
     print('-' * 80)
-    subprocess.check_call(os.path.join(binary_dir, test))
+    if is_fuchsia:
+      _RunOnFuchsiaTarget(binary_dir, test, zircon_nodename)
+    else:
+      subprocess.check_call(os.path.join(binary_dir, test))
 
   if sys.platform == 'win32':
     script = 'snapshot/win/end_to_end_test.py'
@@ -67,7 +211,7 @@ def main(args):
     print(script)
     print('-' * 80)
     subprocess.check_call(
-        [sys.executable, os.path.join(crashpad_dir, script), binary_dir])
+        [sys.executable, os.path.join(CRASHPAD_DIR, script), binary_dir])
 
   return 0
 

test/BUILD.gn

@@ -108,11 +108,14 @@ source_set("test_test") {
   sources = [
     "hex_string_test.cc",
     "main_arguments_test.cc",
-    "multiprocess_exec_test.cc",
     "scoped_temp_dir_test.cc",
     "test_paths_test.cc",
   ]
 
+  if (is_posix && !is_fuchsia) {
+    sources += [ "multiprocess_posix_test.cc" ]
+  }
+
   if (is_mac) {
     sources += [ "mac/mach_multiprocess_test.cc" ]
   }
@@ -124,8 +127,12 @@ source_set("test_test") {
     ]
   }
 
-  if (is_posix) {
-    sources += [ "multiprocess_posix_test.cc" ]
+  if (!is_fuchsia) {
+    sources += [
+      # TODO(scottmg): A MultiprocessExecFuchsia is probably desirable, but
+      # hasn't been implemented yet.
+      "multiprocess_exec_test.cc",
+    ]
   }
 
   deps = [

test/test_paths.cc

@@ -43,6 +43,13 @@ bool IsTestDataRoot(const base::FilePath& candidate) {
 }
 
 base::FilePath TestDataRootInternal() {
+#if defined(OS_FUCHSIA)
+  base::FilePath asset_path("/pkg/assets");
+  if (!IsTestDataRoot(asset_path)) {
+    LOG(WARNING) << "Test data root seems invalid, continuing anyway";
+  }
+  return asset_path;
+#else  // defined(OS_FUCHSIA)
 #if !defined(OS_WIN)
   const char* environment_value = getenv("CRASHPAD_TEST_DATA_ROOT");
 #else  // defined(OS_WIN)
@@ -88,6 +95,7 @@ base::FilePath TestDataRootInternal() {
   }
 
   return base::FilePath(base::FilePath::kCurrentDirectory);
+#endif  // defined(OS_FUCHSIA)
 }
 
 #if defined(OS_WIN) && defined(ARCH_CPU_64_BITS)

util/BUILD.gn

@@ -309,6 +309,10 @@ static_library("util") {
     }
   }
 
+  if (is_fuchsia) {
+    sources += [ "misc/paths_fuchsia.cc" ]
+  }
+
   public_configs = [ "..:crashpad_config" ]
 
   # Include generated files starting with "util".

util/file/file_io_posix.cc

@@ -109,8 +109,12 @@ FileOperationResult ReadFile(FileHandle file, void* buffer, size_t size) {
 }
 
 FileHandle OpenFileForRead(const base::FilePath& path) {
-  return HANDLE_EINTR(
-      open(path.value().c_str(), O_RDONLY | O_NOCTTY | O_CLOEXEC));
+  int flags = O_RDONLY;
+#if !defined(OS_FUCHSIA)
+  // O_NOCTTY is invalid on Fuchsia, and O_CLOEXEC isn't necessary.
+  flags |= O_NOCTTY | O_CLOEXEC;
+#endif
+  return HANDLE_EINTR(open(path.value().c_str(), flags));
 }
 
 FileHandle OpenFileForWrite(const base::FilePath& path,

util/misc/paths_fuchsia.cc

@@ -0,0 +1,35 @@
+// Copyright 2017 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util/misc/paths.h"
+
+#include <sys/stat.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+#include "base/logging.h"
+
+namespace crashpad {
+
+// static
+bool Paths::Executable(base::FilePath* path) {
+  // Assume the environment has been set up following
+  // https://fuchsia.googlesource.com/docs/+/master/namespaces.md#typical-directory-structure
+  // . The actual executable name is not known, but it's conceptually in this
+  // location.
+  *path = base::FilePath("/pkg/bin/unknown");
+  return true;
+}
+
+}  // namespace crashpad