Python 3 support for Python scripts (without compromising Python 2)

It’s better to be prepared for the future than…to not be.

This is mostly the result of running 2to3 on all .py files, with some
small shims to maintain compatibility with Python 2.
http_transport_test_server.py was slightly more involved, requiring many
objects to change from “str” to “bytes”.

The #! lines and invokers still haven’t changed, so these scripts will
still normally be interpreted by Python 2.

Change-Id: Idda3c5650f967401a5942c4d8abee86151642a2e
Reviewed-on: https://chromium-review.googlesource.com/797434
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Commit-Queue: Mark Mentovai <mark@chromium.org>

build/gyp_crashpad.py

@@ -17,6 +17,9 @@
 import os
 import sys
 
+if sys.version_info[0] < 3:
+  range = xrange
+
 
 def ChooseDependencyPath(local_path, external_path):
   """Chooses between a dependency located at local path and an external path.
@@ -78,7 +81,7 @@ def main(args):
     # Check to make sure that no target_arch was specified. target_arch may be
     # set during a cross build, such as a cross build for Android.
     has_target_arch = False
-    for arg_index in xrange(0, len(args)):
+    for arg_index in range(0, len(args)):
       arg = args[arg_index]
       if (arg.startswith('-Dtarget_arch=') or
           (arg == '-D' and arg_index + 1 < len(args) and

build/run_tests.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import os
 import subprocess
 import sys
@@ -25,7 +27,7 @@ import sys
 # location in the recipe.
 def main(args):
   if len(args) != 1:
-    print >> sys.stderr, 'usage: run_tests.py <binary_dir>'
+    print('usage: run_tests.py <binary_dir>', file=sys.stderr)
     return 1
 
   crashpad_dir = \
@@ -54,16 +56,16 @@ def main(args):
   ]
 
   for test in tests:
-    print '-' * 80
-    print test
-    print '-' * 80
+    print('-' * 80)
+    print(test)
+    print('-' * 80)
     subprocess.check_call(os.path.join(binary_dir, test))
 
   if sys.platform == 'win32':
     script = 'snapshot/win/end_to_end_test.py'
-    print '-' * 80
-    print script
-    print '-' * 80
+    print('-' * 80)
+    print(script)
+    print('-' * 80)
     subprocess.check_call(
         [sys.executable, os.path.join(crashpad_dir, script), binary_dir])
 

doc/support/generate_doxygen.py

@@ -37,7 +37,7 @@ def main(args):
   elif os.path.exists(output_dir):
     os.unlink(output_dir)
 
-  os.makedirs(output_dir, 0755)
+  os.makedirs(output_dir, 0o755)
 
   doxy_file = os.path.join('doc', 'support', 'crashpad.doxy')
   subprocess.check_call(['doxygen', doxy_file])

snapshot/win/end_to_end_test.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import os
 import platform
 import pywintypes
@@ -99,7 +101,7 @@ def NamedPipeExistsAndReady(pipe_name):
   """
   try:
     win32pipe.WaitNamedPipe(pipe_name, win32pipe.NMPWAIT_WAIT_FOREVER)
-  except pywintypes.error, e:
+  except pywintypes.error as e:
     if e[0] == winerror.ERROR_FILE_NOT_FOUND:
       return False
     raise
@@ -135,7 +137,7 @@ def GetDumpFromProgram(
       printed = False
       while not NamedPipeExistsAndReady(pipe_name):
         if not printed:
-          print 'Waiting for crashpad_handler to be ready...'
+          print('Waiting for crashpad_handler to be ready...')
           printed = True
         time.sleep(0.001)
 
@@ -145,7 +147,7 @@ def GetDumpFromProgram(
                   os.path.join(out_dir, 'crashpad_handler.com'),
                   test_database] +
                  list(args))
-    print 'Running %s' % os.path.basename(command[0])
+    print('Running %s' % os.path.basename(command[0]))
     exit_code = subprocess.call(command)
     if exit_code != expect_exit_code:
       raise subprocess.CalledProcessError(exit_code, executable_name)
@@ -219,16 +221,16 @@ class CdbRun(object):
     if match_obj:
       # Matched. Consume up to end of match.
       self.out = self.out[match_obj.end(0):]
-      print 'ok - %s' % message
+      print('ok - %s' % message)
       sys.stdout.flush()
     else:
-      print >>sys.stderr, '-' * 80
-      print >>sys.stderr, 'FAILED - %s' % message
-      print >>sys.stderr, '-' * 80
-      print >>sys.stderr, 'did not match:\n  %s' % pattern
-      print >>sys.stderr, '-' * 80
-      print >>sys.stderr, 'remaining output was:\n  %s' % self.out
-      print >>sys.stderr, '-' * 80
+      print('-' * 80, file=sys.stderr)
+      print('FAILED - %s' % message, file=sys.stderr)
+      print('-' * 80, file=sys.stderr)
+      print('did not match:\n  %s' % pattern, file=sys.stderr)
+      print('-' * 80, file=sys.stderr)
+      print('remaining output was:\n  %s' % self.out, file=sys.stderr)
+      print('-' * 80, file=sys.stderr)
       sys.stderr.flush()
       global g_had_failures
       g_had_failures = True
@@ -430,12 +432,12 @@ def RunTests(cdb_path,
 def main(args):
   try:
     if len(args) != 1:
-      print >>sys.stderr, 'must supply binary dir'
+      print('must supply binary dir', file=sys.stderr)
       return 1
 
     cdb_path = GetCdbPath()
     if not cdb_path:
-      print >>sys.stderr, 'could not find cdb'
+      print('could not find cdb', file=sys.stderr)
       return 1
 
     # Make sure we can download Windows symbols.

util/net/http_transport_test_server.py

@@ -30,17 +30,25 @@ because parsing chunked encoding is safer and easier in a memory-safe language.
 This could easily have been written in C++ instead.
 """
 
-import BaseHTTPServer
+import os
 import struct
 import sys
 import zlib
 
+if sys.platform == 'win32':
+  import msvcrt
+
+if sys.version_info[0] < 3:
+  import BaseHTTPServer as http_server
+else:
+  import http.server as http_server
+
 class BufferedReadFile(object):
   """A File-like object that stores all read contents into a buffer."""
 
   def __init__(self, real_file):
     self.file = real_file
-    self.buffer = ""
+    self.buffer = b''
 
   def read(self, size=-1):
     buf = self.file.read(size)
@@ -59,27 +67,27 @@ class BufferedReadFile(object):
     self.file.close()
 
 
-class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class RequestHandler(http_server.BaseHTTPRequestHandler):
   # Everything to be written to stdout is collected into this string. It can’t
   # be written to stdout until after the HTTP transaction is complete, because
   # stdout is a pipe being read by a test program that’s also the HTTP client.
   # The test program expects to complete the entire HTTP transaction before it
   # even starts reading this script’s stdout. If the stdout pipe buffer fills up
   # during an HTTP transaction, deadlock would result.
-  raw_request = ''
+  raw_request = b''
 
   response_code = 500
-  response_body = ''
+  response_body = b''
 
   def handle_one_request(self):
     # Wrap the rfile in the buffering file object so that the raw header block
     # can be written to stdout after it is parsed.
     self.rfile = BufferedReadFile(self.rfile)
-    BaseHTTPServer.BaseHTTPRequestHandler.handle_one_request(self)
+    http_server.BaseHTTPRequestHandler.handle_one_request(self)
 
   def do_POST(self):
     RequestHandler.raw_request = self.rfile.buffer
-    self.rfile.buffer = ''
+    self.rfile.buffer = b''
 
     if self.headers.get('Transfer-Encoding', '').lower() == 'chunked':
       if 'Content-Length' in self.headers:
@@ -102,13 +110,13 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     self.end_headers()
     if self.response_code == 200:
       self.wfile.write(self.response_body)
-      self.wfile.write('\r\n')
+      self.wfile.write(b'\r\n')
 
   def handle_chunked_encoding(self):
-    """This parses a "Transfer-Encoding: Chunked" body in accordance with
-    RFC 7230 §4.1. This returns the result as a string.
+    """This parses a "Transfer-Encoding: Chunked" body in accordance with RFC
+    7230 §4.1. This returns the result as a string.
     """
-    body = ''
+    body = b''
     chunk_size = self.read_chunk_size()
     while chunk_size > 0:
       # Read the body.
@@ -120,7 +128,7 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       if chunk_size == 0:
         # Read through any trailer fields.
         trailer_line = self.rfile.readline()
-        while trailer_line.strip() != '':
+        while trailer_line.strip() != b'':
           trailer_line = self.rfile.readline()
 
         # Read the chunk size.
@@ -131,10 +139,10 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     # Read the whole line, including the \r\n.
     chunk_size_and_ext_line = self.rfile.readline()
     # Look for a chunk extension.
-    chunk_size_end = chunk_size_and_ext_line.find(';')
+    chunk_size_end = chunk_size_and_ext_line.find(b';')
     if chunk_size_end == -1:
       # No chunk extensions; just encounter the end of line.
-      chunk_size_end = chunk_size_and_ext_line.find('\r')
+      chunk_size_end = chunk_size_and_ext_line.find(b'\r')
     if chunk_size_end == -1:
       self.send_response(400)  # Bad request.
       return -1
@@ -145,29 +153,49 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     pass
 
 
+def StdioBinaryEquivalent(file):
+  """Return a file object equivalent to sys.stdin or sys.stdout capable of
+  reading or writing binary “bytes”.
+
+  struct.unpack consumes “bytes”, and struct.pack produces “bytes”. These are
+  distinct from “str” in Python 3 (but not 2). In order to read and write these
+  from stdin and stdout, the underlying binary buffer must be used in place of
+  the upper-layer text wrapper. This function returns a suitable file.
+
+  There is no underlying buffer in Python 2, but on Windows, the file mode must
+  still be set to binary in order to cleanly pass binary data. Note that in this
+  case, the mode of |file| itself is changed, as it’s not distinct from the
+  returned file.
+  """
+  if hasattr(file, 'buffer'):
+    file = file.buffer
+  elif sys.platform == 'win32':
+    msvcrt.setmode(file.fileno(), os.O_BINARY)
+  return file
+
+
 def Main():
-  if sys.platform == 'win32':
-    import os, msvcrt
-    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+  in_file = StdioBinaryEquivalent(sys.stdin)
+  out_file = StdioBinaryEquivalent(sys.stdout)
 
   # Start the server.
-  server = BaseHTTPServer.HTTPServer(('127.0.0.1', 0), RequestHandler)
+  server = http_server.HTTPServer(('127.0.0.1', 0), RequestHandler)
 
   # Write the port as an unsigned short to the parent process.
-  sys.stdout.write(struct.pack('=H', server.server_address[1]))
-  sys.stdout.flush()
+  out_file.write(struct.pack('=H', server.server_address[1]))
+  out_file.flush()
 
   # Read the desired test response code as an unsigned short and the desired
   # response body as a 16-byte string from the parent process.
   RequestHandler.response_code, RequestHandler.response_body = \
-      struct.unpack('=H16s', sys.stdin.read(struct.calcsize('=H16s')))
+      struct.unpack('=H16s', in_file.read(struct.calcsize('=H16s')))
 
   # Handle the request.
   server.handle_request()
 
   # Share the entire request with the test program, which will validate it.
-  sys.stdout.write(RequestHandler.raw_request)
-  sys.stdout.flush()
+  out_file.write(RequestHandler.raw_request)
+  out_file.flush()
 
 if __name__ == '__main__':
   Main()