Copies the latest setup_ios_gn.py from Chromium.

Also includes the latest convert_gn_xcodeproj.py.

BUG=None

Change-Id: Icd82dd248ebf30b98054336d82f73a70d63f1d71
Reviewed-on: https://chromium-review.googlesource.com/c/crashpad/crashpad/+/2276602
Reviewed-by: Mark Mentovai <mark@chromium.org>
Commit-Queue: Rohit Rao <rohitrao@chromium.org>

build/ios/convert_gn_xcodeproj.py

@@ -29,8 +29,6 @@ import filecmp
 import json
 import hashlib
 import os
-import plistlib
-import random
 import shutil
 import subprocess
 import sys
@@ -47,7 +45,8 @@ class XcodeProject(object):
         while True:
             self.counter += 1
             str_id = "%s %s %d" % (parent_name, obj['isa'], self.counter)
-            new_id = hashlib.sha1(str_id).hexdigest()[:24].upper()
+            new_id = hashlib.sha1(
+                str_id.encode("utf-8")).hexdigest()[:24].upper()
 
             # Make sure ID is unique. It's possible there could be an id
             # conflict since this is run after GN runs.
@@ -56,46 +55,65 @@ class XcodeProject(object):
                 return new_id
 
 
+def check_output(command):
+    """Wrapper around subprocess.check_output that decode output as utf-8."""
+    return subprocess.check_output(command).decode('utf-8')
+
+
 def CopyFileIfChanged(source_path, target_path):
-    """Copy |source_path| to |target_path| is different."""
+    """Copy |source_path| to |target_path| if different."""
     target_dir = os.path.dirname(target_path)
     if not os.path.isdir(target_dir):
         os.makedirs(target_dir)
-    if (not os.path.exists(target_path) or
+    if not os.path.exists(target_path) or \
-            not filecmp.cmp(source_path, target_path)):
+        not filecmp.cmp(source_path, target_path):
         shutil.copyfile(source_path, target_path)
 
 
-def LoadXcodeProjectAsJSON(path):
+def CopyTreeIfChanged(source, target):
+    """Copy |source| to |target| recursively; files are copied iff changed."""
+    if os.path.isfile(source):
+        return CopyFileIfChanged(source, target)
+    if not os.path.isdir(target):
+        os.makedirs(target)
+    for name in os.listdir(source):
+        CopyTreeIfChanged(os.path.join(source, name),
+                          os.path.join(target, name))
+
+
+def LoadXcodeProjectAsJSON(project_dir):
     """Return Xcode project at |path| as a JSON string."""
-    return subprocess.check_output(
+    return check_output([
-        ['plutil', '-convert', 'json', '-o', '-', path])
+        'plutil', '-convert', 'json', '-o', '-',
+        os.path.join(project_dir, 'project.pbxproj')
+    ])
 
 
 def WriteXcodeProject(output_path, json_string):
     """Save Xcode project to |output_path| as XML."""
     with tempfile.NamedTemporaryFile() as temp_file:
-        temp_file.write(json_string)
+        temp_file.write(json_string.encode("utf-8"))
         temp_file.flush()
         subprocess.check_call(['plutil', '-convert', 'xml1', temp_file.name])
-        CopyFileIfChanged(temp_file.name, output_path)
+        CopyFileIfChanged(temp_file.name,
+                          os.path.join(output_path, 'project.pbxproj'))
 
 
-def UpdateProductsProject(file_input, file_output, configurations, root_dir):
+def UpdateXcodeProject(project_dir, configurations, root_dir):
-    """Update Xcode project to support multiple configurations.
+    """Update inplace Xcode project to support multiple configurations.
 
     Args:
-        file_input: path to the input Xcode project
+      project_dir: path to the input Xcode project
-        file_output: path to the output file
+      configurations: list of string corresponding to the configurations that
-        configurations: list of string corresponding to the configurations that
         need to be supported by the tweaked Xcode projects, must contains at
         least one value.
+      root_dir: path to the root directory used to find markdown files
     """
-    json_data = json.loads(LoadXcodeProjectAsJSON(file_input))
+    json_data = json.loads(LoadXcodeProjectAsJSON(project_dir))
     project = XcodeProject(json_data['objects'])
 
     objects_to_remove = []
-    for value in project.objects.values():
+    for value in list(project.objects.values()):
         isa = value['isa']
 
         # Teach build shell script to look for the configuration and platform.
@@ -111,10 +129,8 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir):
 
             build_config_template = project.objects[value['buildConfigurations']
                                                     [0]]
-            build_settings = build_config_template['buildSettings']
+            build_config_template['buildSettings']['CONFIGURATION_BUILD_DIR'] =\
-            build_settings['CONFIGURATION_BUILD_DIR'] = (
+                '$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)'
-                '$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)')
-            build_settings['CODE_SIGN_IDENTITY'] = ''
 
             value['buildConfigurations'] = []
             for configuration in configurations:
@@ -126,20 +142,96 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir):
     for object_id in objects_to_remove:
         del project.objects[object_id]
 
-    AddMarkdownToProject(project, root_dir, json_data['rootObject'])
+    source = GetOrCreateRootGroup(project, json_data['rootObject'], 'Source')
+    AddMarkdownToProject(project, root_dir, source)
+    SortFileReferencesByName(project, source)
 
-    objects = collections.OrderedDict(sorted(project.objects.iteritems()))
+    objects = collections.OrderedDict(sorted(project.objects.items()))
-    WriteXcodeProject(file_output, json.dumps(json_data))
+    WriteXcodeProject(project_dir, json.dumps(json_data))
 
 
-def AddMarkdownToProject(project, root_dir, root_object):
+def CreateGroup(project, parent_group, group_name, path=None):
+    group_object = {
+        'children': [],
+        'isa': 'PBXGroup',
+        'name': group_name,
+        'sourceTree': '<group>',
+    }
+    if path is not None:
+        group_object['path'] = path
+    parent_group_name = parent_group.get('name', '')
+    group_object_key = project.AddObject(parent_group_name, group_object)
+    parent_group['children'].append(group_object_key)
+    return group_object
+
+
+def GetOrCreateRootGroup(project, root_object, group_name):
+    main_group = project.objects[project.objects[root_object]['mainGroup']]
+    for child_key in main_group['children']:
+        child = project.objects[child_key]
+        if child['name'] == group_name:
+            return child
+    return CreateGroup(project, main_group, group_name, path='../..')
+
+
+class ObjectKey(object):
+    """Wrapper around PBXFileReference and PBXGroup for sorting.
+
+    A PBXGroup represents a "directory" containing a list of files in an
+    Xcode project; it can contain references to a list of directories or
+    files.
+
+    A PBXFileReference represents a "file".
+
+    The type is stored in the object "isa" property as a string. Since we
+    want to sort all directories before all files, the < and > operators
+    are defined so that if "isa" is different, they are sorted in the
+    reverse of alphabetic ordering, otherwise the name (or path) property
+    is checked and compared in alphabetic order.
+    """
+
+    def __init__(self, obj):
+        self.isa = obj['isa']
+        if 'name' in obj:
+            self.name = obj['name']
+        else:
+            self.name = obj['path']
+
+    def __lt__(self, other):
+        if self.isa != other.isa:
+            return self.isa > other.isa
+        return self.name < other.name
+
+    def __gt__(self, other):
+        if self.isa != other.isa:
+            return self.isa < other.isa
+        return self.name > other.name
+
+    def __eq__(self, other):
+        return self.isa == other.isa and self.name == other.name
+
+
+def SortFileReferencesByName(project, group_object):
+    SortFileReferencesByNameWithSortKey(
+        project, group_object, lambda ref: ObjectKey(project.objects[ref]))
+
+
+def SortFileReferencesByNameWithSortKey(project, group_object, sort_key):
+    group_object['children'].sort(key=sort_key)
+    for key in group_object['children']:
+        child = project.objects[key]
+        if child['isa'] == 'PBXGroup':
+            SortFileReferencesByNameWithSortKey(project, child, sort_key)
+
+
+def AddMarkdownToProject(project, root_dir, group_object):
     list_files_cmd = ['git', '-C', root_dir, 'ls-files', '*.md']
-    paths = subprocess.check_output(list_files_cmd).splitlines()
+    paths = check_output(list_files_cmd).splitlines()
     ios_internal_dir = os.path.join(root_dir, 'ios_internal')
     if os.path.exists(ios_internal_dir):
         list_files_cmd = ['git', '-C', ios_internal_dir, 'ls-files', '*.md']
-        ios_paths = subprocess.check_output(list_files_cmd).splitlines()
+        ios_paths = check_output(list_files_cmd).splitlines()
-        paths.extend(["ios_internal/" + path for path in ios_paths])
+        paths.extend([os.path.join("ios_internal", path) for path in ios_paths])
     for path in paths:
         new_markdown_entry = {
             "fileEncoding": "4",
@@ -150,96 +242,71 @@ def AddMarkdownToProject(project, root_dir, root_object):
             "sourceTree": "<group>"
         }
         new_markdown_entry_id = project.AddObject('sources', new_markdown_entry)
-        folder = GetFolderForPath(project, root_object, os.path.dirname(path))
+        folder = GetFolderForPath(project, group_object, os.path.dirname(path))
         folder['children'].append(new_markdown_entry_id)
 
 
-def GetFolderForPath(project, rootObject, path):
+def GetFolderForPath(project, group_object, path):
     objects = project.objects
-    # 'Sources' is always the first child of
-    # project->rootObject->mainGroup->children.
-    root = objects[objects[objects[rootObject]['mainGroup']]['children'][0]]
     if not path:
-        return root
+        return group_object
     for folder in path.split('/'):
-        children = root['children']
+        children = group_object['children']
         new_root = None
         for child in children:
-            if (objects[child]['isa'] == 'PBXGroup' and
+            if objects[child]['isa'] == 'PBXGroup' and \
-                    objects[child]['name'] == folder):
+               objects[child]['name'] == folder:
                 new_root = objects[child]
                 break
         if not new_root:
             # If the folder isn't found we could just cram it into the leaf
             # existing folder, but that leads to folders with tons of README.md
             # inside.
-            new_group = {
+            new_root = CreateGroup(project, group_object, folder)
-                "children": [],
+        group_object = new_root
-                "isa": "PBXGroup",
+    return group_object
-                "name": folder,
-                "sourceTree": "<group>"
-            }
-            new_group_id = project.AddObject('sources', new_group)
-            children.append(new_group_id)
-            new_root = objects[new_group_id]
-        root = new_root
-    return root
-
-
-def DisableNewBuildSystem(output_dir):
-    """Disables the new build system due to crbug.com/852522 """
-    xcwspacesharedsettings = os.path.join(output_dir, 'all.xcworkspace',
-                                          'xcshareddata',
-                                          'WorkspaceSettings.xcsettings')
-    if os.path.isfile(xcwspacesharedsettings):
-        json_data = json.loads(LoadXcodeProjectAsJSON(xcwspacesharedsettings))
-    else:
-        json_data = {}
-    json_data['BuildSystemType'] = 'Original'
-    WriteXcodeProject(xcwspacesharedsettings, json.dumps(json_data))
 
 
 def ConvertGnXcodeProject(root_dir, input_dir, output_dir, configurations):
     '''Tweak the Xcode project generated by gn to support multiple
     configurations.
 
-    The Xcode projects generated by "gn gen --ide" only supports a single
+  The Xcode projects generated by "gn gen --ide" only supports a single
-    platform and configuration (as the platform and configuration are set per
+  platform and configuration (as the platform and configuration are set
-    output directory). This method takes as input such projects and add support
+  per output directory). This method takes as input such projects and
-    for multiple configurations and platforms (to allow devs to select them in
+  add support for multiple configurations and platforms (to allow devs
-    Xcode).
+  to select them in Xcode).
 
-    Args:
+  Args:
-        input_dir: directory containing the XCode projects created by "gn gen
+    input_dir: directory containing the XCode projects created by "gn gen --ide"
-            --ide"
+    output_dir: directory where the tweaked Xcode projects will be saved
-        output_dir: directory where the tweaked Xcode projects will be saved
+    configurations: list of string corresponding to the configurations that
-        configurations: list of string corresponding to the configurations that
+      need to be supported by the tweaked Xcode projects, must contains at
-            need to be supported by the tweaked Xcode projects, must contains at
+      least one value.
-            least one value.
+  '''
-    '''
-    # Update products project.
-    products = os.path.join('products.xcodeproj', 'project.pbxproj')
-    product_input = os.path.join(input_dir, products)
-    product_output = os.path.join(output_dir, products)
-    UpdateProductsProject(product_input, product_output, configurations,
-                          root_dir)
 
-    # Copy all workspace.
+    # Update the project (supports legacy name "products.xcodeproj" or the new
-    xcwspace = os.path.join('all.xcworkspace', 'contents.xcworkspacedata')
+    # project name "all.xcodeproj").
-    CopyFileIfChanged(os.path.join(input_dir, xcwspace),
+    for project_name in ('all.xcodeproj', 'products.xcodeproj'):
-                      os.path.join(output_dir, xcwspace))
+        if os.path.exists(os.path.join(input_dir, project_name)):
+            UpdateXcodeProject(os.path.join(input_dir, project_name),
+                               configurations, root_dir)
 
-    # TODO(crbug.com/852522): Disable new BuildSystemType.
+            CopyTreeIfChanged(os.path.join(input_dir, project_name),
-    DisableNewBuildSystem(output_dir)
+                              os.path.join(output_dir, project_name))
 
-    # TODO(crbug.com/679110): gn has been modified to remove 'sources.xcodeproj'
+        else:
-    # and keep 'all.xcworkspace' and 'products.xcodeproj'. The following code is
+            shutil.rmtree(os.path.join(output_dir, project_name),
-    # here to support both old and new projects setup and will be removed once
+                          ignore_errors=True)
-    # gn has rolled past it.
+
-    sources = os.path.join('sources.xcodeproj', 'project.pbxproj')
+    # Copy all.xcworkspace if it exists (will be removed in a future gn version).
-    if os.path.isfile(os.path.join(input_dir, sources)):
+    workspace_name = 'all.xcworkspace'
-        CopyFileIfChanged(os.path.join(input_dir, sources),
+    if os.path.exists(os.path.join(input_dir, workspace_name)):
-                          os.path.join(output_dir, sources))
+        CopyTreeIfChanged(os.path.join(input_dir, workspace_name),
+                          os.path.join(output_dir, workspace_name))
+    else:
+        shutil.rmtree(os.path.join(output_dir, workspace_name),
+                      ignore_errors=True)
 
 
 def Main(args):
@@ -264,8 +331,18 @@ def Main(args):
         sys.stderr.write('Input directory does not exists.\n')
         return 1
 
-    required = set(['products.xcodeproj', 'all.xcworkspace'])
+    # Depending on the version of "gn", there should be either one project file
-    if not required.issubset(os.listdir(args.input)):
+    # named "all.xcodeproj" or a project file named "products.xcodeproj" and a
+    # workspace named "all.xcworkspace".
+    required_files_sets = [
+        set(("all.xcodeproj",)),
+        set(("products.xcodeproj", "all.xcworkspace")),
+    ]
+
+    for required_files in required_files_sets:
+        if required_files.issubset(os.listdir(args.input)):
+            break
+    else:
         sys.stderr.write(
             'Input directory does not contain all necessary Xcode projects.\n')
         return 1

build/ios/setup_ios_gn.py

@@ -23,25 +23,29 @@ import shutil
 import subprocess
 import sys
 import tempfile
-import ConfigParser
 
 try:
-    import cStringIO as StringIO
+    import configparser
 except ImportError:
-    import StringIO
+    import ConfigParser as configparser
+
+try:
+    import StringIO as io
+except ImportError:
+    import io
 
 SUPPORTED_TARGETS = ('iphoneos', 'iphonesimulator')
 SUPPORTED_CONFIGS = ('Debug', 'Release', 'Profile', 'Official', 'Coverage')
 
 
-class ConfigParserWithStringInterpolation(ConfigParser.SafeConfigParser):
+class ConfigParserWithStringInterpolation(configparser.SafeConfigParser):
     '''A .ini file parser that supports strings and environment variables.'''
 
     ENV_VAR_PATTERN = re.compile(r'\$([A-Za-z0-9_]+)')
 
     def values(self, section):
-        return map(lambda (k, v): self._UnquoteString(self._ExpandEnvVar(v)),
+        return map(lambda kv: self._UnquoteString(self._ExpandEnvVar(kv[1])),
-                   ConfigParser.SafeConfigParser.items(self, section))
+                   configparser.ConfigParser.items(self, section))
 
     def getstring(self, section, option):
         return self._UnquoteString(self._ExpandEnvVar(self.get(section,
@@ -94,7 +98,25 @@ class GnGenerator(object):
         """
         args = []
 
+        # build/config/ios/ios_sdk.gni asserts that goma is not enabled when
+        # building Official, so ignore the value of goma.enabled when creating
+        # args.gn for Official.
+        if self._config != 'Official':
+            if self._settings.getboolean('goma', 'enabled'):
+                args.append(('use_goma', True))
+                goma_dir = self._settings.getstring('goma', 'install')
+                if goma_dir:
+                    args.append(
+                        ('goma_dir', '"%s"' % os.path.expanduser(goma_dir)))
+
         args.append(('is_debug', self._config in ('Debug', 'Coverage')))
+        args.append(('enable_dsyms', self._config in ('Profile', 'Official')))
+        args.append(('enable_stripping', 'enable_dsyms'))
+        args.append(('is_official_build', self._config == 'Official'))
+        args.append(('is_chrome_branded', 'is_official_build'))
+        args.append(('use_xcode_clang', 'false'))
+        args.append(('use_clang_coverage', self._config == 'Coverage'))
+        args.append(('is_component_build', False))
 
         if os.environ.get('FORCE_MAC_TOOLCHAIN', '0') == '1':
             args.append(('use_system_xcode', False))
@@ -116,7 +138,7 @@ class GnGenerator(object):
         return args
 
     def Generate(self, gn_path, root_path, out_path):
-        buf = StringIO.StringIO()
+        buf = io.StringIO()
         self.WriteArgsGn(buf)
         WriteToFileIfChanged(os.path.join(out_path, 'args.gn'),
                              buf.getvalue(),
@@ -126,20 +148,20 @@ class GnGenerator(object):
             self.GetGnCommand(gn_path, root_path, out_path, True))
 
     def CreateGnRules(self, gn_path, root_path, out_path):
-        buf = StringIO.StringIO()
+        buf = io.StringIO()
         self.WriteArgsGn(buf)
         WriteToFileIfChanged(os.path.join(out_path, 'args.gn'),
                              buf.getvalue(),
                              overwrite=True)
 
-        buf = StringIO.StringIO()
+        buf = io.StringIO()
         gn_command = self.GetGnCommand(gn_path, root_path, out_path, False)
         self.WriteBuildNinja(buf, gn_command)
         WriteToFileIfChanged(os.path.join(out_path, 'build.ninja'),
                              buf.getvalue(),
                              overwrite=False)
 
-        buf = StringIO.StringIO()
+        buf = io.StringIO()
         self.WriteBuildNinjaDeps(buf)
         WriteToFileIfChanged(os.path.join(out_path, 'build.ninja.d'),
                              buf.getvalue(),
@@ -196,16 +218,13 @@ class GnGenerator(object):
         if generate_xcode_project:
             gn_command.append('--ide=xcode')
             gn_command.append('--root-target=gn_all')
-            if self._settings.getboolean('goma', 'enabled'):
+            gn_command.append('--ninja-executable=autoninja')
-                ninja_jobs = self._settings.getint('xcode', 'jobs') or 200
-                gn_command.append('--ninja-extra-args=-j%s' % ninja_jobs)
             if self._settings.has_section('filters'):
                 target_filters = self._settings.values('filters')
                 if target_filters:
                     gn_command.append('--filters=%s' % ';'.join(target_filters))
-        # TODO(justincohen): --check is currently failing in crashpad.
+        else:
-        # else:
+            gn_command.append('--check')
-        # gn_command.append('--check')
         gn_command.append('gen')
         gn_command.append('//%s' % os.path.relpath(os.path.abspath(out_path),
                                                    os.path.abspath(src_path)))
@@ -296,6 +315,13 @@ def Main(args):
                         dest='import_rules',
                         default=[],
                         help='path to file defining default gn variables')
+    parser.add_argument('--gn-path',
+                        default=None,
+                        help='path to gn binary (default: look up in $PATH)')
+    parser.add_argument(
+        '--build-dir',
+        default='out',
+        help='path where the build should be created (default: %(default)s)')
     args = parser.parse_args(args)
 
     # Load configuration (first global and then any user overrides).
@@ -320,25 +346,16 @@ def Main(args):
                          settings.getstring('build', 'arch'))
         sys.exit(1)
 
-    if settings.getboolean('goma', 'enabled'):
+    # Find path to gn binary either from command-line or in PATH.
-        if settings.getint('xcode', 'jobs') < 0:
+    if args.gn_path:
-            sys.stderr.write('ERROR: invalid value for xcode.jobs: %s\n' %
+        gn_path = args.gn_path
-                             settings.get('xcode', 'jobs'))
+    else:
+        gn_path = FindGn()
+        if gn_path is None:
+            sys.stderr.write('ERROR: cannot find gn in PATH\n')
             sys.exit(1)
-        goma_install = os.path.expanduser(settings.getstring('goma', 'install'))
-        if not os.path.isdir(goma_install):
-            sys.stderr.write('WARNING: goma.install directory not found: %s\n' %
-                             settings.get('goma', 'install'))
-            sys.stderr.write('WARNING: disabling goma\n')
-            settings.set('goma', 'enabled', 'false')
 
-    # Find gn binary in PATH.
+    out_dir = os.path.join(args.root, args.build_dir)
-    gn_path = FindGn()
-    if gn_path is None:
-        sys.stderr.write('ERROR: cannot find gn in PATH\n')
-        sys.exit(1)
-
-    out_dir = os.path.join(args.root, 'out')
     if not os.path.isdir(out_dir):
         os.makedirs(out_dir)