From 17b08b5aab64a1bac9d78e3a8043c213ae2656c4 Mon Sep 17 00:00:00 2001 From: Rohit Rao Date: Mon, 6 Jul 2020 21:18:05 -0400 Subject: [PATCH] Copies the latest setup_ios_gn.py from Chromium. Also includes the latest convert_gn_xcodeproj.py. BUG=None Change-Id: Icd82dd248ebf30b98054336d82f73a70d63f1d71 Reviewed-on: https://chromium-review.googlesource.com/c/crashpad/crashpad/+/2276602 Reviewed-by: Mark Mentovai Commit-Queue: Rohit Rao --- build/ios/convert_gn_xcodeproj.py | 271 +++++++++++++++++++----------- build/ios/setup_ios_gn.py | 83 +++++---- 2 files changed, 224 insertions(+), 130 deletions(-) diff --git a/build/ios/convert_gn_xcodeproj.py b/build/ios/convert_gn_xcodeproj.py index c00d3318..3bbbb4ed 100755 --- a/build/ios/convert_gn_xcodeproj.py +++ b/build/ios/convert_gn_xcodeproj.py @@ -29,8 +29,6 @@ import filecmp import json import hashlib import os -import plistlib -import random import shutil import subprocess import sys @@ -47,7 +45,8 @@ class XcodeProject(object): while True: self.counter += 1 str_id = "%s %s %d" % (parent_name, obj['isa'], self.counter) - new_id = hashlib.sha1(str_id).hexdigest()[:24].upper() + new_id = hashlib.sha1( + str_id.encode("utf-8")).hexdigest()[:24].upper() # Make sure ID is unique. It's possible there could be an id # conflict since this is run after GN runs. @@ -56,46 +55,65 @@ class XcodeProject(object): return new_id +def check_output(command): + """Wrapper around subprocess.check_output that decode output as utf-8.""" + return subprocess.check_output(command).decode('utf-8') + + def CopyFileIfChanged(source_path, target_path): - """Copy |source_path| to |target_path| is different.""" + """Copy |source_path| to |target_path| if different.""" target_dir = os.path.dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) - if (not os.path.exists(target_path) or - not filecmp.cmp(source_path, target_path)): + if not os.path.exists(target_path) or \ + not filecmp.cmp(source_path, target_path): shutil.copyfile(source_path, target_path) -def LoadXcodeProjectAsJSON(path): +def CopyTreeIfChanged(source, target): + """Copy |source| to |target| recursively; files are copied iff changed.""" + if os.path.isfile(source): + return CopyFileIfChanged(source, target) + if not os.path.isdir(target): + os.makedirs(target) + for name in os.listdir(source): + CopyTreeIfChanged(os.path.join(source, name), + os.path.join(target, name)) + + +def LoadXcodeProjectAsJSON(project_dir): """Return Xcode project at |path| as a JSON string.""" - return subprocess.check_output( - ['plutil', '-convert', 'json', '-o', '-', path]) + return check_output([ + 'plutil', '-convert', 'json', '-o', '-', + os.path.join(project_dir, 'project.pbxproj') + ]) def WriteXcodeProject(output_path, json_string): """Save Xcode project to |output_path| as XML.""" with tempfile.NamedTemporaryFile() as temp_file: - temp_file.write(json_string) + temp_file.write(json_string.encode("utf-8")) temp_file.flush() subprocess.check_call(['plutil', '-convert', 'xml1', temp_file.name]) - CopyFileIfChanged(temp_file.name, output_path) + CopyFileIfChanged(temp_file.name, + os.path.join(output_path, 'project.pbxproj')) -def UpdateProductsProject(file_input, file_output, configurations, root_dir): - """Update Xcode project to support multiple configurations. +def UpdateXcodeProject(project_dir, configurations, root_dir): + """Update inplace Xcode project to support multiple configurations. Args: - file_input: path to the input Xcode project - file_output: path to the output file - configurations: list of string corresponding to the configurations that + project_dir: path to the input Xcode project + configurations: list of string corresponding to the configurations that need to be supported by the tweaked Xcode projects, must contains at least one value. + root_dir: path to the root directory used to find markdown files """ - json_data = json.loads(LoadXcodeProjectAsJSON(file_input)) + json_data = json.loads(LoadXcodeProjectAsJSON(project_dir)) project = XcodeProject(json_data['objects']) objects_to_remove = [] - for value in project.objects.values(): + for value in list(project.objects.values()): isa = value['isa'] # Teach build shell script to look for the configuration and platform. @@ -111,10 +129,8 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir): build_config_template = project.objects[value['buildConfigurations'] [0]] - build_settings = build_config_template['buildSettings'] - build_settings['CONFIGURATION_BUILD_DIR'] = ( - '$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)') - build_settings['CODE_SIGN_IDENTITY'] = '' + build_config_template['buildSettings']['CONFIGURATION_BUILD_DIR'] =\ + '$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)' value['buildConfigurations'] = [] for configuration in configurations: @@ -126,20 +142,96 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir): for object_id in objects_to_remove: del project.objects[object_id] - AddMarkdownToProject(project, root_dir, json_data['rootObject']) + source = GetOrCreateRootGroup(project, json_data['rootObject'], 'Source') + AddMarkdownToProject(project, root_dir, source) + SortFileReferencesByName(project, source) - objects = collections.OrderedDict(sorted(project.objects.iteritems())) - WriteXcodeProject(file_output, json.dumps(json_data)) + objects = collections.OrderedDict(sorted(project.objects.items())) + WriteXcodeProject(project_dir, json.dumps(json_data)) -def AddMarkdownToProject(project, root_dir, root_object): +def CreateGroup(project, parent_group, group_name, path=None): + group_object = { + 'children': [], + 'isa': 'PBXGroup', + 'name': group_name, + 'sourceTree': '', + } + if path is not None: + group_object['path'] = path + parent_group_name = parent_group.get('name', '') + group_object_key = project.AddObject(parent_group_name, group_object) + parent_group['children'].append(group_object_key) + return group_object + + +def GetOrCreateRootGroup(project, root_object, group_name): + main_group = project.objects[project.objects[root_object]['mainGroup']] + for child_key in main_group['children']: + child = project.objects[child_key] + if child['name'] == group_name: + return child + return CreateGroup(project, main_group, group_name, path='../..') + + +class ObjectKey(object): + """Wrapper around PBXFileReference and PBXGroup for sorting. + + A PBXGroup represents a "directory" containing a list of files in an + Xcode project; it can contain references to a list of directories or + files. + + A PBXFileReference represents a "file". + + The type is stored in the object "isa" property as a string. Since we + want to sort all directories before all files, the < and > operators + are defined so that if "isa" is different, they are sorted in the + reverse of alphabetic ordering, otherwise the name (or path) property + is checked and compared in alphabetic order. + """ + + def __init__(self, obj): + self.isa = obj['isa'] + if 'name' in obj: + self.name = obj['name'] + else: + self.name = obj['path'] + + def __lt__(self, other): + if self.isa != other.isa: + return self.isa > other.isa + return self.name < other.name + + def __gt__(self, other): + if self.isa != other.isa: + return self.isa < other.isa + return self.name > other.name + + def __eq__(self, other): + return self.isa == other.isa and self.name == other.name + + +def SortFileReferencesByName(project, group_object): + SortFileReferencesByNameWithSortKey( + project, group_object, lambda ref: ObjectKey(project.objects[ref])) + + +def SortFileReferencesByNameWithSortKey(project, group_object, sort_key): + group_object['children'].sort(key=sort_key) + for key in group_object['children']: + child = project.objects[key] + if child['isa'] == 'PBXGroup': + SortFileReferencesByNameWithSortKey(project, child, sort_key) + + +def AddMarkdownToProject(project, root_dir, group_object): list_files_cmd = ['git', '-C', root_dir, 'ls-files', '*.md'] - paths = subprocess.check_output(list_files_cmd).splitlines() + paths = check_output(list_files_cmd).splitlines() ios_internal_dir = os.path.join(root_dir, 'ios_internal') if os.path.exists(ios_internal_dir): list_files_cmd = ['git', '-C', ios_internal_dir, 'ls-files', '*.md'] - ios_paths = subprocess.check_output(list_files_cmd).splitlines() - paths.extend(["ios_internal/" + path for path in ios_paths]) + ios_paths = check_output(list_files_cmd).splitlines() + paths.extend([os.path.join("ios_internal", path) for path in ios_paths]) for path in paths: new_markdown_entry = { "fileEncoding": "4", @@ -150,96 +242,71 @@ def AddMarkdownToProject(project, root_dir, root_object): "sourceTree": "" } new_markdown_entry_id = project.AddObject('sources', new_markdown_entry) - folder = GetFolderForPath(project, root_object, os.path.dirname(path)) + folder = GetFolderForPath(project, group_object, os.path.dirname(path)) folder['children'].append(new_markdown_entry_id) -def GetFolderForPath(project, rootObject, path): +def GetFolderForPath(project, group_object, path): objects = project.objects - # 'Sources' is always the first child of - # project->rootObject->mainGroup->children. - root = objects[objects[objects[rootObject]['mainGroup']]['children'][0]] if not path: - return root + return group_object for folder in path.split('/'): - children = root['children'] + children = group_object['children'] new_root = None for child in children: - if (objects[child]['isa'] == 'PBXGroup' and - objects[child]['name'] == folder): + if objects[child]['isa'] == 'PBXGroup' and \ + objects[child]['name'] == folder: new_root = objects[child] break if not new_root: # If the folder isn't found we could just cram it into the leaf # existing folder, but that leads to folders with tons of README.md # inside. - new_group = { - "children": [], - "isa": "PBXGroup", - "name": folder, - "sourceTree": "" - } - new_group_id = project.AddObject('sources', new_group) - children.append(new_group_id) - new_root = objects[new_group_id] - root = new_root - return root - - -def DisableNewBuildSystem(output_dir): - """Disables the new build system due to crbug.com/852522 """ - xcwspacesharedsettings = os.path.join(output_dir, 'all.xcworkspace', - 'xcshareddata', - 'WorkspaceSettings.xcsettings') - if os.path.isfile(xcwspacesharedsettings): - json_data = json.loads(LoadXcodeProjectAsJSON(xcwspacesharedsettings)) - else: - json_data = {} - json_data['BuildSystemType'] = 'Original' - WriteXcodeProject(xcwspacesharedsettings, json.dumps(json_data)) + new_root = CreateGroup(project, group_object, folder) + group_object = new_root + return group_object def ConvertGnXcodeProject(root_dir, input_dir, output_dir, configurations): '''Tweak the Xcode project generated by gn to support multiple configurations. - The Xcode projects generated by "gn gen --ide" only supports a single - platform and configuration (as the platform and configuration are set per - output directory). This method takes as input such projects and add support - for multiple configurations and platforms (to allow devs to select them in - Xcode). + The Xcode projects generated by "gn gen --ide" only supports a single + platform and configuration (as the platform and configuration are set + per output directory). This method takes as input such projects and + add support for multiple configurations and platforms (to allow devs + to select them in Xcode). - Args: - input_dir: directory containing the XCode projects created by "gn gen - --ide" - output_dir: directory where the tweaked Xcode projects will be saved - configurations: list of string corresponding to the configurations that - need to be supported by the tweaked Xcode projects, must contains at - least one value. - ''' - # Update products project. - products = os.path.join('products.xcodeproj', 'project.pbxproj') - product_input = os.path.join(input_dir, products) - product_output = os.path.join(output_dir, products) - UpdateProductsProject(product_input, product_output, configurations, - root_dir) + Args: + input_dir: directory containing the XCode projects created by "gn gen --ide" + output_dir: directory where the tweaked Xcode projects will be saved + configurations: list of string corresponding to the configurations that + need to be supported by the tweaked Xcode projects, must contains at + least one value. + ''' - # Copy all workspace. - xcwspace = os.path.join('all.xcworkspace', 'contents.xcworkspacedata') - CopyFileIfChanged(os.path.join(input_dir, xcwspace), - os.path.join(output_dir, xcwspace)) + # Update the project (supports legacy name "products.xcodeproj" or the new + # project name "all.xcodeproj"). + for project_name in ('all.xcodeproj', 'products.xcodeproj'): + if os.path.exists(os.path.join(input_dir, project_name)): + UpdateXcodeProject(os.path.join(input_dir, project_name), + configurations, root_dir) - # TODO(crbug.com/852522): Disable new BuildSystemType. - DisableNewBuildSystem(output_dir) + CopyTreeIfChanged(os.path.join(input_dir, project_name), + os.path.join(output_dir, project_name)) - # TODO(crbug.com/679110): gn has been modified to remove 'sources.xcodeproj' - # and keep 'all.xcworkspace' and 'products.xcodeproj'. The following code is - # here to support both old and new projects setup and will be removed once - # gn has rolled past it. - sources = os.path.join('sources.xcodeproj', 'project.pbxproj') - if os.path.isfile(os.path.join(input_dir, sources)): - CopyFileIfChanged(os.path.join(input_dir, sources), - os.path.join(output_dir, sources)) + else: + shutil.rmtree(os.path.join(output_dir, project_name), + ignore_errors=True) + + # Copy all.xcworkspace if it exists (will be removed in a future gn version). + workspace_name = 'all.xcworkspace' + if os.path.exists(os.path.join(input_dir, workspace_name)): + CopyTreeIfChanged(os.path.join(input_dir, workspace_name), + os.path.join(output_dir, workspace_name)) + else: + shutil.rmtree(os.path.join(output_dir, workspace_name), + ignore_errors=True) def Main(args): @@ -264,8 +331,18 @@ def Main(args): sys.stderr.write('Input directory does not exists.\n') return 1 - required = set(['products.xcodeproj', 'all.xcworkspace']) - if not required.issubset(os.listdir(args.input)): + # Depending on the version of "gn", there should be either one project file + # named "all.xcodeproj" or a project file named "products.xcodeproj" and a + # workspace named "all.xcworkspace". + required_files_sets = [ + set(("all.xcodeproj",)), + set(("products.xcodeproj", "all.xcworkspace")), + ] + + for required_files in required_files_sets: + if required_files.issubset(os.listdir(args.input)): + break + else: sys.stderr.write( 'Input directory does not contain all necessary Xcode projects.\n') return 1 diff --git a/build/ios/setup_ios_gn.py b/build/ios/setup_ios_gn.py index 934b67c6..5c70332d 100755 --- a/build/ios/setup_ios_gn.py +++ b/build/ios/setup_ios_gn.py @@ -23,25 +23,29 @@ import shutil import subprocess import sys import tempfile -import ConfigParser try: - import cStringIO as StringIO + import configparser except ImportError: - import StringIO + import ConfigParser as configparser + +try: + import StringIO as io +except ImportError: + import io SUPPORTED_TARGETS = ('iphoneos', 'iphonesimulator') SUPPORTED_CONFIGS = ('Debug', 'Release', 'Profile', 'Official', 'Coverage') -class ConfigParserWithStringInterpolation(ConfigParser.SafeConfigParser): +class ConfigParserWithStringInterpolation(configparser.SafeConfigParser): '''A .ini file parser that supports strings and environment variables.''' ENV_VAR_PATTERN = re.compile(r'\$([A-Za-z0-9_]+)') def values(self, section): - return map(lambda (k, v): self._UnquoteString(self._ExpandEnvVar(v)), - ConfigParser.SafeConfigParser.items(self, section)) + return map(lambda kv: self._UnquoteString(self._ExpandEnvVar(kv[1])), + configparser.ConfigParser.items(self, section)) def getstring(self, section, option): return self._UnquoteString(self._ExpandEnvVar(self.get(section, @@ -94,7 +98,25 @@ class GnGenerator(object): """ args = [] + # build/config/ios/ios_sdk.gni asserts that goma is not enabled when + # building Official, so ignore the value of goma.enabled when creating + # args.gn for Official. + if self._config != 'Official': + if self._settings.getboolean('goma', 'enabled'): + args.append(('use_goma', True)) + goma_dir = self._settings.getstring('goma', 'install') + if goma_dir: + args.append( + ('goma_dir', '"%s"' % os.path.expanduser(goma_dir))) + args.append(('is_debug', self._config in ('Debug', 'Coverage'))) + args.append(('enable_dsyms', self._config in ('Profile', 'Official'))) + args.append(('enable_stripping', 'enable_dsyms')) + args.append(('is_official_build', self._config == 'Official')) + args.append(('is_chrome_branded', 'is_official_build')) + args.append(('use_xcode_clang', 'false')) + args.append(('use_clang_coverage', self._config == 'Coverage')) + args.append(('is_component_build', False)) if os.environ.get('FORCE_MAC_TOOLCHAIN', '0') == '1': args.append(('use_system_xcode', False)) @@ -116,7 +138,7 @@ class GnGenerator(object): return args def Generate(self, gn_path, root_path, out_path): - buf = StringIO.StringIO() + buf = io.StringIO() self.WriteArgsGn(buf) WriteToFileIfChanged(os.path.join(out_path, 'args.gn'), buf.getvalue(), @@ -126,20 +148,20 @@ class GnGenerator(object): self.GetGnCommand(gn_path, root_path, out_path, True)) def CreateGnRules(self, gn_path, root_path, out_path): - buf = StringIO.StringIO() + buf = io.StringIO() self.WriteArgsGn(buf) WriteToFileIfChanged(os.path.join(out_path, 'args.gn'), buf.getvalue(), overwrite=True) - buf = StringIO.StringIO() + buf = io.StringIO() gn_command = self.GetGnCommand(gn_path, root_path, out_path, False) self.WriteBuildNinja(buf, gn_command) WriteToFileIfChanged(os.path.join(out_path, 'build.ninja'), buf.getvalue(), overwrite=False) - buf = StringIO.StringIO() + buf = io.StringIO() self.WriteBuildNinjaDeps(buf) WriteToFileIfChanged(os.path.join(out_path, 'build.ninja.d'), buf.getvalue(), @@ -196,16 +218,13 @@ class GnGenerator(object): if generate_xcode_project: gn_command.append('--ide=xcode') gn_command.append('--root-target=gn_all') - if self._settings.getboolean('goma', 'enabled'): - ninja_jobs = self._settings.getint('xcode', 'jobs') or 200 - gn_command.append('--ninja-extra-args=-j%s' % ninja_jobs) + gn_command.append('--ninja-executable=autoninja') if self._settings.has_section('filters'): target_filters = self._settings.values('filters') if target_filters: gn_command.append('--filters=%s' % ';'.join(target_filters)) - # TODO(justincohen): --check is currently failing in crashpad. - # else: - # gn_command.append('--check') + else: + gn_command.append('--check') gn_command.append('gen') gn_command.append('//%s' % os.path.relpath(os.path.abspath(out_path), os.path.abspath(src_path))) @@ -296,6 +315,13 @@ def Main(args): dest='import_rules', default=[], help='path to file defining default gn variables') + parser.add_argument('--gn-path', + default=None, + help='path to gn binary (default: look up in $PATH)') + parser.add_argument( + '--build-dir', + default='out', + help='path where the build should be created (default: %(default)s)') args = parser.parse_args(args) # Load configuration (first global and then any user overrides). @@ -320,25 +346,16 @@ def Main(args): settings.getstring('build', 'arch')) sys.exit(1) - if settings.getboolean('goma', 'enabled'): - if settings.getint('xcode', 'jobs') < 0: - sys.stderr.write('ERROR: invalid value for xcode.jobs: %s\n' % - settings.get('xcode', 'jobs')) + # Find path to gn binary either from command-line or in PATH. + if args.gn_path: + gn_path = args.gn_path + else: + gn_path = FindGn() + if gn_path is None: + sys.stderr.write('ERROR: cannot find gn in PATH\n') sys.exit(1) - goma_install = os.path.expanduser(settings.getstring('goma', 'install')) - if not os.path.isdir(goma_install): - sys.stderr.write('WARNING: goma.install directory not found: %s\n' % - settings.get('goma', 'install')) - sys.stderr.write('WARNING: disabling goma\n') - settings.set('goma', 'enabled', 'false') - # Find gn binary in PATH. - gn_path = FindGn() - if gn_path is None: - sys.stderr.write('ERROR: cannot find gn in PATH\n') - sys.exit(1) - - out_dir = os.path.join(args.root, 'out') + out_dir = os.path.join(args.root, args.build_dir) if not os.path.isdir(out_dir): os.makedirs(out_dir)