Copies the latest setup_ios_gn.py from Chromium.

Also includes the latest convert_gn_xcodeproj.py.

BUG=None

Change-Id: Icd82dd248ebf30b98054336d82f73a70d63f1d71
Reviewed-on: https://chromium-review.googlesource.com/c/crashpad/crashpad/+/2276602
Reviewed-by: Mark Mentovai <mark@chromium.org>
Commit-Queue: Rohit Rao <rohitrao@chromium.org>
This commit is contained in:
Rohit Rao 2020-07-06 21:18:05 -04:00 committed by Commit Bot
parent d2c22d6b4b
commit 17b08b5aab
2 changed files with 224 additions and 130 deletions

View File

@ -29,8 +29,6 @@ import filecmp
import json
import hashlib
import os
import plistlib
import random
import shutil
import subprocess
import sys
@ -47,7 +45,8 @@ class XcodeProject(object):
while True:
self.counter += 1
str_id = "%s %s %d" % (parent_name, obj['isa'], self.counter)
new_id = hashlib.sha1(str_id).hexdigest()[:24].upper()
new_id = hashlib.sha1(
str_id.encode("utf-8")).hexdigest()[:24].upper()
# Make sure ID is unique. It's possible there could be an id
# conflict since this is run after GN runs.
@ -56,46 +55,65 @@ class XcodeProject(object):
return new_id
def check_output(command):
"""Wrapper around subprocess.check_output that decode output as utf-8."""
return subprocess.check_output(command).decode('utf-8')
def CopyFileIfChanged(source_path, target_path):
"""Copy |source_path| to |target_path| is different."""
"""Copy |source_path| to |target_path| if different."""
target_dir = os.path.dirname(target_path)
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
if (not os.path.exists(target_path) or
not filecmp.cmp(source_path, target_path)):
if not os.path.exists(target_path) or \
not filecmp.cmp(source_path, target_path):
shutil.copyfile(source_path, target_path)
def LoadXcodeProjectAsJSON(path):
def CopyTreeIfChanged(source, target):
"""Copy |source| to |target| recursively; files are copied iff changed."""
if os.path.isfile(source):
return CopyFileIfChanged(source, target)
if not os.path.isdir(target):
os.makedirs(target)
for name in os.listdir(source):
CopyTreeIfChanged(os.path.join(source, name),
os.path.join(target, name))
def LoadXcodeProjectAsJSON(project_dir):
"""Return Xcode project at |path| as a JSON string."""
return subprocess.check_output(
['plutil', '-convert', 'json', '-o', '-', path])
return check_output([
'plutil', '-convert', 'json', '-o', '-',
os.path.join(project_dir, 'project.pbxproj')
])
def WriteXcodeProject(output_path, json_string):
"""Save Xcode project to |output_path| as XML."""
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(json_string)
temp_file.write(json_string.encode("utf-8"))
temp_file.flush()
subprocess.check_call(['plutil', '-convert', 'xml1', temp_file.name])
CopyFileIfChanged(temp_file.name, output_path)
CopyFileIfChanged(temp_file.name,
os.path.join(output_path, 'project.pbxproj'))
def UpdateProductsProject(file_input, file_output, configurations, root_dir):
"""Update Xcode project to support multiple configurations.
def UpdateXcodeProject(project_dir, configurations, root_dir):
"""Update inplace Xcode project to support multiple configurations.
Args:
file_input: path to the input Xcode project
file_output: path to the output file
configurations: list of string corresponding to the configurations that
project_dir: path to the input Xcode project
configurations: list of string corresponding to the configurations that
need to be supported by the tweaked Xcode projects, must contains at
least one value.
root_dir: path to the root directory used to find markdown files
"""
json_data = json.loads(LoadXcodeProjectAsJSON(file_input))
json_data = json.loads(LoadXcodeProjectAsJSON(project_dir))
project = XcodeProject(json_data['objects'])
objects_to_remove = []
for value in project.objects.values():
for value in list(project.objects.values()):
isa = value['isa']
# Teach build shell script to look for the configuration and platform.
@ -111,10 +129,8 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir):
build_config_template = project.objects[value['buildConfigurations']
[0]]
build_settings = build_config_template['buildSettings']
build_settings['CONFIGURATION_BUILD_DIR'] = (
'$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)')
build_settings['CODE_SIGN_IDENTITY'] = ''
build_config_template['buildSettings']['CONFIGURATION_BUILD_DIR'] =\
'$(PROJECT_DIR)/../$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)'
value['buildConfigurations'] = []
for configuration in configurations:
@ -126,20 +142,96 @@ def UpdateProductsProject(file_input, file_output, configurations, root_dir):
for object_id in objects_to_remove:
del project.objects[object_id]
AddMarkdownToProject(project, root_dir, json_data['rootObject'])
source = GetOrCreateRootGroup(project, json_data['rootObject'], 'Source')
AddMarkdownToProject(project, root_dir, source)
SortFileReferencesByName(project, source)
objects = collections.OrderedDict(sorted(project.objects.iteritems()))
WriteXcodeProject(file_output, json.dumps(json_data))
objects = collections.OrderedDict(sorted(project.objects.items()))
WriteXcodeProject(project_dir, json.dumps(json_data))
def AddMarkdownToProject(project, root_dir, root_object):
def CreateGroup(project, parent_group, group_name, path=None):
group_object = {
'children': [],
'isa': 'PBXGroup',
'name': group_name,
'sourceTree': '<group>',
}
if path is not None:
group_object['path'] = path
parent_group_name = parent_group.get('name', '')
group_object_key = project.AddObject(parent_group_name, group_object)
parent_group['children'].append(group_object_key)
return group_object
def GetOrCreateRootGroup(project, root_object, group_name):
main_group = project.objects[project.objects[root_object]['mainGroup']]
for child_key in main_group['children']:
child = project.objects[child_key]
if child['name'] == group_name:
return child
return CreateGroup(project, main_group, group_name, path='../..')
class ObjectKey(object):
"""Wrapper around PBXFileReference and PBXGroup for sorting.
A PBXGroup represents a "directory" containing a list of files in an
Xcode project; it can contain references to a list of directories or
files.
A PBXFileReference represents a "file".
The type is stored in the object "isa" property as a string. Since we
want to sort all directories before all files, the < and > operators
are defined so that if "isa" is different, they are sorted in the
reverse of alphabetic ordering, otherwise the name (or path) property
is checked and compared in alphabetic order.
"""
def __init__(self, obj):
self.isa = obj['isa']
if 'name' in obj:
self.name = obj['name']
else:
self.name = obj['path']
def __lt__(self, other):
if self.isa != other.isa:
return self.isa > other.isa
return self.name < other.name
def __gt__(self, other):
if self.isa != other.isa:
return self.isa < other.isa
return self.name > other.name
def __eq__(self, other):
return self.isa == other.isa and self.name == other.name
def SortFileReferencesByName(project, group_object):
SortFileReferencesByNameWithSortKey(
project, group_object, lambda ref: ObjectKey(project.objects[ref]))
def SortFileReferencesByNameWithSortKey(project, group_object, sort_key):
group_object['children'].sort(key=sort_key)
for key in group_object['children']:
child = project.objects[key]
if child['isa'] == 'PBXGroup':
SortFileReferencesByNameWithSortKey(project, child, sort_key)
def AddMarkdownToProject(project, root_dir, group_object):
list_files_cmd = ['git', '-C', root_dir, 'ls-files', '*.md']
paths = subprocess.check_output(list_files_cmd).splitlines()
paths = check_output(list_files_cmd).splitlines()
ios_internal_dir = os.path.join(root_dir, 'ios_internal')
if os.path.exists(ios_internal_dir):
list_files_cmd = ['git', '-C', ios_internal_dir, 'ls-files', '*.md']
ios_paths = subprocess.check_output(list_files_cmd).splitlines()
paths.extend(["ios_internal/" + path for path in ios_paths])
ios_paths = check_output(list_files_cmd).splitlines()
paths.extend([os.path.join("ios_internal", path) for path in ios_paths])
for path in paths:
new_markdown_entry = {
"fileEncoding": "4",
@ -150,96 +242,71 @@ def AddMarkdownToProject(project, root_dir, root_object):
"sourceTree": "<group>"
}
new_markdown_entry_id = project.AddObject('sources', new_markdown_entry)
folder = GetFolderForPath(project, root_object, os.path.dirname(path))
folder = GetFolderForPath(project, group_object, os.path.dirname(path))
folder['children'].append(new_markdown_entry_id)
def GetFolderForPath(project, rootObject, path):
def GetFolderForPath(project, group_object, path):
objects = project.objects
# 'Sources' is always the first child of
# project->rootObject->mainGroup->children.
root = objects[objects[objects[rootObject]['mainGroup']]['children'][0]]
if not path:
return root
return group_object
for folder in path.split('/'):
children = root['children']
children = group_object['children']
new_root = None
for child in children:
if (objects[child]['isa'] == 'PBXGroup' and
objects[child]['name'] == folder):
if objects[child]['isa'] == 'PBXGroup' and \
objects[child]['name'] == folder:
new_root = objects[child]
break
if not new_root:
# If the folder isn't found we could just cram it into the leaf
# existing folder, but that leads to folders with tons of README.md
# inside.
new_group = {
"children": [],
"isa": "PBXGroup",
"name": folder,
"sourceTree": "<group>"
}
new_group_id = project.AddObject('sources', new_group)
children.append(new_group_id)
new_root = objects[new_group_id]
root = new_root
return root
def DisableNewBuildSystem(output_dir):
"""Disables the new build system due to crbug.com/852522 """
xcwspacesharedsettings = os.path.join(output_dir, 'all.xcworkspace',
'xcshareddata',
'WorkspaceSettings.xcsettings')
if os.path.isfile(xcwspacesharedsettings):
json_data = json.loads(LoadXcodeProjectAsJSON(xcwspacesharedsettings))
else:
json_data = {}
json_data['BuildSystemType'] = 'Original'
WriteXcodeProject(xcwspacesharedsettings, json.dumps(json_data))
new_root = CreateGroup(project, group_object, folder)
group_object = new_root
return group_object
def ConvertGnXcodeProject(root_dir, input_dir, output_dir, configurations):
'''Tweak the Xcode project generated by gn to support multiple
configurations.
The Xcode projects generated by "gn gen --ide" only supports a single
platform and configuration (as the platform and configuration are set per
output directory). This method takes as input such projects and add support
for multiple configurations and platforms (to allow devs to select them in
Xcode).
The Xcode projects generated by "gn gen --ide" only supports a single
platform and configuration (as the platform and configuration are set
per output directory). This method takes as input such projects and
add support for multiple configurations and platforms (to allow devs
to select them in Xcode).
Args:
input_dir: directory containing the XCode projects created by "gn gen
--ide"
output_dir: directory where the tweaked Xcode projects will be saved
configurations: list of string corresponding to the configurations that
need to be supported by the tweaked Xcode projects, must contains at
least one value.
'''
# Update products project.
products = os.path.join('products.xcodeproj', 'project.pbxproj')
product_input = os.path.join(input_dir, products)
product_output = os.path.join(output_dir, products)
UpdateProductsProject(product_input, product_output, configurations,
root_dir)
Args:
input_dir: directory containing the XCode projects created by "gn gen --ide"
output_dir: directory where the tweaked Xcode projects will be saved
configurations: list of string corresponding to the configurations that
need to be supported by the tweaked Xcode projects, must contains at
least one value.
'''
# Copy all workspace.
xcwspace = os.path.join('all.xcworkspace', 'contents.xcworkspacedata')
CopyFileIfChanged(os.path.join(input_dir, xcwspace),
os.path.join(output_dir, xcwspace))
# Update the project (supports legacy name "products.xcodeproj" or the new
# project name "all.xcodeproj").
for project_name in ('all.xcodeproj', 'products.xcodeproj'):
if os.path.exists(os.path.join(input_dir, project_name)):
UpdateXcodeProject(os.path.join(input_dir, project_name),
configurations, root_dir)
# TODO(crbug.com/852522): Disable new BuildSystemType.
DisableNewBuildSystem(output_dir)
CopyTreeIfChanged(os.path.join(input_dir, project_name),
os.path.join(output_dir, project_name))
# TODO(crbug.com/679110): gn has been modified to remove 'sources.xcodeproj'
# and keep 'all.xcworkspace' and 'products.xcodeproj'. The following code is
# here to support both old and new projects setup and will be removed once
# gn has rolled past it.
sources = os.path.join('sources.xcodeproj', 'project.pbxproj')
if os.path.isfile(os.path.join(input_dir, sources)):
CopyFileIfChanged(os.path.join(input_dir, sources),
os.path.join(output_dir, sources))
else:
shutil.rmtree(os.path.join(output_dir, project_name),
ignore_errors=True)
# Copy all.xcworkspace if it exists (will be removed in a future gn version).
workspace_name = 'all.xcworkspace'
if os.path.exists(os.path.join(input_dir, workspace_name)):
CopyTreeIfChanged(os.path.join(input_dir, workspace_name),
os.path.join(output_dir, workspace_name))
else:
shutil.rmtree(os.path.join(output_dir, workspace_name),
ignore_errors=True)
def Main(args):
@ -264,8 +331,18 @@ def Main(args):
sys.stderr.write('Input directory does not exists.\n')
return 1
required = set(['products.xcodeproj', 'all.xcworkspace'])
if not required.issubset(os.listdir(args.input)):
# Depending on the version of "gn", there should be either one project file
# named "all.xcodeproj" or a project file named "products.xcodeproj" and a
# workspace named "all.xcworkspace".
required_files_sets = [
set(("all.xcodeproj",)),
set(("products.xcodeproj", "all.xcworkspace")),
]
for required_files in required_files_sets:
if required_files.issubset(os.listdir(args.input)):
break
else:
sys.stderr.write(
'Input directory does not contain all necessary Xcode projects.\n')
return 1

View File

@ -23,25 +23,29 @@ import shutil
import subprocess
import sys
import tempfile
import ConfigParser
try:
import cStringIO as StringIO
import configparser
except ImportError:
import StringIO
import ConfigParser as configparser
try:
import StringIO as io
except ImportError:
import io
SUPPORTED_TARGETS = ('iphoneos', 'iphonesimulator')
SUPPORTED_CONFIGS = ('Debug', 'Release', 'Profile', 'Official', 'Coverage')
class ConfigParserWithStringInterpolation(ConfigParser.SafeConfigParser):
class ConfigParserWithStringInterpolation(configparser.SafeConfigParser):
'''A .ini file parser that supports strings and environment variables.'''
ENV_VAR_PATTERN = re.compile(r'\$([A-Za-z0-9_]+)')
def values(self, section):
return map(lambda (k, v): self._UnquoteString(self._ExpandEnvVar(v)),
ConfigParser.SafeConfigParser.items(self, section))
return map(lambda kv: self._UnquoteString(self._ExpandEnvVar(kv[1])),
configparser.ConfigParser.items(self, section))
def getstring(self, section, option):
return self._UnquoteString(self._ExpandEnvVar(self.get(section,
@ -94,7 +98,25 @@ class GnGenerator(object):
"""
args = []
# build/config/ios/ios_sdk.gni asserts that goma is not enabled when
# building Official, so ignore the value of goma.enabled when creating
# args.gn for Official.
if self._config != 'Official':
if self._settings.getboolean('goma', 'enabled'):
args.append(('use_goma', True))
goma_dir = self._settings.getstring('goma', 'install')
if goma_dir:
args.append(
('goma_dir', '"%s"' % os.path.expanduser(goma_dir)))
args.append(('is_debug', self._config in ('Debug', 'Coverage')))
args.append(('enable_dsyms', self._config in ('Profile', 'Official')))
args.append(('enable_stripping', 'enable_dsyms'))
args.append(('is_official_build', self._config == 'Official'))
args.append(('is_chrome_branded', 'is_official_build'))
args.append(('use_xcode_clang', 'false'))
args.append(('use_clang_coverage', self._config == 'Coverage'))
args.append(('is_component_build', False))
if os.environ.get('FORCE_MAC_TOOLCHAIN', '0') == '1':
args.append(('use_system_xcode', False))
@ -116,7 +138,7 @@ class GnGenerator(object):
return args
def Generate(self, gn_path, root_path, out_path):
buf = StringIO.StringIO()
buf = io.StringIO()
self.WriteArgsGn(buf)
WriteToFileIfChanged(os.path.join(out_path, 'args.gn'),
buf.getvalue(),
@ -126,20 +148,20 @@ class GnGenerator(object):
self.GetGnCommand(gn_path, root_path, out_path, True))
def CreateGnRules(self, gn_path, root_path, out_path):
buf = StringIO.StringIO()
buf = io.StringIO()
self.WriteArgsGn(buf)
WriteToFileIfChanged(os.path.join(out_path, 'args.gn'),
buf.getvalue(),
overwrite=True)
buf = StringIO.StringIO()
buf = io.StringIO()
gn_command = self.GetGnCommand(gn_path, root_path, out_path, False)
self.WriteBuildNinja(buf, gn_command)
WriteToFileIfChanged(os.path.join(out_path, 'build.ninja'),
buf.getvalue(),
overwrite=False)
buf = StringIO.StringIO()
buf = io.StringIO()
self.WriteBuildNinjaDeps(buf)
WriteToFileIfChanged(os.path.join(out_path, 'build.ninja.d'),
buf.getvalue(),
@ -196,16 +218,13 @@ class GnGenerator(object):
if generate_xcode_project:
gn_command.append('--ide=xcode')
gn_command.append('--root-target=gn_all')
if self._settings.getboolean('goma', 'enabled'):
ninja_jobs = self._settings.getint('xcode', 'jobs') or 200
gn_command.append('--ninja-extra-args=-j%s' % ninja_jobs)
gn_command.append('--ninja-executable=autoninja')
if self._settings.has_section('filters'):
target_filters = self._settings.values('filters')
if target_filters:
gn_command.append('--filters=%s' % ';'.join(target_filters))
# TODO(justincohen): --check is currently failing in crashpad.
# else:
# gn_command.append('--check')
else:
gn_command.append('--check')
gn_command.append('gen')
gn_command.append('//%s' % os.path.relpath(os.path.abspath(out_path),
os.path.abspath(src_path)))
@ -296,6 +315,13 @@ def Main(args):
dest='import_rules',
default=[],
help='path to file defining default gn variables')
parser.add_argument('--gn-path',
default=None,
help='path to gn binary (default: look up in $PATH)')
parser.add_argument(
'--build-dir',
default='out',
help='path where the build should be created (default: %(default)s)')
args = parser.parse_args(args)
# Load configuration (first global and then any user overrides).
@ -320,25 +346,16 @@ def Main(args):
settings.getstring('build', 'arch'))
sys.exit(1)
if settings.getboolean('goma', 'enabled'):
if settings.getint('xcode', 'jobs') < 0:
sys.stderr.write('ERROR: invalid value for xcode.jobs: %s\n' %
settings.get('xcode', 'jobs'))
# Find path to gn binary either from command-line or in PATH.
if args.gn_path:
gn_path = args.gn_path
else:
gn_path = FindGn()
if gn_path is None:
sys.stderr.write('ERROR: cannot find gn in PATH\n')
sys.exit(1)
goma_install = os.path.expanduser(settings.getstring('goma', 'install'))
if not os.path.isdir(goma_install):
sys.stderr.write('WARNING: goma.install directory not found: %s\n' %
settings.get('goma', 'install'))
sys.stderr.write('WARNING: disabling goma\n')
settings.set('goma', 'enabled', 'false')
# Find gn binary in PATH.
gn_path = FindGn()
if gn_path is None:
sys.stderr.write('ERROR: cannot find gn in PATH\n')
sys.exit(1)
out_dir = os.path.join(args.root, 'out')
out_dir = os.path.join(args.root, args.build_dir)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)