From d63fb70c693093d435d81755d576eb4c373613ac Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Tue, 27 Sep 2016 17:29:13 +0200 Subject: [PATCH] Publish the amalgamation tools PUBLISHED_FROM=27ed0bd32e33252495b92361d2943a3450448f62 --- README.md | 4 ++ tools/README.md | 22 +++++++ tools/amalgam | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ tools/unamalgam | 43 ++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 tools/README.md create mode 100755 tools/amalgam create mode 100755 tools/unamalgam diff --git a/README.md b/README.md index aee4b408..ea1313d6 100644 --- a/README.md +++ b/README.md @@ -73,4 +73,8 @@ To submit contributions, sign [Cesanta CLA](https://docs.cesanta.com/contributors_la.shtml) and send GitHub pull request. You retain the copyright on your contributions. +## Working with the Source Code + +See [tools](https://github.com/cesanta/mongoose/tree/master/tools) directory. + [![Analytics](https://ga-beacon.appspot.com/UA-42732794-5/project-page)](https://github.com/cesanta/mongoose) diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..e05db5c4 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,22 @@ +# Amalgamation + +Mongoose is distributed as two files, `mongoose.c` and `mongoose.h` for ease of integration. +However, when developing Mongoose itself, it can be quite difficult to work with them. +Internally, these files are an _amalgamation_ of source an header modules. +This directory contains utilities to split and re-constitute amalgamated files. + +Here's how `mongoose.c` can be split into its consituent parts: +``` +$ tools/unamalgam mongoose.c +=> mongoose/src/internal.h +=> common/cs_dbg.h +... +``` + +This produces directories and files under `mongoose/` and `common/` that are easeier to work with. +It also produces `mongoose.c.manifest` which can later be used to reconstruct the file back: +``` +$ tools/amalgam --prefix=MG --public-header=mongoose.h $(cat mongoose.c.manifest) > mongoose.c +``` + +The same applies to `mongoose.h`, except `--public-header` should be omitted during amalgamation. diff --git a/tools/amalgam b/tools/amalgam new file mode 100755 index 00000000..b60dc85a --- /dev/null +++ b/tools/amalgam @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# +# Generate a reversible amalgamation of several C source files +# along with their required internal headers. +# +# This script assumes that there are a bunch of C files, a bunch +# of private header files and one public header file. +# +# The script takes a list of C file names, parses `#include` directives +# found in them and recursively resolves dependencies in such a way +# that a header referenced from an included header will be emitted before the +# header that depends on it. All headers will always be emitted before the +# source files. +# +# The embedded include files usually contain private internals. +# However sometimes it's necessary for some other tools or for advanced users +# to have access to internal definitions. One such example is the generated +# C source containing frozen heap. The amalgamation script will allow users +# to include the amalgamated C file and cause extract only the internal headers: +# +# #define NS_EXPORT_INTERNAL_HEADERS +# #include "v7.c" +# +# Where `NS` can be overridden via the --prefix flag. +# This feature can be enabled with the --exportable-headers, and basically +# all it does is to wrap the C body in a preprocessor guard. +# +# TODO(mkm): make it work also for mongoose where we also generate +# the public header from a bunch of unamalgamated header files. +# Currently this script can handle mongoose amalgamation because it doesn't +# flip the --autoinc flag. +# + +import argparse +import re +import sys +import os +from StringIO import StringIO + +parser = argparse.ArgumentParser(description='Produce an amalgamated source') +parser.add_argument('--prefix', default="NS", + help='prefix for MODULE_LINES guard') +parser.add_argument('--srcdir', default=".", help='source dir') +parser.add_argument('--ignore', default="", + help='comma separated list of files to not amalgam') +# hack, teach amalgam to render the LICENSE file instead +parser.add_argument('--first', type=str, help='put this file in first position.' + ' Usually contains licensing info') +parser.add_argument('--public-header', dest="public", + help='name of the public header file that will be' + ' included at the beginning of the file') +parser.add_argument('--autoinc', action='store_true', + help='automatically embed include files') +parser.add_argument('--exportable-headers', dest="export", action='store_true', + help='allow exporting internal headers') +parser.add_argument('-I', default=".", dest='include_path', help='include path') +parser.add_argument('sources', nargs='*', help='sources') + +class File(object): + def __init__(self, name): + self.name = name + self.buf = StringIO() + emit_file(self.buf, self.name) + + def emit(self): + print self.buf.getvalue(), + + +args = parser.parse_args() + +sources = [] +includes = [] + +already_included = set() + +ignore_files = [i.strip() for i in args.ignore.split(',')] + +def should_ignore(name): + return (name in already_included + or not os.path.exists(resolve(name)) + or name in ignore_files) + +def resolve(path): + if os.path.isabs(path) or os.path.exists(path): + p = path + else: + p = os.path.join(args.include_path, path) + if os.path.exists(p): + p = os.path.realpath(p).replace('%s/' % os.getcwd(), '') +# print >>sys.stderr, '%s -> %s' % (path, p) + return p + +def emit_line_directive(out, name): + print >>out, '''#ifdef %(prefix)s_MODULE_LINES +#line 1 "%(name)s" +#endif''' % dict( + prefix = args.prefix, + name = resolve(name), +) + +def emit_body(out, name): + path = resolve(name) + if not os.path.exists(path): + print >>out, '#include "%s"' % (name,) + return + + with open(resolve(name)) as f: + for l in f: + match = re.match('(#include "(.*)")', l) + if match: + all, path = match.groups() + if args.autoinc: + if not should_ignore(path): + already_included.add(path) + includes.append(File(path)) + print >>out, '/* Amalgamated: %s */' % (all,) + else: + print >>out, l, + + +def emit_file(out, name): + emit_line_directive(out, name) + emit_body(out, name) + +for i in args.sources: + sources.append(File(i)) + +if args.first: + for inc in reversed(args.first.split(',')): + for i, f in enumerate(includes): + if f.name == inc: + del includes[i] + includes.insert(0, f) + break + +# emitting + +if args.public: + print '#include "%s"' % (args.public) + +for i in includes: + i.emit() + +if args.export: + print '#ifndef %s_EXPORT_INTERNAL_HEADERS' % (args.prefix,) +for i in sources: + i.emit() +if args.export: + print '#endif /* %s_EXPORT_INTERNAL_HEADERS */' % (args.prefix,) diff --git a/tools/unamalgam b/tools/unamalgam new file mode 100755 index 00000000..2fd6a41e --- /dev/null +++ b/tools/unamalgam @@ -0,0 +1,43 @@ +#!/usr/bin/env python +import sys +import re +import os + +cur_src = None +in_mod = False +ofile = None + +strip_re = re.compile(r'/\* Amalgamated: (.*) \*/') +def clean(l): + return strip_re.sub(r'\1', l) + +manifest = [] +fname = sys.argv[1] +with open(fname) as f: + for l in f: + if re.match('#ifdef .*_MODULE_LINES', l): + l = next(f) + g = re.match(r'#line [01] "(.*)"', l) + cur_src = g.group(1) + + # if there is currently opened file, close it + if ofile: + ofile.close() + + cur_src = re.sub(r'\.\./', '', cur_src) + + # create directory for the next file if needed + cur_src_dir = os.path.dirname(cur_src) + if cur_src_dir != '' and not os.path.exists(cur_src_dir): + os.makedirs(cur_src_dir) + + # open next file for writing + ofile = open(cur_src, "w") + print >>sys.stderr, '=> %s' % cur_src + manifest.append(cur_src) + next(f) + elif ofile: + ofile.write(clean(l)) + +m = open('%s.manifest' % os.path.basename(fname), 'w') +print >>m, '\n'.join(manifest)