Skip to content

Commit 18d0987

Browse files
jujjuj
authored
Closure externs and annotations (emscripten-core#8990)
* Add support for --closure-externs and --closure-annotations * Add tests and docs * Flake * fix bad rebase * Use --closure-args instead of --closure-annotations & --closure-externs * Drop handling of extra quotes in --closure-args
1 parent 4ea6325 commit 18d0987

8 files changed

Lines changed: 84 additions & 52 deletions

File tree

‎emcc.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ def __init__(self):
214214
self.llvm_lto = None
215215
self.default_cxx_std = '-std=c++03' # Enforce a consistent C++ standard when compiling .cpp files, if user does not specify one on the cmdline.
216216
self.use_closure_compiler = None
217+
self.closure_args = []
217218
self.js_transform = None
218219
self.pre_js = '' # before all js
219220
self.post_js = '' # after all js
@@ -271,6 +272,7 @@ def __init__(self, target, options, js_transform_tempfiles, in_temp):
271272
self.emit_symbol_map = options.emit_symbol_map
272273
self.profiling_funcs = options.profiling_funcs
273274
self.use_closure_compiler = options.use_closure_compiler
275+
self.closure_args = options.closure_args
274276

275277
self.js_transform_tempfiles = js_transform_tempfiles
276278
self.in_temp = in_temp
@@ -341,7 +343,8 @@ def run_passes(self, passes, title, just_split, just_concat):
341343
final = shared.Building.js_optimizer(final, passes, use_source_map(self),
342344
self.extra_info, just_split=just_split,
343345
just_concat=just_concat,
344-
output_filename=self.in_temp(os.path.basename(final) + '.jsopted.js'))
346+
output_filename=self.in_temp(os.path.basename(final) + '.jsopted.js'),
347+
extra_closure_args=self.closure_args)
345348
self.js_transform_tempfiles.append(final)
346349
save_intermediate(title, suffix='js' if 'emitJSON' not in passes else 'json')
347350

@@ -2338,7 +2341,8 @@ def get_eliminate():
23382341
logger.debug('running closure')
23392342
# no need to add this to js_transform_tempfiles, because closure and
23402343
# debug_level > 0 are never simultaneously true
2341-
final = shared.Building.closure_compiler(final, pretty=options.debug_level >= 1)
2344+
final = shared.Building.closure_compiler(final, pretty=options.debug_level >= 1,
2345+
extra_closure_args=options.closure_args)
23422346
save_intermediate('closure')
23432347

23442348
log_time('js opts')
@@ -2462,6 +2466,12 @@ def check_bad_eq(arg):
24622466
options.llvm_lto = int(newargs[i + 1])
24632467
newargs[i] = ''
24642468
newargs[i + 1] = ''
2469+
elif newargs[i].startswith('--closure-args'):
2470+
check_bad_eq(newargs[i])
2471+
args = newargs[i + 1]
2472+
options.closure_args += shlex.split(args)
2473+
newargs[i] = ''
2474+
newargs[i + 1] = ''
24652475
elif newargs[i].startswith('--closure'):
24662476
check_bad_eq(newargs[i])
24672477
options.use_closure_compiler = int(newargs[i + 1])
@@ -2923,7 +2933,8 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target,
29232933
save_intermediate_with_wasm('postclean', wasm_binary_target)
29242934

29252935
def run_closure_compiler(final):
2926-
final = shared.Building.closure_compiler(final, pretty=not optimizer.minify_whitespace)
2936+
final = shared.Building.closure_compiler(final, pretty=not optimizer.minify_whitespace,
2937+
extra_closure_args=options.closure_args)
29272938
save_intermediate_with_wasm('closure', wasm_binary_target)
29282939
return final
29292940

‎site/build/text/docs/tools_reference/emcc.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,15 @@ Options that are modified or new in *emcc* are listed below:
306306
* Closure is only run if JavaScript opts are being done ("-O2"
307307
or above, or "--js-opts 1").
308308

309+
"--closure-args <args>"
310+
Specifies extra arguments to pass to Closure compiler. Use quotes
311+
to escape multiple arguments to be passed, e.g.
312+
313+
--closure-args "--externs myExterns.js"
314+
315+
Multiple passed --closure-args directives will be concatenated
316+
left to right.
317+
309318
"--pre-js <file>"
310319
Specify a file whose contents are added before the emitted code and
311320
optimized together with it. Note that this might not literally be

‎tests/test_closure_annotations.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/**
2+
* @suppress {duplicate, undefinedVars}
3+
*/
4+
var someExtern;

‎tests/test_closure_externs.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
var someExtern;

‎tests/test_closure_externs_pre_js.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
someExtern.var = 42;

‎tests/test_other.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8530,6 +8530,10 @@ def test_closure_full_js_library(self):
85308530
with env_modify({'EMCC_CLOSURE_ARGS': '--jscomp_off undefinedVars'}):
85318531
run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-O2', '--closure', '1', '-g1', '-s', 'INCLUDE_FULL_LIBRARY=1', '-s', 'ERROR_ON_UNDEFINED_SYMBOLS=0'])
85328532

8533+
# Tests --closure-args command line flag
8534+
def test_closure_externs(self):
8535+
run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '--closure', '1', '--pre-js', path_from_root('tests', 'test_closure_externs_pre_js.js'), '--closure-args', '--externs ' + path_from_root('tests', 'test_closure_externs.js')])
8536+
85338537
def test_toolchain_profiler(self):
85348538
environ = os.environ.copy()
85358539
environ['EM_PROFILE_TOOLCHAIN'] = '1'

‎tools/js_optimizer.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ def run_on_chunk(command):
297297
raise Exception()
298298

299299

300-
def run_on_js(filename, passes, js_engine, source_map=False, extra_info=None, just_split=False, just_concat=False):
300+
def run_on_js(filename, passes, js_engine, source_map=False, extra_info=None, just_split=False, just_concat=False, extra_closure_args=[]):
301301
with ToolchainProfiler.profile_block('js_optimizer.split_markers'):
302302
if not isinstance(passes, list):
303303
passes = [passes]
@@ -492,7 +492,8 @@ def write_chunk(chunk, i):
492492
if closure:
493493
if DEBUG:
494494
print('running closure on shell code', file=sys.stderr)
495-
cld = shared.Building.closure_compiler(cld, pretty='minifyWhitespace' not in passes)
495+
cld = shared.Building.closure_compiler(cld, pretty='minifyWhitespace' not in passes,
496+
extra_closure_args=extra_closure_args)
496497
temp_files.note(cld)
497498
elif cleanup:
498499
if DEBUG:
@@ -562,14 +563,14 @@ def write_chunk(chunk, i):
562563
return filename
563564

564565

565-
def run(filename, passes, js_engine=shared.NODE_JS, source_map=False, extra_info=None, just_split=False, just_concat=False):
566+
def run(filename, passes, js_engine=shared.NODE_JS, source_map=False, extra_info=None, just_split=False, just_concat=False, extra_closure_args=[]):
566567
if 'receiveJSON' in passes:
567568
just_split = True
568569
if 'emitJSON' in passes:
569570
just_concat = True
570571
js_engine = shared.listify(js_engine)
571572
with ToolchainProfiler.profile_block('js_optimizer.run_on_js'):
572-
return run_on_js(filename, passes, js_engine, source_map, extra_info, just_split, just_concat)
573+
return run_on_js(filename, passes, js_engine, source_map, extra_info, just_split, just_concat, extra_closure_args)
573574

574575

575576
def main():

‎tools/shared.py‎

Lines changed: 46 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2309,9 +2309,9 @@ def opt_level_to_str(opt_level, shrink_level=0):
23092309
return '-O' + str(min(opt_level, 3))
23102310

23112311
@staticmethod
2312-
def js_optimizer(filename, passes, debug=False, extra_info=None, output_filename=None, just_split=False, just_concat=False):
2312+
def js_optimizer(filename, passes, debug=False, extra_info=None, output_filename=None, just_split=False, just_concat=False, extra_closure_args=[]):
23132313
from . import js_optimizer
2314-
ret = js_optimizer.run(filename, passes, NODE_JS, debug, extra_info, just_split, just_concat)
2314+
ret = js_optimizer.run(filename, passes, NODE_JS, debug, extra_info, just_split, just_concat, extra_closure_args)
23152315
if output_filename:
23162316
safe_move(ret, output_filename)
23172317
ret = output_filename
@@ -2417,62 +2417,26 @@ def calculate_reachable_functions(infile, initial_list, can_reach=True):
24172417
return {'reachable': list(advised), 'total_funcs': len(can_call)}
24182418

24192419
@staticmethod
2420-
def closure_compiler(filename, pretty=True, advanced=True):
2420+
def closure_compiler(filename, pretty=True, advanced=True, extra_closure_args=[]):
24212421
with ToolchainProfiler.profile_block('closure_compiler'):
24222422
if not check_closure_compiler():
24232423
logger.error('Cannot run closure compiler')
24242424
raise Exception('closure compiler check failed')
24252425

24262426
# Closure annotations file contains suppressions and annotations to different symbols
2427-
CLOSURE_ANNOTATIONS = ['--js', path_from_root('src', 'closure-annotations.js')]
2427+
CLOSURE_ANNOTATIONS = [path_from_root('src', 'closure-annotations.js')]
24282428

24292429
if not Settings.ASMFS:
24302430
# If we have filesystem disabled, tell Closure not to bark when there are syscalls emitted that still reference the nonexisting FS object.
24312431
if Settings.FILESYSTEM:
2432-
CLOSURE_ANNOTATIONS += ['--js', path_from_root('src', 'closure-defined-fs-annotation.js')]
2432+
CLOSURE_ANNOTATIONS += [path_from_root('src', 'closure-defined-fs-annotation.js')]
24332433
else:
2434-
CLOSURE_ANNOTATIONS += ['--js', path_from_root('src', 'closure-undefined-fs-annotation.js')]
2434+
CLOSURE_ANNOTATIONS += [path_from_root('src', 'closure-undefined-fs-annotation.js')]
24352435

24362436
# Closure externs file contains known symbols to be extern to the minification, Closure
24372437
# should not minify these symbol names.
2438-
CLOSURE_EXTERNS = path_from_root('src', 'closure-externs.js')
2439-
NODE_EXTERNS_BASE = path_from_root('third_party', 'closure-compiler', 'node-externs')
2440-
NODE_EXTERNS = os.listdir(NODE_EXTERNS_BASE)
2441-
NODE_EXTERNS = [os.path.join(NODE_EXTERNS_BASE, name) for name in NODE_EXTERNS
2442-
if name.endswith('.js')]
2443-
NODE_EXTERNS = [path_from_root('src', 'node-externs.js')] + NODE_EXTERNS
2444-
V8_EXTERNS = [path_from_root('src', 'v8-externs.js')]
2445-
SPIDERMONKEY_EXTERNS = [path_from_root('src', 'spidermonkey-externs.js')]
2446-
BROWSER_EXTERNS_BASE = path_from_root('third_party', 'closure-compiler', 'browser-externs')
2447-
BROWSER_EXTERNS = os.listdir(BROWSER_EXTERNS_BASE)
2448-
BROWSER_EXTERNS = [os.path.join(BROWSER_EXTERNS_BASE, name) for name in BROWSER_EXTERNS
2449-
if name.endswith('.js')]
2438+
CLOSURE_EXTERNS = [path_from_root('src', 'closure-externs.js')]
24502439

2451-
# Something like this (adjust memory as needed):
2452-
# java -Xmx1024m -jar CLOSURE_COMPILER --compilation_level ADVANCED_OPTIMIZATIONS --variable_map_output_file src.cpp.o.js.vars --js src.cpp.o.js --js_output_file src.cpp.o.cc.js
2453-
outfile = filename + '.cc.js'
2454-
args = [JAVA,
2455-
'-Xmx' + (os.environ.get('JAVA_HEAP_SIZE') or '1024m'), # if you need a larger Java heap, use this environment variable
2456-
'-jar', CLOSURE_COMPILER,
2457-
'--compilation_level', 'ADVANCED_OPTIMIZATIONS' if advanced else 'SIMPLE_OPTIMIZATIONS',
2458-
'--language_in', 'ECMASCRIPT5']
2459-
if advanced:
2460-
args += CLOSURE_ANNOTATIONS
2461-
args += ['--externs', CLOSURE_EXTERNS]
2462-
args += ['--js_output_file', outfile]
2463-
2464-
if Settings.target_environment_may_be('node'):
2465-
for extern in NODE_EXTERNS:
2466-
args.append('--externs')
2467-
args.append(extern)
2468-
if Settings.target_environment_may_be('shell'):
2469-
for extern in V8_EXTERNS + SPIDERMONKEY_EXTERNS:
2470-
args.append('--externs')
2471-
args.append(extern)
2472-
if Settings.target_environment_may_be('web') or Settings.target_environment_may_be('worker'):
2473-
for extern in BROWSER_EXTERNS:
2474-
args.append('--externs')
2475-
args.append(extern)
24762440
# Closure compiler needs to know about all exports that come from the asm.js/wasm module, because to optimize for small code size,
24772441
# the exported symbols are added to global scope via a foreach loop in a way that evades Closure's static analysis. With an explicit
24782442
# externs file for the exports, Closure is able to reason about the exports.
@@ -2483,14 +2447,51 @@ def closure_compiler(filename, pretty=True, advanced=True):
24832447
exports_file.write(module_exports_suppressions.encode())
24842448
exports_file.close()
24852449

2486-
args.append('--externs')
2487-
args.append(exports_file.name)
2450+
CLOSURE_EXTERNS += [exports_file.name]
2451+
2452+
# Node.js specific externs
2453+
if Settings.target_environment_may_be('node'):
2454+
NODE_EXTERNS_BASE = path_from_root('third_party', 'closure-compiler', 'node-externs')
2455+
NODE_EXTERNS = os.listdir(NODE_EXTERNS_BASE)
2456+
NODE_EXTERNS = [os.path.join(NODE_EXTERNS_BASE, name) for name in NODE_EXTERNS
2457+
if name.endswith('.js')]
2458+
CLOSURE_EXTERNS += [path_from_root('src', 'node-externs.js')] + NODE_EXTERNS
2459+
2460+
# V8/SpiderMonkey shell specific externs
2461+
if Settings.target_environment_may_be('shell'):
2462+
V8_EXTERNS = [path_from_root('src', 'v8-externs.js')]
2463+
SPIDERMONKEY_EXTERNS = [path_from_root('src', 'spidermonkey-externs.js')]
2464+
CLOSURE_EXTERNS += V8_EXTERNS + SPIDERMONKEY_EXTERNS
2465+
2466+
# Web environment specific externs
2467+
if Settings.target_environment_may_be('web') or Settings.target_environment_may_be('worker'):
2468+
BROWSER_EXTERNS_BASE = path_from_root('third_party', 'closure-compiler', 'browser-externs')
2469+
BROWSER_EXTERNS = os.listdir(BROWSER_EXTERNS_BASE)
2470+
BROWSER_EXTERNS = [os.path.join(BROWSER_EXTERNS_BASE, name) for name in BROWSER_EXTERNS
2471+
if name.endswith('.js')]
2472+
CLOSURE_EXTERNS += BROWSER_EXTERNS
2473+
2474+
# Something like this (adjust memory as needed):
2475+
# java -Xmx1024m -jar CLOSURE_COMPILER --compilation_level ADVANCED_OPTIMIZATIONS --variable_map_output_file src.cpp.o.js.vars --js src.cpp.o.js --js_output_file src.cpp.o.cc.js
2476+
outfile = filename + '.cc.js'
2477+
args = [JAVA,
2478+
'-Xmx' + (os.environ.get('JAVA_HEAP_SIZE') or '1024m'), # if you need a larger Java heap, use this environment variable
2479+
'-jar', CLOSURE_COMPILER,
2480+
'--compilation_level', 'ADVANCED_OPTIMIZATIONS' if advanced else 'SIMPLE_OPTIMIZATIONS',
2481+
'--language_in', 'ECMASCRIPT5']
2482+
for a in CLOSURE_ANNOTATIONS:
2483+
args += ['--js', a]
2484+
for e in CLOSURE_EXTERNS:
2485+
args += ['--externs', e]
2486+
args += ['--js_output_file', outfile]
2487+
24882488
if Settings.IGNORE_CLOSURE_COMPILER_ERRORS:
24892489
args.append('--jscomp_off=*')
24902490
if pretty:
24912491
args += ['--formatting', 'PRETTY_PRINT']
24922492
if os.environ.get('EMCC_CLOSURE_ARGS'):
24932493
args += shlex.split(os.environ.get('EMCC_CLOSURE_ARGS'))
2494+
args += extra_closure_args
24942495
args += ['--js', filename]
24952496
logger.debug('closure compiler: ' + ' '.join(args))
24962497
proc = run_process(args, stderr=PIPE, check=False)

0 commit comments

Comments
 (0)