Skip to content

Commit fdb2dca

Browse files
committed
Implement a file-based lock mechanism to access the Emscripten cache. Closes emscripten-core#3850. Also fix an issue from earlier commit where at least on Windows, calling pool.terminate / pool.join might not succeed, but instead throws an "Access is denied" exception.
1 parent 7ea0bf8 commit fdb2dca

4 files changed

Lines changed: 466 additions & 24 deletions

File tree

tools/cache.py

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import os.path, sys, shutil, time, logging
2-
3-
import tempfiles
2+
import tempfiles, filelock
43

54
# Permanent cache for dlmalloc and stdlibc++
65
class Cache:
@@ -17,15 +16,24 @@ def __init__(self, dirname=None, debug=False, use_subdir=True):
1716
self.dirname = dirname
1817
self.debug = debug
1918

19+
def reverse_chop(thestring, ending):
20+
if thestring.endswith(ending):
21+
return thestring[:-len(ending)]
22+
return thestring
23+
24+
self.filelock = filelock.FileLock(reverse_chop(reverse_chop(self.dirname, '/'), '\\') + '.lock')
25+
2026
def ensure(self):
21-
shared.safe_ensure_dirs(self.dirname)
27+
with self.filelock:
28+
shared.safe_ensure_dirs(self.dirname)
2229

2330
def erase(self):
24-
tempfiles.try_delete(self.dirname)
25-
try:
26-
open(self.dirname + '__last_clear', 'w').write('last clear: ' + time.asctime() + '\n')
27-
except Exception, e:
28-
print >> sys.stderr, 'failed to save last clear time: ', e
31+
with self.filelock:
32+
tempfiles.try_delete(self.dirname)
33+
try:
34+
open(self.dirname + '__last_clear', 'w').write('last clear: ' + time.asctime() + '\n')
35+
except Exception, e:
36+
print >> sys.stderr, 'failed to save last clear time: ', e
2937

3038
def get_path(self, shortname):
3139
return os.path.join(self.dirname, shortname)
@@ -35,18 +43,21 @@ def get_path(self, shortname):
3543
def get(self, shortname, creator, extension='.bc', what=None, force=False):
3644
if not shortname.endswith(extension): shortname += extension
3745
cachename = os.path.join(self.dirname, shortname)
38-
if os.path.exists(cachename) and not force:
39-
return cachename
40-
if what is None:
41-
if shortname.endswith(('.bc', '.so', '.a')): what = 'system library'
42-
else: what = 'system asset'
43-
message = 'generating ' + what + ': ' + shortname + '...'
44-
logging.warn(message)
45-
self.ensure()
46-
temp = creator()
47-
if temp != cachename:
48-
shutil.copyfile(temp, cachename)
49-
logging.warn(' '*len(message) + 'ok')
46+
47+
with self.filelock:
48+
if os.path.exists(cachename) and not force:
49+
return cachename
50+
if what is None:
51+
if shortname.endswith(('.bc', '.so', '.a')): what = 'system library'
52+
else: what = 'system asset'
53+
message = 'generating ' + what + ': ' + shortname + '...'
54+
logging.warn(message)
55+
self.ensure()
56+
temp = creator()
57+
if temp != cachename:
58+
shutil.copyfile(temp, cachename)
59+
logging.warn(' '*len(message) + 'ok')
60+
5061
return cachename
5162

5263
# Given a set of functions of form (ident, text), and a preferred chunk size,

tools/duplicate_function_eliminator.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,11 @@ def write_chunk(chunk, i):
247247
if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.))
248248
pool = multiprocessing.Pool(processes=cores)
249249
filenames = pool.map(run_on_chunk, commands, chunksize=1)
250-
pool.terminate()
251-
pool.join()
250+
try:
251+
pool.terminate()
252+
pool.join()
253+
except Exception, e:
254+
pass
252255
else:
253256
# We can't parallize, but still break into chunks to avoid uglify/node memory issues
254257
if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks))

0 commit comments

Comments
 (0)