1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import re
import shutil
from mozpack.executables import (
is_executable,
may_strip,
strip,
may_elfhack,
elfhack,
)
from mozpack.chrome.manifest import ManifestEntry
from io import BytesIO
from mozpack.errors import ErrorMessage
from mozpack.mozjar import JarReader
import mozpack.path
from collections import OrderedDict
class Dest(object):
'''
Helper interface for BaseFile.copy. The interface works as follows:
- read() and write() can be used to sequentially read/write from the
underlying file.
- a call to read() after a write() will re-open the underlying file and
read from it.
- a call to write() after a read() will re-open the underlying file,
emptying it, and write to it.
'''
def __init__(self, path):
self.path = path
self.mode = None
def read(self, length=-1):
if self.mode != 'r':
self.file = open(self.path, 'rb')
self.mode = 'r'
return self.file.read(length)
def write(self, data):
if self.mode != 'w':
self.file = open(self.path, 'wb')
self.mode = 'w'
return self.file.write(data)
def exists(self):
return os.path.exists(self.path)
def close(self):
if self.mode:
self.mode = None
self.file.close()
class BaseFile(object):
'''
Base interface and helper for file copying. Derived class may implement
their own copy function, or rely on BaseFile.copy using the open() member
function and/or the path property.
'''
def copy(self, dest, skip_if_older=True):
'''
Copy the BaseFile content to the destination given as a string or a
Dest instance. Avoids replacing existing files if the BaseFile content
matches that of the destination, or in case of plain files, if the
destination is newer than the original file. This latter behaviour is
disabled when skip_if_older is False.
Returns whether a copy was actually performed (True) or not (False).
'''
if isinstance(dest, basestring):
dest = Dest(dest)
else:
assert isinstance(dest, Dest)
can_skip_content_check = False
if not dest.exists():
can_skip_content_check = True
elif getattr(self, 'path', None) and getattr(dest, 'path', None):
# os.path.getmtime returns a result in seconds with precision up to
# the microsecond. But microsecond is too precise because
# shutil.copystat only copies milliseconds, and seconds is not
# enough precision.
if skip_if_older and int(os.path.getmtime(self.path) * 1000) \
<= int(os.path.getmtime(dest.path) * 1000):
return False
elif os.path.getsize(self.path) != os.path.getsize(dest.path):
can_skip_content_check = True
if can_skip_content_check:
if getattr(self, 'path', None) and getattr(dest, 'path', None):
shutil.copy2(self.path, dest.path)
else:
# Ensure the file is always created
if not dest.exists():
dest.write('')
shutil.copyfileobj(self.open(), dest)
return True
src = self.open()
copy_content = ''
while True:
dest_content = dest.read(32768)
src_content = src.read(32768)
copy_content += src_content
if len(dest_content) == len(src_content) == 0:
break
# If the read content differs between origin and destination,
# write what was read up to now, and copy the remainder.
if dest_content != src_content:
dest.write(copy_content)
shutil.copyfileobj(src, dest)
break
if hasattr(self, 'path') and hasattr(dest, 'path'):
shutil.copystat(self.path, dest.path)
return True
def open(self):
'''
Return a file-like object allowing to read() the content of the
associated file. This is meant to be overloaded in subclasses to return
a custom file-like object.
'''
assert self.path is not None
return open(self.path, 'rb')
class File(BaseFile):
'''
File class for plain files.
'''
def __init__(self, path):
self.path = path
class ExecutableFile(File):
'''
File class for executable and library files on OS/X and ELF systems.
(see mozpack.executables.is_executable documentation).
'''
def copy(self, dest, skip_if_older=True):
assert isinstance(dest, basestring)
# If File.copy didn't actually copy because dest is newer, check the
# file sizes. If dest is smaller, it means it is already stripped and
# elfhacked, so we can skip.
if not File.copy(self, dest, skip_if_older) and \
os.path.getsize(self.path) > os.path.getsize(dest):
return False
try:
if may_strip(dest):
strip(dest)
if may_elfhack(dest):
elfhack(dest)
except ErrorMessage:
os.remove(dest)
raise
return True
class GeneratedFile(BaseFile):
'''
File class for content with no previous existence on the filesystem.
'''
def __init__(self, content):
self.content = content
def open(self):
return BytesIO(self.content)
class DeflatedFile(BaseFile):
'''
File class for members of a jar archive. DeflatedFile.copy() effectively
extracts the file from the jar archive.
'''
def __init__(self, file):
from mozpack.mozjar import JarFileReader
assert isinstance(file, JarFileReader)
self.file = file
def open(self):
self.file.seek(0)
return self.file
class XPTFile(GeneratedFile):
'''
File class for a linked XPT file. It takes several XPT files as input
(using the add() and remove() member functions), and links them at copy()
time.
'''
def __init__(self):
self._files = set()
def add(self, xpt):
'''
Add the given XPT file (as a BaseFile instance) to the list of XPTs
to link.
'''
assert isinstance(xpt, BaseFile)
self._files.add(xpt)
def remove(self, xpt):
'''
Remove the given XPT file (as a BaseFile instance) from the list of
XPTs to link.
'''
assert isinstance(xpt, BaseFile)
self._files.remove(xpt)
def copy(self, dest, skip_if_older=True):
'''
Link the registered XPTs and place the resulting linked XPT at the
destination given as a string or a Dest instance. Avoids an expensive
XPT linking if the interfaces in an existing destination match those of
the individual XPTs to link.
skip_if_older is ignored.
'''
if isinstance(dest, basestring):
dest = Dest(dest)
assert isinstance(dest, Dest)
from xpt import xpt_link, Typelib, Interface
all_typelibs = [Typelib.read(f.open()) for f in self._files]
if dest.exists():
# Typelib.read() needs to seek(), so use a BytesIO for dest
# content.
dest_interfaces = \
dict((i.name, i)
for i in Typelib.read(BytesIO(dest.read())).interfaces
if i.iid != Interface.UNRESOLVED_IID)
identical = True
for f in self._files:
typelib = Typelib.read(f.open())
for i in typelib.interfaces:
if i.iid != Interface.UNRESOLVED_IID and \
not (i.name in dest_interfaces and
i == dest_interfaces[i.name]):
identical = False
break
if identical:
return False
s = BytesIO()
xpt_link(all_typelibs).write(s)
dest.write(s.getvalue())
return True
def open(self):
raise RuntimeError("Unsupported")
def isempty(self):
'''
Return whether there are XPT files to link.
'''
return len(self._files) == 0
class ManifestFile(BaseFile):
'''
File class for a manifest file. It takes individual manifest entries (using
the add() and remove() member functions), and adjusts them to be relative
to the base path for the manifest, given at creation.
Example:
There is a manifest entry "content webapprt webapprt/content/" relative
to "webapprt/chrome". When packaging, the entry will be stored in
jar:webapprt/omni.ja!/chrome/chrome.manifest, which means the entry
will have to be relative to "chrome" instead of "webapprt/chrome". This
doesn't really matter when serializing the entry, since this base path
is not written out, but it matters when moving the entry at the same
time, e.g. to jar:webapprt/omni.ja!/chrome.manifest, which we don't do
currently but could in the future.
'''
def __init__(self, base, entries=None):
self._entries = entries if entries else []
self._base = base
def add(self, entry):
'''
Add the given entry to the manifest. Entries are rebased at open() time
instead of add() time so that they can be more easily remove()d.
'''
assert isinstance(entry, ManifestEntry)
self._entries.append(entry)
def remove(self, entry):
'''
Remove the given entry from the manifest.
'''
assert isinstance(entry, ManifestEntry)
self._entries.remove(entry)
def open(self):
'''
Return a file-like object allowing to read() the serialized content of
the manifest.
'''
return BytesIO(''.join('%s\n' % e.rebase(self._base)
for e in self._entries))
def __iter__(self):
'''
Iterate over entries in the manifest file.
'''
return iter(self._entries)
def isempty(self):
'''
Return whether there are manifest entries to write
'''
return len(self._entries) == 0
class MinifiedProperties(BaseFile):
'''
File class for minified properties. This wraps around a BaseFile instance,
and removes lines starting with a # from its content.
'''
def __init__(self, file):
assert isinstance(file, BaseFile)
self._file = file
def open(self):
'''
Return a file-like object allowing to read() the minified content of
the properties file.
'''
return BytesIO(''.join(l for l in self._file.open().readlines()
if not l.startswith('#')))
class BaseFinder(object):
def __init__(self, base, minify=False):
'''
Initializes the instance with a reference base directory. The
optional minify argument specifies whether file types supporting
minification (currently only "*.properties") should be minified.
'''
self.base = base
self._minify = minify
def find(self, pattern):
'''
Yield path, BaseFile_instance pairs for all files under the base
directory and its subdirectories that match the given pattern. See the
mozpack.path.match documentation for a description of the handled
patterns.
'''
while pattern.startswith('/'):
pattern = pattern[1:]
for p, f in self._find(pattern):
yield p, self._minify_file(p, f)
def __iter__(self):
'''
Iterates over all files under the base directory (excluding files
starting with a '.' and files at any level under a directory starting
with a '.').
for path, file in finder:
...
'''
return self.find('')
def __contains__(self, pattern):
raise RuntimeError("'in' operator forbidden for %s. Use contains()." %
self.__class__.__name__)
def contains(self, pattern):
'''
Return whether some files under the base directory match the given
pattern. See the mozpack.path.match documentation for a description of
the handled patterns.
'''
return any(self.find(pattern))
def _minify_file(self, path, file):
'''
Return an appropriate MinifiedSomething wrapper for the given BaseFile
instance (file), according to the file type (determined by the given
path), if the FileFinder was created with minification enabled.
Otherwise, just return the given BaseFile instance.
Currently, only "*.properties" files are handled.
'''
if self._minify and not isinstance(file, ExecutableFile):
if path.endswith('.properties'):
return MinifiedProperties(file)
return file
class FileFinder(BaseFinder):
'''
Helper to get appropriate BaseFile instances from the file system.
'''
def __init__(self, base, **kargs):
'''
Create a FileFinder for files under the given base directory.
'''
BaseFinder.__init__(self, base, **kargs)
def _find(self, pattern):
'''
Actual implementation of FileFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
Note all files with a name starting with a '.' are ignored when
scanning directories, but are not ignored when explicitely requested.
'''
if '*' in pattern:
return self._find_glob('', mozpack.path.split(pattern))
elif os.path.isdir(os.path.join(self.base, pattern)):
return self._find_dir(pattern)
else:
return self._find_file(pattern)
def _find_dir(self, path):
'''
Actual implementation of FileFinder.find() when the given pattern
corresponds to an existing directory under the base directory.
Ignores file names starting with a '.' under the given path. If the
path itself has leafs starting with a '.', they are not ignored.
'''
for p in os.listdir(os.path.join(self.base, path)):
if p.startswith('.'):
continue
for p_, f in self._find(mozpack.path.join(path, p)):
yield p_, f
def _find_file(self, path):
'''
Actual implementation of FileFinder.find() when the given pattern
corresponds to an existing file under the base directory.
'''
srcpath = os.path.join(self.base, path)
if not os.path.exists(srcpath):
return
if is_executable(srcpath):
yield path, ExecutableFile(srcpath)
else:
yield path, File(srcpath)
def _find_glob(self, base, pattern):
'''
Actual implementation of FileFinder.find() when the given pattern
contains globbing patterns ('*' or '**'). This is meant to be an
equivalent of:
for p, f in self:
if mozpack.path.match(p, pattern):
yield p, f
but avoids scanning the entire tree.
'''
if not pattern:
for p, f in self._find(base):
yield p, f
elif pattern[0] == '**':
for p, f in self._find(base):
if mozpack.path.match(p, mozpack.path.join(*pattern)):
yield p, f
elif '*' in pattern[0]:
if not os.path.exists(os.path.join(self.base, base)):
return
for p in os.listdir(os.path.join(self.base, base)):
if p.startswith('.') and not pattern[0].startswith('.'):
continue
if re.match(mozpack.path.translate(pattern[0]), p):
for p_, f in self._find_glob(mozpack.path.join(base, p),
pattern[1:]):
yield p_, f
else:
for p, f in self._find_glob(mozpack.path.join(base, pattern[0]),
pattern[1:]):
yield p, f
class JarFinder(BaseFinder):
'''
Helper to get appropriate DeflatedFile instances from a JarReader.
'''
def __init__(self, base, reader, **kargs):
'''
Create a JarFinder for files in the given JarReader. The base argument
is used as an indication of the Jar file location.
'''
assert isinstance(reader, JarReader)
BaseFinder.__init__(self, base, **kargs)
self._files = OrderedDict((f.filename, f) for f in reader)
def _find(self, pattern):
'''
Actual implementation of JarFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
'''
if '*' in pattern:
for p in self._files:
if mozpack.path.match(p, pattern):
yield p, DeflatedFile(self._files[p])
elif pattern == '':
for p in self._files:
yield p, DeflatedFile(self._files[p])
elif pattern in self._files:
yield pattern, DeflatedFile(self._files[pattern])
else:
for p in self._files:
if mozpack.path.basedir(p, [pattern]) == pattern:
yield p, DeflatedFile(self._files[p])
|