summaryrefslogtreecommitdiff
path: root/config/expandlibs_exec.py
blob: c05343022959ad48bebb2eac7d566f0a64ea71d4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

'''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
a given command line, and executes that command line with the expanded
arguments.

With the --extract argument (useful for e.g. $(AR)), it extracts object files
from static libraries (or use those listed in library descriptors directly).

With the --uselist argument (useful for e.g. $(CC)), it replaces all object
files with a list file. This can be used to avoid limitations in the length
of a command line. The kind of list file format used depends on the
EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
or 'linkerscript' for GNU ld linker scripts.
See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.

With the --symbol-order argument, followed by a file name, it will add the
relevant linker options to change the order in which the linker puts the
symbols appear in the resulting binary. Only works for ELF targets.
'''
from __future__ import with_statement
import sys
import os
from expandlibs import (
    ExpandArgs,
    relativize,
    isDynamicLib,
    isObject,
)
import expandlibs_config as conf
from optparse import OptionParser
import subprocess
import tempfile
import shutil
import subprocess
import re
from mozbuild.makeutil import Makefile

# The are the insert points for a GNU ld linker script, assuming a more
# or less "standard" default linker script. This is not a dict because
# order is important.
SECTION_INSERT_BEFORE = [
  ('.text', '.fini'),
  ('.rodata', '.rodata1'),
  ('.data.rel.ro', '.dynamic'),
  ('.data', '.data1'),
]

class ExpandArgsMore(ExpandArgs):
    ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
    def __enter__(self):
        self.tmp = []
        return self
        
    def __exit__(self, type, value, tb):
        '''Automatically remove temporary files'''
        for tmp in self.tmp:
            if os.path.isdir(tmp):
                shutil.rmtree(tmp, True)
            else:
                os.remove(tmp)

    def extract(self):
        self[0:] = self._extract(self)

    def _extract(self, args):
        '''When a static library name is found, either extract its contents
        in a temporary directory or use the information found in the
        corresponding lib descriptor.
        '''
        ar_extract = conf.AR_EXTRACT.split()
        newlist = []

        def lookup(base, f):
            for root, dirs, files in os.walk(base):
                if f in files:
                    return os.path.join(root, f)

        for arg in args:
            if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
                if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
                    newlist += self._extract(self._expand_desc(arg))
                    continue
                elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
                    tmp = tempfile.mkdtemp(dir=os.curdir)
                    self.tmp.append(tmp)
                    if conf.AR == 'lib':
                        out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
                        files = out.splitlines()
                        # If lib -list returns a list full of dlls, it's an
                        # import lib.
                        if all(isDynamicLib(f) for f in files):
                            newlist += [arg]
                            continue
                        for f in files:
                            subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
                    else:
                        subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
                    objs = []
                    basedir = os.path.dirname(arg)
                    for root, dirs, files in os.walk(tmp):
                        for f in files:
                            if isObject(f):
                                # If the file extracted from the library also
                                # exists in the directory containing the
                                # library, or one of its subdirectories, use
                                # that instead.
                                maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f)
                                if maybe_obj:
                                    objs.append(relativize(maybe_obj))
                                else:
                                    objs.append(relativize(os.path.join(root, f)))
                    newlist += sorted(objs)
                    continue
            newlist += [arg]
        return newlist

    def makelist(self):
        '''Replaces object file names with a temporary list file, using a
        list format depending on the EXPAND_LIBS_LIST_STYLE variable
        '''
        objs = [o for o in self if isObject(o)]
        if not len(objs): return
        fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
        if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
            content = ['INPUT("%s")\n' % obj for obj in objs]
            ref = tmp
        elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
            content = ["%s\n" % obj for obj in objs]
            ref = "-Wl,-filelist," + tmp
        elif conf.EXPAND_LIBS_LIST_STYLE == "list":
            content = ["%s\n" % obj for obj in objs]
            ref = "@" + tmp
        else:
            os.close(fd)
            os.remove(tmp)
            return
        self.tmp.append(tmp)
        f = os.fdopen(fd, "w")
        f.writelines(content)
        f.close()
        idx = self.index(objs[0])
        newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
        self[0:] = newlist

    def _getFoldedSections(self):
        '''Returns a dict about folded sections.
        When section A and B are folded into section C, the dict contains:
        { 'A': 'C',
          'B': 'C',
          'C': ['A', 'B'] }'''
        if not conf.LD_PRINT_ICF_SECTIONS:
            return {}

        proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        (stdout, stderr) = proc.communicate()
        result = {}
        # gold's --print-icf-sections output looks like the following:
        # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
        # In terms of words, chances are this will change in the future,
        # especially considering "into" is misplaced. Splitting on quotes
        # seems safer.
        for l in stderr.split('\n'):
            quoted = l.split("'")
            if len(quoted) > 5 and quoted[1] != quoted[5]:
                result[quoted[1]] = [quoted[5]]
                if quoted[5] in result:
                    result[quoted[5]].append(quoted[1])
                else:
                    result[quoted[5]] = [quoted[1]]
        return result

    def _getOrderedSections(self, ordered_symbols):
        '''Given an ordered list of symbols, returns the corresponding list
        of sections following the order.'''
        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
        finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
        folded = self._getFoldedSections()
        sections = set()
        ordered_sections = []
        for symbol in ordered_symbols:
            symbol_sections = finder.getSections(symbol)
            all_symbol_sections = []
            for section in symbol_sections:
                if section in folded:
                    if isinstance(folded[section], str):
                        section = folded[section]
                    all_symbol_sections.append(section)
                    all_symbol_sections.extend(folded[section])
                else:
                    all_symbol_sections.append(section)
            for section in all_symbol_sections:
                if not section in sections:
                    ordered_sections.append(section)
                    sections.add(section)
        return ordered_sections

    def orderSymbols(self, order):
        '''Given a file containing a list of symbols, adds the appropriate
        argument to make the linker put the symbols in that order.'''
        with open(order) as file:
            sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
        split_sections = {}
        linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
        for s in sections:
            for linked_section in linked_sections:
                if s.startswith(linked_section):
                    if linked_section in split_sections:
                        split_sections[linked_section].append(s)
                    else:
                        split_sections[linked_section] = [s]
                    break
        content = []
        # Order is important
        linked_sections = [s for s in linked_sections if s in split_sections]

        if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
            option = '-Wl,--section-ordering-file,%s'
            content = sections
            for linked_section in linked_sections:
                content.extend(split_sections[linked_section])
                content.append('%s.*' % linked_section)
                content.append(linked_section)

        elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
            option = '-Wl,-T,%s'
            section_insert_before = dict(SECTION_INSERT_BEFORE)
            for linked_section in linked_sections:
                content.append('SECTIONS {')
                content.append('  %s : {' % linked_section)
                content.extend('    *(%s)' % s for s in split_sections[linked_section])
                content.append('  }')
                content.append('}')
                content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
        else:
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)

        fd, tmp = tempfile.mkstemp(dir=os.curdir)
        f = os.fdopen(fd, "w")
        f.write('\n'.join(content)+'\n')
        f.close()
        self.tmp.append(tmp)
        self.append(option % tmp)

class SectionFinder(object):
    '''Instances of this class allow to map symbol names to sections in
    object files.'''

    def __init__(self, objs):
        '''Creates an instance, given a list of object files.'''
        if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
            raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
        self.mapping = {}
        for obj in objs:
            if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
                raise Exception('%s is not an object nor a static library' % obj)
            for symbol, section in SectionFinder._getSymbols(obj):
                sym = SectionFinder._normalize(symbol)
                if sym in self.mapping:
                    if not section in self.mapping[sym]:
                        self.mapping[sym].append(section)
                else:
                    self.mapping[sym] = [section]

    def getSections(self, symbol):
        '''Given a symbol, returns a list of sections containing it or the
        corresponding thunks. When the given symbol is a thunk, returns the
        list of sections containing its corresponding normal symbol and the
        other thunks for that symbol.'''
        sym = SectionFinder._normalize(symbol)
        if sym in self.mapping:
            return self.mapping[sym]
        return []

    @staticmethod
    def _normalize(symbol):
        '''For normal symbols, return the given symbol. For thunks, return
        the corresponding normal symbol.'''
        if re.match('^_ZThn[0-9]+_', symbol):
            return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
        return symbol

    @staticmethod
    def _getSymbols(obj):
        '''Returns a list of (symbol, section) contained in the given object
        file.'''
        proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        (stdout, stderr) = proc.communicate()
        syms = []
        for line in stdout.splitlines():
            # Each line has the following format:
            # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
            tmp = line.split(' ',1)
            # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
            # We only need to consider cases where "<section>\t<length> <symbol>" is present,
            # and where the [FfO] flag is either F (function) or O (object).
            if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
                tmp = tmp[1][8:].split()
                # That gives us ["<section>","<length>", "<symbol>"]
                syms.append((tmp[-1], tmp[0]))
        return syms

def print_command(out, args):
    print >>out, "Executing: " + " ".join(args)
    for tmp in [f for f in args.tmp if os.path.isfile(f)]:
        print >>out, tmp + ":"
        with open(tmp) as file:
            print >>out, "".join(["    " + l for l in file.readlines()])
    out.flush()

def main(args, proc_callback=None):
    parser = OptionParser()
    parser.add_option("--extract", action="store_true", dest="extract",
        help="when a library has no descriptor file, extract it first, when possible")
    parser.add_option("--uselist", action="store_true", dest="uselist",
        help="use a list file for objects when executing a command")
    parser.add_option("--verbose", action="store_true", dest="verbose",
        help="display executed command and temporary files content")
    parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
        help="use the given list of symbols to order symbols in the resulting binary when using with a linker")

    (options, args) = parser.parse_args(args)

    with ExpandArgsMore(args) as args:
        if options.extract:
            args.extract()
        if options.symbol_order:
            args.orderSymbols(options.symbol_order)
        if options.uselist:
            args.makelist()

        if options.verbose:
            print_command(sys.stderr, args)
        try:
            proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
            if proc_callback:
                proc_callback(proc)
        except Exception, e:
            print >>sys.stderr, 'error: Launching', args, ':', e
            raise e
        (stdout, stderr) = proc.communicate()
        if proc.returncode and not options.verbose:
            print_command(sys.stderr, args)
        sys.stderr.write(stdout)
        sys.stderr.flush()
        if proc.returncode:
            return proc.returncode
        return 0

if __name__ == '__main__':
    exit(main(sys.argv[1:]))