samba_deps.py 46.7 KB
Newer Older
1
# Samba automatic dependency handling and project rules
2

3
import os, sys, re
4

5
from waflib import Build, Options, Logs, Utils, Errors, Task
6
7
8
from waflib.Logs import debug
from waflib.Configure import conf
from waflib import ConfigSet
9

10
from samba_utils import LOCAL_CACHE, TO_LIST, get_tgt_list, unique_list
11
from samba_autoconf import library_flags
12
13
14
15
16
17
18
19
20

@conf
def ADD_GLOBAL_DEPENDENCY(ctx, dep):
    '''add a dependency for all binaries and libraries'''
    if not 'GLOBAL_DEPENDENCIES' in ctx.env:
        ctx.env.GLOBAL_DEPENDENCIES = []
    ctx.env.GLOBAL_DEPENDENCIES.append(dep)


21
22
23
24
25
26
@conf
def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
    '''indicate that circular dependencies between libraries should be broken.'''
    ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True


27
28
29
30
31
32
33
@conf
def SET_SYSLIB_DEPS(conf, target, deps):
    '''setup some implied dependencies for a SYSLIB'''
    cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
    cache[target] = deps


34
35
def expand_subsystem_deps(bld):
    '''expand the reverse dependencies resulting from subsystem
36
37
38
39
40
       attributes of modules. This is walking over the complete list
       of declared subsystems, and expands the samba_deps_extended list for any
       module<->subsystem dependencies'''

    subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
41
42
    targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')

43
44
45
    for subsystem_name in subsystem_list:
        bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
        type = targets[subsystem_name]
46
47
        if type == 'DISABLED' or type == 'EMPTY':
            continue
48

49
50
51
52
53
54
        # for example,
        #    subsystem_name = dcerpc_server (a subsystem)
        #    subsystem      = dcerpc_server (a subsystem object)
        #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
        #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)

55
        subsystem = bld.get_tgen_by_name(subsystem_name)
56
        bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
57
58
59
60
61
62
63
64
65
66
67
68
69
        for d in subsystem_list[subsystem_name]:
            module_name = d['TARGET']
            module_type = targets[module_name]
            if module_type in ['DISABLED', 'EMPTY']:
                continue
            bld.ASSERT(subsystem is not None,
                       "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
            if module_type in ['SUBSYSTEM']:
                # if a module is a plain object type (not a library) then the
                # subsystem it is part of needs to have it as a dependency, so targets
                # that depend on this subsystem get the modules of that subsystem
                subsystem.samba_deps_extended.append(module_name)
        subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
70
71
72
73
74
75
76



def build_dependencies(self):
    '''This builds the dependency list for a target. It runs after all the targets are declared

    The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
77
    the full dependency list for a target until we have all of the targets declared.
78
79
    '''

80
    if self.samba_type in ['LIBRARY', 'PLUGIN', 'BINARY', 'PYTHON']:
81
82
83
84
85
86
87
        self.uselib        = list(self.final_syslibs)
        self.uselib_local  = list(self.final_libs)
        self.add_objects   = list(self.final_objects)

        # extra link flags from pkg_config
        libs = self.final_syslibs.copy()

ita1024's avatar
ita1024 committed
88
        (cflags, ldflags, cpppath) = library_flags(self, list(libs))
89
        new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
90
91
92
        new_ldflags.extend(ldflags)
        self.ldflags       = new_ldflags

93
94
95
96
        if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
            for f in self.env.undefined_ldflags:
                self.ldflags.remove(f)

97
98
99
100
        if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
            for f in self.env.undefined_ignore_ldflags:
                self.ldflags.append(f)

101
102
103
        debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
              self.sname, self.uselib, self.uselib_local, self.add_objects)

Stefan Metzmacher's avatar
Stefan Metzmacher committed
104
    if self.samba_type in ['SUBSYSTEM', 'BUILTIN']:
ita1024's avatar
ita1024 committed
105
        # this is needed for the cflags of libs that come from pkg_config
106
107
108
        self.uselib = list(self.final_syslibs)
        self.uselib.extend(list(self.direct_syslibs))
        for lib in self.final_libs:
109
            t = self.bld.get_tgen_by_name(lib)
110
111
            self.uselib.extend(list(t.final_syslibs))
        self.uselib = unique_list(self.uselib)
112

113
114
    if getattr(self, 'uselib', None):
        up_list = []
115
116
117
        for l in self.uselib:
           up_list.append(l.upper())
        self.uselib = up_list
118

119

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def build_includes(self):
    '''This builds the right set of includes for a target.

    One tricky part of this is that the includes= attribute for a
    target needs to use paths which are relative to that targets
    declaration directory (which we can get at via t.path).

    The way this works is the includes list gets added as
    samba_includes in the main build task declaration. Then this
    function runs after all of the tasks are declared, and it
    processes the samba_includes attribute to produce a includes=
    attribute
    '''

    if getattr(self, 'samba_includes', None) is None:
        return

    bld = self.bld

139
    inc_deps = includes_objects(bld, self, set(), {})
140
141
142

    includes = []

143
    # maybe add local includes
144
    if getattr(self, 'local_include', True) and getattr(self, 'local_include_first', True):
145
146
        includes.append('.')

147
    includes.extend(self.samba_includes_extended)
148

149
    if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
150
151
152
153
        includes.extend(bld.env['EXTRA_INCLUDES'])

    includes.append('#')

154
155
    inc_set = set()
    inc_abs = []
156

157
    for d in inc_deps:
158
        t = bld.get_tgen_by_name(d)
159
        bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
160
        inclist = getattr(t, 'samba_includes_extended', [])[:]
161
        if getattr(t, 'local_include', True):
162
163
164
            inclist.append('.')
        if inclist == []:
            continue
165
        tpath = t.samba_abspath
166
        for inc in inclist:
167
168
169
170
171
172
173
            npath = tpath + '/' + inc
            if not npath in inc_set:
                inc_abs.append(npath)
                inc_set.add(npath)

    mypath = self.path.abspath(bld.env)
    for inc in inc_abs:
174
        relpath = os.path.relpath(inc, mypath)
175
        includes.append(relpath)
176

177
    if getattr(self, 'local_include', True) and not getattr(self, 'local_include_first', True):
178
179
        includes.append('.')

180
181
182
183
184
185
186
187
188
189
    # now transform the includes list to be relative to the top directory
    # which is represented by '#' in waf. This allows waf to cache the
    # includes lists more efficiently
    includes_top = []
    for i in includes:
        if i[0] == '#':
            # some are already top based
            includes_top.append(i)
            continue
        absinc = os.path.join(self.path.abspath(), i)
190
        relinc = os.path.relpath(absinc, self.bld.srcnode.abspath())
191
192
193
194
195
196
        includes_top.append('#' + relinc)

    self.includes = unique_list(includes_top)
    debug('deps: includes for target %s: includes=%s',
          self.sname, self.includes)

197

198
199
200
201
202
203
204
def add_init_functions(self):
    '''This builds the right set of init functions'''

    bld = self.bld

    subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')

205
206
207
208
209
    # cope with the separated object lists from BINARY and LIBRARY targets
    sname = self.sname
    if sname.endswith('.objlist'):
        sname = sname[0:-8]

210
    modules = []
211
212
    if sname in subsystems:
        modules.append(sname)
213
214
215
216
217
218
219
220
221

    m = getattr(self, 'samba_modules', None)
    if m is not None:
        modules.extend(TO_LIST(m))

    m = getattr(self, 'samba_subsystem', None)
    if m is not None:
        modules.append(m)

222
223
224
    if 'pyembed' in self.features:
        return

Simo Sorce's avatar
Simo Sorce committed
225
    sentinel = getattr(self, 'init_function_sentinel', 'NULL')
226

227
    targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
228
    cflags = getattr(self, 'samba_cflags', [])[:]
229
230

    if modules == []:
231
        sname = sname.replace('-','_')
232
        sname = sname.replace('.','_')
233
        sname = sname.replace('/','_')
Simo Sorce's avatar
Simo Sorce committed
234
235
        cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel))
        if sentinel == 'NULL':
236
237
            proto = "extern void __%s_dummy_module_proto(void)" % (sname)
            cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (sname, proto))
ita1024's avatar
ita1024 committed
238
        self.cflags = cflags
239
240
        return

241
242
    for m in modules:
        bld.ASSERT(m in subsystems,
243
244
245
                   "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
        init_fn_list = []
        for d in subsystems[m]:
246
247
248
            if targets[d['TARGET']] != 'DISABLED':
                init_fn_list.append(d['INIT_FUNCTION'])
        if init_fn_list == []:
Simo Sorce's avatar
Simo Sorce committed
249
250
            cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel))
            if sentinel == 'NULL':
251
252
                proto = "extern void __%s_dummy_module_proto(void)" % (m)
                cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
253
        else:
Simo Sorce's avatar
Simo Sorce committed
254
            cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel))
255
256
            proto = "".join('_MODULE_PROTO(%s)' % f for f in init_fn_list) +\
                    "extern void __%s_dummy_module_proto(void)" % (m)
257
            cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
ita1024's avatar
ita1024 committed
258
    self.cflags = cflags
259
260


261
def check_duplicate_sources(bld, tgt_list):
262
    '''see if we are compiling the same source file more than once'''
263
264
265

    debug('deps: checking for duplicate sources')
    targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
266
267

    for t in tgt_list:
268
        source_list = TO_LIST(getattr(t, 'source', ''))
269
        tpath = os.path.normpath(os.path.relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
270
        obj_sources = set()
271
        for s in source_list:
ita1024's avatar
ita1024 committed
272
273
274
            if not isinstance(s, str):
                print('strange path in check_duplicate_sources %r' % s)
                s = s.abspath()
275
276
277
278
279
280
281
282
            p = os.path.normpath(os.path.join(tpath, s))
            if p in obj_sources:
                Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
                sys.exit(1)
            obj_sources.add(p)
        t.samba_source_set = obj_sources

    subsystems = {}
283

284
    # build a list of targets that each source file is part of
285
    for t in tgt_list:
286
        if not targets[t.sname] in [ 'LIBRARY', 'PLUGIN', 'BINARY', 'PYTHON' ]:
287
288
            continue
        for obj in t.add_objects:
289
            t2 = t.bld.get_tgen_by_name(obj)
290
            source_set = getattr(t2, 'samba_source_set', set())
291
292
293
294
295
296
297
298
299
300
301
302
            for s in source_set:
                if not s in subsystems:
                    subsystems[s] = {}
                if not t.sname in subsystems[s]:
                    subsystems[s][t.sname] = []
                subsystems[s][t.sname].append(t2.sname)

    for s in subsystems:
        if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
            Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
        for tname in subsystems[s]:
            if len(subsystems[s][tname]) > 1:
303
                raise Errors.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
304

305
    return True
306

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
def check_group_ordering(bld, tgt_list):
    '''see if we have any dependencies that violate the group ordering

    It is an error for a target to depend on a target from a later
    build group
    '''

    def group_name(g):
        tm = bld.task_manager
        return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]

    for g in bld.task_manager.groups:
        gname = group_name(g)
        for t in g.tasks_gen:
            t.samba_group = gname

    grp_map = {}
    idx = 0
    for g in bld.task_manager.groups:
        name = group_name(g)
        grp_map[name] = idx
        idx += 1

    targets = LOCAL_CACHE(bld, 'TARGET_TYPE')

    ret = True
    for t in tgt_list:
        tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
        for d in tdeps:
336
            t2 = bld.get_tgen_by_name(d)
337
338
339
340
341
342
343
344
345
346
347
            if t2 is None:
                continue
            map1 = grp_map[t.samba_group]
            map2 = grp_map[t2.samba_group]

            if map2 > map1:
                Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
                           t.sname, t.samba_group, t2.sname, t2.samba_group))
                ret = False

    return ret
348
Build.BuildContext.check_group_ordering = check_group_ordering
349

350
351
352
353
def show_final_deps(bld, tgt_list):
    '''show the final dependencies for all targets'''

    targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
354

355
    for t in tgt_list:
356
        if not targets[t.sname] in ['LIBRARY', 'PLUGIN', 'BINARY', 'PYTHON', 'SUBSYSTEM', 'BUILTIN']:
357
            continue
358
        debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
359
              t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375


def add_samba_attributes(bld, tgt_list):
    '''ensure a target has a the required samba attributes'''

    targets = LOCAL_CACHE(bld, 'TARGET_TYPE')

    for t in tgt_list:
        if t.name != '':
            t.sname = t.name
        else:
            t.sname = t.target
        t.samba_type = targets[t.sname]
        t.samba_abspath = t.path.abspath(bld.env)
        t.samba_deps_extended = t.samba_deps[:]
        t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
ita1024's avatar
ita1024 committed
376
        t.cflags = getattr(t, 'samba_cflags', '')
377

Stefan Metzmacher's avatar
Stefan Metzmacher committed
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def replace_builtin_subsystem_deps(bld, tgt_list):
    '''replace dependencies based on builtin subsystems/libraries

    '''

    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')

    # If either the target or the dependency require builtin linking
    # we should replace the dependency
    for t in tgt_list:
        t_require_builtin_deps = getattr(t, 'samba_require_builtin_deps', False)
        if t_require_builtin_deps:
            debug("deps: target %s: requires builtin dependencies..." % (t.sname))
        else:
            debug("deps: target %s: does not require builtin dependencies..." % (t.sname))

        replacing = {}

        for dep in t.samba_deps_extended:
            bld.ASSERT(dep in targets, "target %s: dependency target %s not declared" % (t.sname, dep))
            dtype = targets[dep]
            bld.ASSERT(dtype != 'BUILTIN', "target %s: dependency target %s is BUILTIN" % (t.sname, dep))
            bld.ASSERT(dtype != 'PLUGIN', "target %s: dependency target %s is PLUGIN" % (t.sname, dep))
            if dtype not in ['SUBSYSTEM', 'LIBRARY']:
                debug("deps: target %s: keep %s dependency %s" % (t.sname, dtype, dep))
                continue
            dt = bld.get_tgen_by_name(dep)
            bld.ASSERT(dt is not None, "target %s: dependency target %s not found by name" % (t.sname, dep))
            dt_require_builtin_deps = getattr(dt, 'samba_require_builtin_deps', False)
            if not dt_require_builtin_deps and not t_require_builtin_deps:
                # both target and dependency don't require builtin linking
                continue
            sdt = getattr(dt, 'samba_builtin_subsystem', None)
            if not t_require_builtin_deps:
                if sdt is None:
                    debug("deps: target %s: dependency %s requires builtin deps only" % (t.sname, dep))
                    continue
                debug("deps: target %s: dependency %s requires builtin linking" % (t.sname, dep))
            bld.ASSERT(sdt is not None, "target %s: dependency target %s is missing samba_builtin_subsystem" % (t.sname, dep))
            sdep = sdt.sname
            bld.ASSERT(sdep in targets, "target %s: builtin dependency target %s (from %s) not declared" % (t.sname, sdep, dep))
            sdt = targets[sdep]
            bld.ASSERT(sdt == 'BUILTIN', "target %s: builtin dependency target %s (from %s) is not BUILTIN" % (t.sname, sdep, dep))
            replacing[dep] = sdep

        for i in range(len(t.samba_deps_extended)):
            dep = t.samba_deps_extended[i]
            if dep in replacing:
                sdep = replacing[dep]
                debug("deps: target %s: replacing dependency %s with builtin subsystem %s" % (t.sname, dep, sdep))
                t.samba_deps_extended[i] = sdep

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
def replace_grouping_libraries(bld, tgt_list):
    '''replace dependencies based on grouping libraries

    If a library is marked as a grouping library, then any target that
    depends on a subsystem that is part of that grouping library gets
    that dependency replaced with a dependency on the grouping library
    '''

    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')

    grouping = {}

    # find our list of grouping libraries, mapped from the subsystems they depend on
    for t in tgt_list:
        if not getattr(t, 'grouping_library', False):
            continue
        for dep in t.samba_deps_extended:
447
            bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
448
449
450
451
452
453
454
455
456
457
458
459
460
            if targets[dep] == 'SUBSYSTEM':
                grouping[dep] = t.sname

    # now replace any dependencies on elements of grouping libraries
    for t in tgt_list:
        for i in range(len(t.samba_deps_extended)):
            dep = t.samba_deps_extended[i]
            if dep in grouping:
                if t.sname != grouping[dep]:
                    debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
                    t.samba_deps_extended[i] = grouping[dep]


461

462
463
464
465
def build_direct_deps(bld, tgt_list):
    '''build the direct_objects and direct_libs sets for each target'''

    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
466
    syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
467

468
    global_deps = bld.env.GLOBAL_DEPENDENCIES
469
470
    global_deps_exclude = set()
    for dep in global_deps:
471
        t = bld.get_tgen_by_name(dep)
472
473
474
475
        for d in t.samba_deps:
            # prevent loops from the global dependencies list
            global_deps_exclude.add(d)
            global_deps_exclude.add(d + '.objlist')
476
477
478
479
480

    for t in tgt_list:
        t.direct_objects = set()
        t.direct_libs = set()
        t.direct_syslibs = set()
481
        deps = t.samba_deps_extended[:]
482
        if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
483
            deps.extend(global_deps)
484
        for d in deps:
485
            if d == t.sname: continue
486
            if not d in targets:
487
                Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
488
                sys.exit(1)
489
490
            if targets[d] in [ 'EMPTY', 'DISABLED' ]:
                continue
491
492
493
            if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
                # this check should be more restrictive, but for now we have pidl-generated python
                # code that directly depends on other python modules
494
495
                Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
                sys.exit(1)
496
497
            if targets[d] == 'SYSLIB':
                t.direct_syslibs.add(d)
498
499
                if d in syslib_deps:
                    for implied in TO_LIST(syslib_deps[d]):
Stefan Metzmacher's avatar
Stefan Metzmacher committed
500
501
502
503
504
505
                        if targets[implied] == 'SUBSYSTEM':
                            it = bld.get_tgen_by_name(implied)
                            sit = getattr(it, 'samba_builtin_subsystem', None)
                            if sit:
                                implied = sit.sname
                        if targets[implied] == 'BUILTIN':
506
                            t.direct_objects.add(implied)
507
508
509
                        elif targets[implied] == 'SYSLIB':
                            t.direct_syslibs.add(implied)
                        elif targets[implied] in ['LIBRARY', 'MODULE']:
510
                            t.direct_libs.add(implied)
511
512
513
514
                        else:
                            Logs.error('Implied dependency %s in %s is of type %s' % (
                                implied, t.sname, targets[implied]))
                            sys.exit(1)
515
                continue
516
            t2 = bld.get_tgen_by_name(d)
517
            if t2 is None:
518
519
                Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
                sys.exit(1)
520
521
            if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
                t.direct_libs.add(d)
Stefan Metzmacher's avatar
Stefan Metzmacher committed
522
            elif t2.samba_type in [ 'SUBSYSTEM', 'BUILTIN', 'ASN1', 'PYTHON' ]:
523
                t.direct_objects.add(d)
524
525
526
527
            elif t2.samba_type in [ 'PLUGIN' ]:
                Logs.error('Implicit dependency %s in %s is of type %s' % (
                           d, t.sname, t2.samba_type))
                sys.exit(1)
Stefan Metzmacher's avatar
Stefan Metzmacher committed
528

529
530
531
    debug('deps: built direct dependencies')


532
533
534
535
536
537
538
539
540
def dependency_loop(loops, t, target):
    '''add a dependency loop to the loops dictionary'''
    if t.sname == target:
        return
    if not target in loops:
        loops[target] = set()
    if not t.sname in loops[target]:
        loops[target].add(t.sname)

541

542
def indirect_libs(bld, t, chain, loops):
543
544
545
546
547
548
549
550
551
552
553
554
555
    '''recursively calculate the indirect library dependencies for a target

    An indirect library is a library that results from a dependency on
    a subsystem
    '''

    ret = getattr(t, 'indirect_libs', None)
    if ret is not None:
        return ret

    ret = set()
    for obj in t.direct_objects:
        if obj in chain:
556
            dependency_loop(loops, t, obj)
557
            continue
558
        chain.add(obj)
559
        t2 = bld.get_tgen_by_name(obj)
560
        r2 = indirect_libs(bld, t2, chain, loops)
561
562
563
564
        chain.remove(obj)
        ret = ret.union(t2.direct_libs)
        ret = ret.union(r2)

565
    for obj in indirect_objects(bld, t, set(), loops):
566
        if obj in chain:
567
            dependency_loop(loops, t, obj)
568
569
            continue
        chain.add(obj)
570
        t2 = bld.get_tgen_by_name(obj)
571
        r2 = indirect_libs(bld, t2, chain, loops)
572
573
574
        chain.remove(obj)
        ret = ret.union(t2.direct_libs)
        ret = ret.union(r2)
575

576
    t.indirect_libs = ret
577

578
    return ret
579
580


581
def indirect_objects(bld, t, chain, loops):
582
583
584
585
586
587
588
589
590
591
592
593
    '''recursively calculate the indirect object dependencies for a target

    indirect objects are the set of objects from expanding the
    subsystem dependencies
    '''

    ret = getattr(t, 'indirect_objects', None)
    if ret is not None: return ret

    ret = set()
    for lib in t.direct_objects:
        if lib in chain:
594
            dependency_loop(loops, t, lib)
595
596
            continue
        chain.add(lib)
597
        t2 = bld.get_tgen_by_name(lib)
598
        r2 = indirect_objects(bld, t2, chain, loops)
599
600
601
602
603
604
605
606
        chain.remove(lib)
        ret = ret.union(t2.direct_objects)
        ret = ret.union(r2)

    t.indirect_objects = ret
    return ret


607
608
def extended_objects(bld, t, chain):
    '''recursively calculate the extended object dependencies for a target
609

610
611
612
613
    extended objects are the union of:
       - direct objects
       - indirect objects
       - direct and indirect objects of all direct and indirect libraries
614
615
    '''

616
    ret = getattr(t, 'extended_objects', None)
617
618
    if ret is not None: return ret

619
    ret = set()
620
    ret = ret.union(t.final_objects)
621

622
    for lib in t.final_libs:
623
624
        if lib in chain:
            continue
625
        t2 = bld.get_tgen_by_name(lib)
626
627
628
        chain.add(lib)
        r2 = extended_objects(bld, t2, chain)
        chain.remove(lib)
629
        ret = ret.union(t2.final_objects)
630
631
        ret = ret.union(r2)

632
    t.extended_objects = ret
633
634
635
    return ret


636
def includes_objects(bld, t, chain, inc_loops):
637
638
639
640
641
642
643
644
645
646
647
648
649
    '''recursively calculate the includes object dependencies for a target

    includes dependencies come from either library or object dependencies
    '''
    ret = getattr(t, 'includes_objects', None)
    if ret is not None:
        return ret

    ret = t.direct_objects.copy()
    ret = ret.union(t.direct_libs)

    for obj in t.direct_objects:
        if obj in chain:
650
            dependency_loop(inc_loops, t, obj)
651
652
            continue
        chain.add(obj)
653
        t2 = bld.get_tgen_by_name(obj)
654
        r2 = includes_objects(bld, t2, chain, inc_loops)
655
656
657
658
659
660
        chain.remove(obj)
        ret = ret.union(t2.direct_objects)
        ret = ret.union(r2)

    for lib in t.direct_libs:
        if lib in chain:
661
            dependency_loop(inc_loops, t, lib)
662
663
            continue
        chain.add(lib)
664
        t2 = bld.get_tgen_by_name(lib)
665
666
667
668
669
        if t2 is None:
            targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
            Logs.error('Target %s of type %s not found in direct_libs for %s' % (
                lib, targets[lib], t.sname))
            sys.exit(1)
670
        r2 = includes_objects(bld, t2, chain, inc_loops)
671
672
673
674
675
676
677
678
        chain.remove(lib)
        ret = ret.union(t2.direct_objects)
        ret = ret.union(r2)

    t.includes_objects = ret
    return ret


679
680
681
682
def break_dependency_loops(bld, tgt_list):
    '''find and break dependency loops'''
    loops = {}
    inc_loops = {}
683

684
685
686
687
688
    # build up the list of loops
    for t in tgt_list:
        indirect_objects(bld, t, set(), loops)
        indirect_libs(bld, t, set(), loops)
        includes_objects(bld, t, set(), inc_loops)
689

690
    # break the loops
691
    for t in tgt_list:
692
693
694
695
696
697
698
699
        if t.sname in loops:
            for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
                objs = getattr(t, attr, set())
                setattr(t, attr, objs.difference(loops[t.sname]))

    for loop in loops:
        debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])

700
701
702
    for loop in inc_loops:
        debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])

703
704
705
706
707
708
    # expand the loops mapping by one level
    for loop in loops.copy():
        for tgt in loops[loop]:
            if tgt in loops:
                loops[loop] = loops[loop].union(loops[tgt])

709
710
711
712
713
714
    for loop in inc_loops.copy():
        for tgt in inc_loops[loop]:
            if tgt in inc_loops:
                inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])


715
716
    # expand indirect subsystem and library loops
    for loop in loops.copy():
717
        t = bld.get_tgen_by_name(loop)
Stefan Metzmacher's avatar
Stefan Metzmacher committed
718
        if t.samba_type in ['SUBSYSTEM', 'BUILTIN']:
719
720
            loops[loop] = loops[loop].union(t.indirect_objects)
            loops[loop] = loops[loop].union(t.direct_objects)
721
        if t.samba_type in ['LIBRARY', 'PLUGIN', 'PYTHON']:
722
723
            loops[loop] = loops[loop].union(t.indirect_libs)
            loops[loop] = loops[loop].union(t.direct_libs)
724
725
726
        if loop in loops[loop]:
            loops[loop].remove(loop)

727
728
    # expand indirect includes loops
    for loop in inc_loops.copy():
729
        t = bld.get_tgen_by_name(loop)
730
731
732
733
        inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
        if loop in inc_loops[loop]:
            inc_loops[loop].remove(loop)

734
    # add in the replacement dependencies
735
    for t in tgt_list:
736
        for loop in loops:
737
            for attr in ['indirect_objects', 'indirect_libs']:
738
739
740
741
742
743
744
745
746
747
                objs = getattr(t, attr, set())
                if loop in objs:
                    diff = loops[loop].difference(objs)
                    if t.sname in diff:
                        diff.remove(t.sname)
                    if diff:
                        debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
                        objs = objs.union(diff)
                setattr(t, attr, objs)

748
749
750
751
752
753
754
755
756
757
758
        for loop in inc_loops:
            objs = getattr(t, 'includes_objects', set())
            if loop in objs:
                diff = inc_loops[loop].difference(objs)
                if t.sname in diff:
                    diff.remove(t.sname)
                if diff:
                    debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
                    objs = objs.union(diff)
            setattr(t, 'includes_objects', objs)

759

760
761
def reduce_objects(bld, tgt_list):
    '''reduce objects by looking for indirect object dependencies'''
Stefan Metzmacher's avatar
Stefan Metzmacher committed
762
763
    targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')

764
    rely_on = {}
765
766

    for t in tgt_list:
767
        t.extended_objects = None
768

769
770
    changed = False

771
    for type in ['BINARY', 'PYTHON', 'LIBRARY', 'PLUGIN']:
772
773
774
775
776
        for t in tgt_list:
            if t.samba_type != type: continue
            # if we will indirectly link to a target then we don't need it
            new = t.final_objects.copy()
            for l in t.final_libs:
777
                t2 = bld.get_tgen_by_name(l)
778
779
                t2_obj = extended_objects(bld, t2, set())
                dup = new.intersection(t2_obj)
780
781
                if t.sname in rely_on:
                    dup = dup.difference(rely_on[t.sname])
782
                if dup:
783
                    # Do not remove duplicates of BUILTINS
784
                    for d in iter(dup.copy()):
Stefan Metzmacher's avatar
Stefan Metzmacher committed
785
786
787
                        dtype = targets[d]
                        if dtype == 'BUILTIN':
                            debug('deps: BUILTIN SKIP: removing dups from %s of type %s: %s also in %s %s',
788
789
790
                                  t.sname, t.samba_type, d, t2.samba_type, l)
                            dup.remove(d)
                    if len(dup) == 0:
791
792
                        continue

793
794
795
796
                    debug('deps: removing dups from %s of type %s: %s also in %s %s',
                          t.sname, t.samba_type, dup, t2.samba_type, l)
                    new = new.difference(dup)
                    changed = True
797
798
799
                    if not l in rely_on:
                        rely_on[l] = set()
                    rely_on[l] = rely_on[l].union(dup)
Stefan Metzmacher's avatar
Stefan Metzmacher committed
800
801
802
803
804
805
806
807
808
809
810
811
812
            for n in iter(new.copy()):
                # if we got the builtin version as well
                # as the native one, we keep using the
                # builtin one and remove the rest.
                # Otherwise our check_duplicate_sources()
                # checks would trigger!
                if n.endswith('.builtin.objlist'):
                    unused = n.replace('.builtin.objlist', '.objlist')
                    if unused in new:
                        new.remove(unused)
                    unused = n.replace('.builtin.objlist', '')
                    if unused in new:
                        new.remove(unused)
813
814
            t.final_objects = new

815
816
817
    if not changed:
        return False

818
819
    # add back in any objects that were relied upon by the reduction rules
    for r in rely_on:
820
        t = bld.get_tgen_by_name(r)
821
822
        t.final_objects = t.final_objects.union(rely_on[r])

823
824
    return True

825

826
827
828
def show_library_loop(bld, lib1, lib2, path, seen):
    '''show the detailed path of a library loop between lib1 and lib2'''

829
    t = bld.get_tgen_by_name(lib1)
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
    if not lib2 in getattr(t, 'final_libs', set()):
        return

    for d in t.samba_deps_extended:
        if d in seen:
            continue
        seen.add(d)
        path2 = path + '=>' + d
        if d == lib2:
            Logs.warn('library loop path: ' + path2)
            return
        show_library_loop(bld, d, lib2, path2, seen)
        seen.remove(d)


845
846
847
848
849
850
851
852
853
854
855
856
857
858
def calculate_final_deps(bld, tgt_list, loops):
    '''calculate the final library and object dependencies'''
    for t in tgt_list:
        # start with the maximum possible list
        t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
        t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))

    for t in tgt_list:
        # don't depend on ourselves
        if t.sname in t.final_libs:
            t.final_libs.remove(t.sname)
        if t.sname in t.final_objects:
            t.final_objects.remove(t.sname)

859
860
861
    # handle any non-shared binaries
    for t in tgt_list:
        if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
862
863
864
            subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
            targets = LOCAL_CACHE(bld, 'TARGET_TYPE')

865
866
867
            # replace lib deps with objlist deps
            for l in t.final_libs:
                objname = l + '.objlist'
868
                t2 = bld.get_tgen_by_name(objname)
869
870
871
872
873
                if t2 is None:
                    Logs.error('ERROR: subsystem %s not found' % objname)
                    sys.exit(1)
                t.final_objects.add(objname)
                t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
874
875
876
877
878
879
880
881
882
883
                if l in subsystem_list:
                    # its a subsystem - we also need the contents of any modules
                    for d in subsystem_list[l]:
                        module_name = d['TARGET']
                        if targets[module_name] == 'LIBRARY':
                            objname = module_name + '.objlist'
                        elif targets[module_name] == 'SUBSYSTEM':
                            objname = module_name
                        else:
                            continue
884
                        t2 = bld.get_tgen_by_name(objname)
885
886
887
888
889
                        if t2 is None:
                            Logs.error('ERROR: subsystem %s not found' % objname)
                            sys.exit(1)
                        t.final_objects.add(objname)
                        t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
890
            t.final_libs = set()
891

892
893
894
895
    # find any library loops
    for t in tgt_list:
        if t.samba_type in ['LIBRARY', 'PYTHON']:
            for l in t.final_libs.copy():
896
                t2 = bld.get_tgen_by_name(l)
897
                if t.sname in t2.final_libs:
898
899
900
901
902
903
904
905
906
907
                    if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
                        # we could break this in either direction. If one of the libraries
                        # has a version number, and will this be distributed publicly, then
                        # we should make it the lower level library in the DAG
                        Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
                        dependency_loop(loops, t, t2.sname)
                        t2.final_libs.remove(t.sname)
                    else:
                        Logs.error('ERROR: circular library dependency between %s and %s'
                            % (t.sname, t2.sname))
908
909
                        show_library_loop(bld, t.sname, t2.sname, t.sname, set())
                        show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
910
                        sys.exit(1)
911

912
913
    for loop in loops:
        debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
914
915
916
917

    # we now need to make corrections for any library loops we broke up
    # any target that depended on the target of the loop and doesn't
    # depend on the source of the loop needs to get the loop source added
918
    for type in ['BINARY','PYTHON','LIBRARY','PLUGIN','BINARY']:
919
920
921
        for t in tgt_list:
            if t.samba_type != type: continue
            for loop in loops:
922
923
924
925
                if loop in t.final_libs:
                    diff = loops[loop].difference(t.final_libs)
                    if t.sname in diff:
                        diff.remove(t.sname)
926
927
928
929
                    if t.sname in diff:
                        diff.remove(t.sname)
                    # make sure we don't recreate the loop again!
                    for d in diff.copy():
930
                        t2 = bld.get_tgen_by_name(d)
931
932
933
934
                        if t2.samba_type == 'LIBRARY':
                            if t.sname in t2.final_libs:
                                debug('deps: removing expansion %s from %s', d, t.sname)
                                diff.remove(d)
935
                    if diff:
936
937
                        debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
                              loops[loop], diff)
938
                        t.final_libs = t.final_libs.union(diff)
939

940
    # remove objects that are also available in linked libs
941
942
943
944
    count = 0
    while reduce_objects(bld, tgt_list):
        count += 1
        if count > 100:
945
            Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
946
947
            break
    debug('deps: Object reduction took %u iterations', count)
948

949
950
    # add in any syslib dependencies
    for t in tgt_list:
951
        if not t.samba_type in ['BINARY','PYTHON','LIBRARY','PLUGIN','SUBSYSTEM','BUILTIN']:
952
953
954
            continue
        syslibs = set()
        for d in t.final_objects:
955
            t2 = bld.get_tgen_by_name(d)
956
957
958
959
            syslibs = syslibs.union(t2.direct_syslibs)
        # this adds the indirect syslibs as well, which may not be needed
        # depending on the linker flags
        for d in t.final_libs:
960
            t2 = bld.get_tgen_by_name(d)
961
962
963
            syslibs = syslibs.union(t2.direct_syslibs)
        t.final_syslibs = syslibs

964
965
966
967

    # find any unresolved library loops
    lib_loop_error = False
    for t in tgt_list:
968
        if t.samba_type in ['LIBRARY', 'PLUGIN', 'PYTHON']:
969
            for l in t.final_libs.copy():
970
                t2 = bld.get_tgen_by_name(l)
971
972
973
974
975
976
                if t.sname in t2.final_libs:
                    Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
                    lib_loop_error = True
    if lib_loop_error:
        sys.exit(1)

977
978
979
    debug('deps: removed duplicate dependencies')


980
981
982
983
984
985
def show_dependencies(bld, target, seen):
    '''recursively show the dependencies of target'''

    if target in seen:
        return

986
    t = bld.get_tgen_by_name(target)
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
    if t is None:
        Logs.error("ERROR: Unable to find target '%s'" % target)
        sys.exit(1)

    Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
    Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
    Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))

    seen.add(target)

    for t2 in t.direct_objects:
        show_dependencies(bld, t2, seen)


For faster browsing, not all history is shown. View entire blame