third_party/waf/waflib/extras/batched_cc.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2006-2015 (ita)
   4
   5 """
   6 Build as batches.
   7
   8 Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
   9 cc -c ../file1.c ../file2.c ../file3.c
  10
  11 Files are output on the directory where the compiler is called, and dependencies are more difficult
  12 to track (do not run the command on all source files if only one file changes)
  13
  14 As such, we do as if the files were compiled one by one, but no command is actually run:
  15 replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
  16 signatures from each slave and finds out the command-line to run.
  17
  18 Just import this module in the configuration (no other change required).
  19 This is provided as an example, for performance unity builds are recommended (fewer tasks and
  20 fewer jobs to execute). See waflib/extras/unity.py.
  21 """
  22
  23 from waflib import Task, Utils
  24 from waflib.TaskGen import extension, feature, after_method
  25 from waflib.Tools import c, cxx
  26
  27 MAX_BATCH = 50
  28
  29 c_str = '${CC} ${CFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED}'
  30 c_fun, _ = Task.compile_fun_noshell(c_str)
  31
  32 cxx_str = '${CXX} ${CXXFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED}'
  33 cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
  34
  35 count = 70000
  36 class batch_task(Task.Task):
  37         color = 'PINK'
  38
  39         after = ['c', 'cxx']
  40         before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
  41
  42         def uid(self):
  43                 m = Utils.md5()
  44                 m.update(Task.Task.uid(self))
  45                 m.update(str(self.generator.idx).encode())
  46                 return m.digest()
  47
  48         def __str__(self):
  49                 return 'Batch compilation for %d slaves' % len(self.slaves)
  50
  51         def __init__(self, *k, **kw):
  52                 Task.Task.__init__(self, *k, **kw)
  53                 self.slaves = []
  54                 self.inputs = []
  55                 self.hasrun = 0
  56
  57                 global count
  58                 count += 1
  59                 self.idx = count
  60
  61         def add_slave(self, slave):
  62                 self.slaves.append(slave)
  63                 self.set_run_after(slave)
  64
  65         def runnable_status(self):
  66                 for t in self.run_after:
  67                         if not t.hasrun:
  68                                 return Task.ASK_LATER
  69
  70                 for t in self.slaves:
  71                         #if t.executed:
  72                         if t.hasrun != Task.SKIPPED:
  73                                 return Task.RUN_ME
  74
  75                 return Task.SKIP_ME
  76
  77         def run(self):
  78                 self.outputs = []
  79
  80                 srclst = []
  81                 slaves = []
  82                 for t in self.slaves:
  83                         if t.hasrun != Task.SKIPPED:
  84                                 slaves.append(t)
  85                                 srclst.append(t.inputs[0].abspath())
  86
  87                 self.env.SRCLST = srclst
  88                 self.cwd = slaves[0].outputs[0].parent.abspath()
  89
  90                 if self.slaves[0].__class__.__name__ == 'c':
  91                         ret = c_fun(self)
  92                 else:
  93                         ret = cxx_fun(self)
  94
  95                 if ret:
  96                         return ret
  97
  98                 for t in slaves:
  99                         t.old_post_run()
 100
 101 def hook(cls_type):
 102         def n_hook(self, node):
 103
 104                 ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o'
 105                 name = node.name
 106                 k = name.rfind('.')
 107                 if k >= 0:
 108                         basename = name[:k] + ext
 109                 else:
 110                         basename = name + ext
 111
 112                 outdir = node.parent.get_bld().make_node('%d' % self.idx)
 113                 outdir.mkdir()
 114                 out = outdir.find_or_declare(basename)
 115
 116                 task = self.create_task(cls_type, node, out)
 117
 118                 try:
 119                         self.compiled_tasks.append(task)
 120                 except AttributeError:
 121                         self.compiled_tasks = [task]
 122
 123                 if not getattr(self, 'masters', None):
 124                         self.masters = {}
 125                         self.allmasters = []
 126
 127                 def fix_path(tsk):
 128                         if self.env.CC_NAME == 'msvc':
 129                                 tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath())
 130
 131                 if not node.parent in self.masters:
 132                         m = self.masters[node.parent] = self.master = self.create_task('batch')
 133                         fix_path(m)
 134                         self.allmasters.append(m)
 135                 else:
 136                         m = self.masters[node.parent]
 137                         if len(m.slaves) > MAX_BATCH:
 138                                 m = self.masters[node.parent] = self.master = self.create_task('batch')
 139                                 fix_path(m)
 140                                 self.allmasters.append(m)
 141                 m.add_slave(task)
 142                 return task
 143         return n_hook
 144
 145 extension('.c')(hook('c'))
 146 extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))
 147
 148 @feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
 149 @after_method('apply_link')
 150 def link_after_masters(self):
 151         if getattr(self, 'allmasters', None):
 152                 for m in self.allmasters:
 153                         self.link_task.set_run_after(m)
 154
 155 # Modify the c and cxx task classes - in theory it would be best to
 156 # create subclasses and to re-map the c/c++ extensions
 157 for x in ('c', 'cxx'):
 158         t = Task.classes[x]
 159         def run(self):
 160                 pass
 161
 162         def post_run(self):
 163                 pass
 164
 165         setattr(t, 'oldrun', getattr(t, 'run', None))
 166         setattr(t, 'run', run)
 167         setattr(t, 'old_post_run', t.post_run)
 168         setattr(t, 'post_run', post_run)