buildtools/wafadmin/Tools/d.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Carlos Rafael Giani, 2007 (dv)
   4 # Thomas Nagy, 2007-2008 (ita)
   5
   6 import os, sys, re, optparse
   7 import ccroot # <- leave this
   8 import TaskGen, Utils, Task, Configure, Logs, Build
   9 from Logs import debug, error
  10 from TaskGen import taskgen, feature, after, before, extension
  11 from Configure import conftest
  12
  13 EXT_D = ['.d', '.di', '.D']
  14 D_METHS = ['apply_core', 'apply_vnum', 'apply_objdeps'] # additional d methods
  15
  16 DLIB = """
  17 version(D_Version2) {
  18         import std.stdio;
  19         int main() {
  20                 writefln("phobos2");
  21                 return 0;
  22         }
  23 } else {
  24         version(Tango) {
  25                 import tango.stdc.stdio;
  26                 int main() {
  27                         printf("tango");
  28                         return 0;
  29                 }
  30         } else {
  31                 import std.stdio;
  32                 int main() {
  33                         writefln("phobos1");
  34                         return 0;
  35                 }
  36         }
  37 }
  38 """
  39
  40 def filter_comments(filename):
  41         txt = Utils.readf(filename)
  42         i = 0
  43         buf = []
  44         max = len(txt)
  45         begin = 0
  46         while i < max:
  47                 c = txt[i]
  48                 if c == '"' or c == "'":  # skip a string or character literal
  49                         buf.append(txt[begin:i])
  50                         delim = c
  51                         i += 1
  52                         while i < max:
  53                                 c = txt[i]
  54                                 if c == delim: break
  55                                 elif c == '\\':  # skip the character following backslash
  56                                         i += 1
  57                                 i += 1
  58                         i += 1
  59                         begin = i
  60                 elif c == '/':  # try to replace a comment with whitespace
  61                         buf.append(txt[begin:i])
  62                         i += 1
  63                         if i == max: break
  64                         c = txt[i]
  65                         if c == '+':  # eat nesting /+ +/ comment
  66                                 i += 1
  67                                 nesting = 1
  68                                 c = None
  69                                 while i < max:
  70                                         prev = c
  71                                         c = txt[i]
  72                                         if prev == '/' and c == '+':
  73                                                 nesting += 1
  74                                                 c = None
  75                                         elif prev == '+' and c == '/':
  76                                                 nesting -= 1
  77                                                 if nesting == 0: break
  78                                                 c = None
  79                                         i += 1
  80                         elif c == '*':  # eat /* */ comment
  81                                 i += 1
  82                                 c = None
  83                                 while i < max:
  84                                         prev = c
  85                                         c = txt[i]
  86                                         if prev == '*' and c == '/': break
  87                                         i += 1
  88                         elif c == '/':  # eat // comment
  89                                 i += 1
  90                                 while i < max and txt[i] != '\n':
  91                                         i += 1
  92                         else:  # no comment
  93                                 begin = i - 1
  94                                 continue
  95                         i += 1
  96                         begin = i
  97                         buf.append(' ')
  98                 else:
  99                         i += 1
 100         buf.append(txt[begin:])
 101         return buf
 102
 103 class d_parser(object):
 104         def __init__(self, env, incpaths):
 105                 #self.code = ''
 106                 #self.module = ''
 107                 #self.imports = []
 108
 109                 self.allnames = []
 110
 111                 self.re_module = re.compile("module\s+([^;]+)")
 112                 self.re_import = re.compile("import\s+([^;]+)")
 113                 self.re_import_bindings = re.compile("([^:]+):(.*)")
 114                 self.re_import_alias = re.compile("[^=]+=(.+)")
 115
 116                 self.env = env
 117
 118                 self.nodes = []
 119                 self.names = []
 120
 121                 self.incpaths = incpaths
 122
 123         def tryfind(self, filename):
 124                 found = 0
 125                 for n in self.incpaths:
 126                         found = n.find_resource(filename.replace('.', '/') + '.d')
 127                         if found:
 128                                 self.nodes.append(found)
 129                                 self.waiting.append(found)
 130                                 break
 131                 if not found:
 132                         if not filename in self.names:
 133                                 self.names.append(filename)
 134
 135         def get_strings(self, code):
 136                 #self.imports = []
 137                 self.module = ''
 138                 lst = []
 139
 140                 # get the module name (if present)
 141
 142                 mod_name = self.re_module.search(code)
 143                 if mod_name:
 144                         self.module = re.sub('\s+', '', mod_name.group(1)) # strip all whitespaces
 145
 146                 # go through the code, have a look at all import occurrences
 147
 148                 # first, lets look at anything beginning with "import" and ending with ";"
 149                 import_iterator = self.re_import.finditer(code)
 150                 if import_iterator:
 151                         for import_match in import_iterator:
 152                                 import_match_str = re.sub('\s+', '', import_match.group(1)) # strip all whitespaces
 153
 154                                 # does this end with an import bindings declaration?
 155                                 # (import bindings always terminate the list of imports)
 156                                 bindings_match = self.re_import_bindings.match(import_match_str)
 157                                 if bindings_match:
 158                                         import_match_str = bindings_match.group(1)
 159                                         # if so, extract the part before the ":" (since the module declaration(s) is/are located there)
 160
 161                                 # split the matching string into a bunch of strings, separated by a comma
 162                                 matches = import_match_str.split(',')
 163
 164                                 for match in matches:
 165                                         alias_match = self.re_import_alias.match(match)
 166                                         if alias_match:
 167                                                 # is this an alias declaration? (alias = module name) if so, extract the module name
 168                                                 match = alias_match.group(1)
 169
 170                                         lst.append(match)
 171                 return lst
 172
 173         def start(self, node):
 174                 self.waiting = [node]
 175                 # while the stack is not empty, add the dependencies
 176                 while self.waiting:
 177                         nd = self.waiting.pop(0)
 178                         self.iter(nd)
 179
 180         def iter(self, node):
 181                 path = node.abspath(self.env) # obtain the absolute path
 182                 code = "".join(filter_comments(path)) # read the file and filter the comments
 183                 names = self.get_strings(code) # obtain the import strings
 184                 for x in names:
 185                         # optimization
 186                         if x in self.allnames: continue
 187                         self.allnames.append(x)
 188
 189                         # for each name, see if it is like a node or not
 190                         self.tryfind(x)
 191
 192 def scan(self):
 193         "look for .d/.di the .d source need"
 194         env = self.env
 195         gruik = d_parser(env, env['INC_PATHS'])
 196         gruik.start(self.inputs[0])
 197
 198         if Logs.verbose:
 199                 debug('deps: nodes found for %s: %s %s' % (str(self.inputs[0]), str(gruik.nodes), str(gruik.names)))
 200                 #debug("deps found for %s: %s" % (str(node), str(gruik.deps)), 'deps')
 201         return (gruik.nodes, gruik.names)
 202
 203 def get_target_name(self):
 204         "for d programs and libs"
 205         v = self.env
 206         tp = 'program'
 207         for x in self.features:
 208                 if x in ['dshlib', 'dstaticlib']:
 209                         tp = x.lstrip('d')
 210         return v['D_%s_PATTERN' % tp] % self.target
 211
 212 d_params = {
 213 'dflags': '',
 214 'importpaths':'',
 215 'libs':'',
 216 'libpaths':'',
 217 'generate_headers':False,
 218 }
 219
 220 @feature('d')
 221 @before('apply_type_vars')
 222 def init_d(self):
 223         for x in d_params:
 224                 setattr(self, x, getattr(self, x, d_params[x]))
 225
 226 class d_taskgen(TaskGen.task_gen):
 227         def __init__(self, *k, **kw):
 228                 TaskGen.task_gen.__init__(self, *k, **kw)
 229
 230                 # COMPAT
 231                 if len(k) > 1:
 232                         self.features.append('d' + k[1])
 233
 234 # okay, we borrow a few methods from ccroot
 235 TaskGen.bind_feature('d', D_METHS)
 236
 237 @feature('d')
 238 @before('apply_d_libs')
 239 def init_d(self):
 240         Utils.def_attrs(self,
 241                 dflags='',
 242                 importpaths='',
 243                 libs='',
 244                 libpaths='',
 245                 uselib='',
 246                 uselib_local='',
 247                 generate_headers=False, # set to true if you want .di files as well as .o
 248                 compiled_tasks=[],
 249                 add_objects=[],
 250                 link_task=None)
 251
 252 @feature('d')
 253 @after('apply_d_link', 'init_d')
 254 @before('apply_vnum', 'apply_d_vars')
 255 def apply_d_libs(self):
 256         """after apply_link because of 'link_task'
 257         after default_cc because of the attribute 'uselib'"""
 258         env = self.env
 259
 260         # 1. the case of the libs defined in the project (visit ancestors first)
 261         # the ancestors external libraries (uselib) will be prepended
 262         self.uselib = self.to_list(self.uselib)
 263         names = self.to_list(self.uselib_local)
 264
 265         seen = set([])
 266         tmp = Utils.deque(names) # consume a copy of the list of names
 267         while tmp:
 268                 lib_name = tmp.popleft()
 269                 # visit dependencies only once
 270                 if lib_name in seen:
 271                         continue
 272
 273                 y = self.name_to_obj(lib_name)
 274                 if not y:
 275                         raise Utils.WafError('object %r was not found in uselib_local (required by %r)' % (lib_name, self.name))
 276                 y.post()
 277                 seen.add(lib_name)
 278
 279                 # object has ancestors to process (shared libraries): add them to the end of the list
 280                 if getattr(y, 'uselib_local', None):
 281                         lst = y.to_list(y.uselib_local)
 282                         if 'dshlib' in y.features or 'dprogram' in y.features:
 283                                 lst = [x for x in lst if not 'dstaticlib' in self.name_to_obj(x).features]
 284                         tmp.extend(lst)
 285
 286                 # link task and flags
 287                 if getattr(y, 'link_task', None):
 288
 289                         link_name = y.target[y.target.rfind(os.sep) + 1:]
 290                         if 'dstaticlib' in y.features or 'dshlib' in y.features:
 291                                 env.append_unique('DLINKFLAGS', env.DLIB_ST % link_name)
 292                                 env.append_unique('DLINKFLAGS', env.DLIBPATH_ST % y.link_task.outputs[0].parent.bldpath(env))
 293
 294                         # the order
 295                         self.link_task.set_run_after(y.link_task)
 296
 297                         # for the recompilation
 298                         dep_nodes = getattr(self.link_task, 'dep_nodes', [])
 299                         self.link_task.dep_nodes = dep_nodes + y.link_task.outputs
 300
 301                 # add ancestors uselib too - but only propagate those that have no staticlib
 302                 for v in self.to_list(y.uselib):
 303                         if not v in self.uselib:
 304                                 self.uselib.insert(0, v)
 305
 306                 # if the library task generator provides 'export_incdirs', add to the include path
 307                 # the export_incdirs must be a list of paths relative to the other library
 308                 if getattr(y, 'export_incdirs', None):
 309                         for x in self.to_list(y.export_incdirs):
 310                                 node = y.path.find_dir(x)
 311                                 if not node:
 312                                         raise Utils.WafError('object %r: invalid folder %r in export_incdirs' % (y.target, x))
 313                                 self.env.append_unique('INC_PATHS', node)
 314
 315 @feature('dprogram', 'dshlib', 'dstaticlib')
 316 @after('apply_core')
 317 def apply_d_link(self):
 318         link = getattr(self, 'link', None)
 319         if not link:
 320                 if 'dstaticlib' in self.features: link = 'static_link'
 321                 else: link = 'd_link'
 322
 323         outputs = [t.outputs[0] for t in self.compiled_tasks]
 324         self.link_task = self.create_task(link, outputs, self.path.find_or_declare(get_target_name(self)))
 325
 326 @feature('d')
 327 @after('apply_core')
 328 def apply_d_vars(self):
 329         env = self.env
 330         dpath_st   = env['DPATH_ST']
 331         lib_st   = env['DLIB_ST']
 332         libpath_st = env['DLIBPATH_ST']
 333
 334         importpaths = self.to_list(self.importpaths)
 335         libpaths = []
 336         libs = []
 337         uselib = self.to_list(self.uselib)
 338
 339         for i in uselib:
 340                 if env['DFLAGS_' + i]:
 341                         env.append_unique('DFLAGS', env['DFLAGS_' + i])
 342
 343         for x in self.features:
 344                 if not x in ['dprogram', 'dstaticlib', 'dshlib']:
 345                         continue
 346                 x.lstrip('d')
 347                 d_shlib_dflags = env['D_' + x + '_DFLAGS']
 348                 if d_shlib_dflags:
 349                         env.append_unique('DFLAGS', d_shlib_dflags)
 350
 351         # add import paths
 352         for i in uselib:
 353                 if env['DPATH_' + i]:
 354                         for entry in self.to_list(env['DPATH_' + i]):
 355                                 if not entry in importpaths:
 356                                         importpaths.append(entry)
 357
 358         # now process the import paths
 359         for path in importpaths:
 360                 if os.path.isabs(path):
 361                         env.append_unique('_DIMPORTFLAGS', dpath_st % path)
 362                 else:
 363                         node = self.path.find_dir(path)
 364                         self.env.append_unique('INC_PATHS', node)
 365                         env.append_unique('_DIMPORTFLAGS', dpath_st % node.srcpath(env))
 366                         env.append_unique('_DIMPORTFLAGS', dpath_st % node.bldpath(env))
 367
 368         # add library paths
 369         for i in uselib:
 370                 if env['LIBPATH_' + i]:
 371                         for entry in self.to_list(env['LIBPATH_' + i]):
 372                                 if not entry in libpaths:
 373                                         libpaths.append(entry)
 374         libpaths = self.to_list(self.libpaths) + libpaths
 375
 376         # now process the library paths
 377         # apply same path manipulation as used with import paths
 378         for path in libpaths:
 379                 if not os.path.isabs(path):
 380                         node = self.path.find_resource(path)
 381                         if not node:
 382                                 raise Utils.WafError('could not find libpath %r from %r' % (path, self))
 383                         path = node.abspath(self.env)
 384
 385                 env.append_unique('DLINKFLAGS', libpath_st % path)
 386
 387         # add libraries
 388         for i in uselib:
 389                 if env['LIB_' + i]:
 390                         for entry in self.to_list(env['LIB_' + i]):
 391                                 if not entry in libs:
 392                                         libs.append(entry)
 393         libs.extend(self.to_list(self.libs))
 394
 395         # process user flags
 396         for flag in self.to_list(self.dflags):
 397                 env.append_unique('DFLAGS', flag)
 398
 399         # now process the libraries
 400         for lib in libs:
 401                 env.append_unique('DLINKFLAGS', lib_st % lib)
 402
 403         # add linker flags
 404         for i in uselib:
 405                 dlinkflags = env['DLINKFLAGS_' + i]
 406                 if dlinkflags:
 407                         for linkflag in dlinkflags:
 408                                 env.append_unique('DLINKFLAGS', linkflag)
 409
 410 @feature('dshlib')
 411 @after('apply_d_vars')
 412 def add_shlib_d_flags(self):
 413         for linkflag in self.env['D_shlib_LINKFLAGS']:
 414                 self.env.append_unique('DLINKFLAGS', linkflag)
 415
 416 @extension(EXT_D)
 417 def d_hook(self, node):
 418         # create the compilation task: cpp or cc
 419         task = self.create_task(self.generate_headers and 'd_with_header' or 'd')
 420         try: obj_ext = self.obj_ext
 421         except AttributeError: obj_ext = '_%d.o' % self.idx
 422
 423         task.inputs = [node]
 424         task.outputs = [node.change_ext(obj_ext)]
 425         self.compiled_tasks.append(task)
 426
 427         if self.generate_headers:
 428                 header_node = node.change_ext(self.env['DHEADER_ext'])
 429                 task.outputs += [header_node]
 430
 431 d_str = '${D_COMPILER} ${DFLAGS} ${_DIMPORTFLAGS} ${D_SRC_F}${SRC} ${D_TGT_F}${TGT}'
 432 d_with_header_str = '${D_COMPILER} ${DFLAGS} ${_DIMPORTFLAGS} \
 433 ${D_HDR_F}${TGT[1].bldpath(env)} \
 434 ${D_SRC_F}${SRC} \
 435 ${D_TGT_F}${TGT[0].bldpath(env)}'
 436 link_str = '${D_LINKER} ${DLNK_SRC_F}${SRC} ${DLNK_TGT_F}${TGT} ${DLINKFLAGS}'
 437
 438 def override_exec(cls):
 439         """stupid dmd wants -of stuck to the file name"""
 440         old_exec = cls.exec_command
 441         def exec_command(self, *k, **kw):
 442                 if isinstance(k[0], list):
 443                         lst = k[0]
 444                         for i in xrange(len(lst)):
 445                                 if lst[i] == '-of':
 446                                         del lst[i]
 447                                         lst[i] = '-of' + lst[i]
 448                                         break
 449                 return old_exec(self, *k, **kw)
 450         cls.exec_command = exec_command
 451
 452 cls = Task.simple_task_type('d', d_str, 'GREEN', before='static_link d_link', shell=False)
 453 cls.scan = scan
 454 override_exec(cls)
 455
 456 cls = Task.simple_task_type('d_with_header', d_with_header_str, 'GREEN', before='static_link d_link', shell=False)
 457 override_exec(cls)
 458
 459 cls = Task.simple_task_type('d_link', link_str, color='YELLOW', shell=False)
 460 override_exec(cls)
 461
 462 # for feature request #104
 463 @taskgen
 464 def generate_header(self, filename, install_path):
 465         if not hasattr(self, 'header_lst'): self.header_lst = []
 466         self.meths.append('process_header')
 467         self.header_lst.append([filename, install_path])
 468
 469 @before('apply_core')
 470 def process_header(self):
 471         env = self.env
 472         for i in getattr(self, 'header_lst', []):
 473                 node = self.path.find_resource(i[0])
 474
 475                 if not node:
 476                         raise Utils.WafError('file not found on d obj '+i[0])
 477
 478                 task = self.create_task('d_header')
 479                 task.set_inputs(node)
 480                 task.set_outputs(node.change_ext('.di'))
 481
 482 d_header_str = '${D_COMPILER} ${D_HEADER} ${SRC}'
 483 Task.simple_task_type('d_header', d_header_str, color='BLUE', shell=False)
 484
 485 @conftest
 486 def d_platform_flags(conf):
 487         v = conf.env
 488         binfmt = v.DEST_BINFMT or Utils.unversioned_sys_platform_to_binary_format(
 489                 v.DEST_OS or Utils.unversioned_sys_platform())
 490         if binfmt == 'pe':
 491                 v['D_program_PATTERN']   = '%s.exe'
 492                 v['D_shlib_PATTERN']     = 'lib%s.dll'
 493                 v['D_staticlib_PATTERN'] = 'lib%s.a'
 494         else:
 495                 v['D_program_PATTERN']   = '%s'
 496                 v['D_shlib_PATTERN']     = 'lib%s.so'
 497                 v['D_staticlib_PATTERN'] = 'lib%s.a'
 498
 499 @conftest
 500 def check_dlibrary(conf):
 501         ret = conf.check_cc(features='d dprogram', fragment=DLIB, mandatory=True, compile_filename='test.d', execute=True)
 502         conf.env.DLIBRARY = ret.strip()
 503
 504 # quick test #
 505 if __name__ == "__main__":
 506         #Logs.verbose = 2
 507
 508         try: arg = sys.argv[1]
 509         except IndexError: arg = "file.d"
 510
 511         print("".join(filter_comments(arg)))
 512         # TODO
 513         paths = ['.']
 514
 515         #gruik = filter()
 516         #gruik.start(arg)
 517
 518         #code = "".join(gruik.buf)
 519
 520         #print "we have found the following code"
 521         #print code
 522
 523         #print "now parsing"
 524         #print "-------------------------------------------"
 525         """
 526         parser_ = d_parser()
 527         parser_.start(arg)
 528
 529         print "module: %s" % parser_.module
 530         print "imports: ",
 531         for imp in parser_.imports:
 532                 print imp + " ",
 533         print
 534 """
 535