buildtools/wafadmin/Tools/python.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2007 (ita)
   4 # Gustavo Carneiro (gjc), 2007
   5
   6 "Python support"
   7
   8 import os, sys
   9 import TaskGen, Utils, Options
  10 from Logs import debug, warn, info
  11 from TaskGen import extension, before, after, feature
  12 from Configure import conf
  13
  14 EXT_PY = ['.py']
  15 FRAG_2 = '''
  16 #include "Python.h"
  17 #ifdef __cplusplus
  18 extern "C" {
  19 #endif
  20         void Py_Initialize(void);
  21         void Py_Finalize(void);
  22 #ifdef __cplusplus
  23 }
  24 #endif
  25 int main()
  26 {
  27    Py_Initialize();
  28    Py_Finalize();
  29    return 0;
  30 }
  31 '''
  32
  33 @feature('pyext')
  34 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars', 'apply_bundle')
  35 @after('vars_target_cshlib')
  36 def init_pyext(self):
  37         self.default_install_path = '${PYTHONARCHDIR}'
  38         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  39         if not 'PYEXT' in self.uselib:
  40                 self.uselib.append('PYEXT')
  41         self.env['MACBUNDLE'] = True
  42
  43 @before('apply_link', 'apply_lib_vars', 'apply_type_vars')
  44 @after('apply_bundle')
  45 @feature('pyext')
  46 def pyext_shlib_ext(self):
  47         # override shlib_PATTERN set by the osx module
  48         self.env['shlib_PATTERN'] = self.env['pyext_PATTERN']
  49
  50 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars')
  51 @feature('pyembed')
  52 def init_pyembed(self):
  53         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  54         if not 'PYEMBED' in self.uselib:
  55                 self.uselib.append('PYEMBED')
  56
  57 @extension(EXT_PY)
  58 def process_py(self, node):
  59         if not (self.bld.is_install and self.install_path):
  60                 return
  61         def inst_py(ctx):
  62                 install_pyfile(self, node)
  63         self.bld.add_post_fun(inst_py)
  64
  65 def install_pyfile(self, node):
  66         path = self.bld.get_install_path(self.install_path + os.sep + node.name, self.env)
  67
  68         self.bld.install_files(self.install_path, [node], self.env, self.chmod, postpone=False)
  69         if self.bld.is_install < 0:
  70                 info("* removing byte compiled python files")
  71                 for x in 'co':
  72                         try:
  73                                 os.remove(path + x)
  74                         except OSError:
  75                                 pass
  76
  77         if self.bld.is_install > 0:
  78                 if self.env['PYC'] or self.env['PYO']:
  79                         info("* byte compiling %r" % path)
  80
  81                 if self.env['PYC']:
  82                         program = ("""
  83 import sys, py_compile
  84 for pyfile in sys.argv[1:]:
  85         py_compile.compile(pyfile, pyfile + 'c')
  86 """)
  87                         argv = [self.env['PYTHON'], '-c', program, path]
  88                         ret = Utils.pproc.Popen(argv).wait()
  89                         if ret:
  90                                 raise Utils.WafError('bytecode compilation failed %r' % path)
  91
  92                 if self.env['PYO']:
  93                         program = ("""
  94 import sys, py_compile
  95 for pyfile in sys.argv[1:]:
  96         py_compile.compile(pyfile, pyfile + 'o')
  97 """)
  98                         argv = [self.env['PYTHON'], self.env['PYFLAGS_OPT'], '-c', program, path]
  99                         ret = Utils.pproc.Popen(argv).wait()
 100                         if ret:
 101                                 raise Utils.WafError('bytecode compilation failed %r' % path)
 102
 103 # COMPAT
 104 class py_taskgen(TaskGen.task_gen):
 105         def __init__(self, *k, **kw):
 106                 TaskGen.task_gen.__init__(self, *k, **kw)
 107
 108 @before('apply_core')
 109 @after('vars_target_cprogram', 'vars_target_cshlib')
 110 @feature('py')
 111 def init_py(self):
 112         self.default_install_path = '${PYTHONDIR}'
 113
 114 def _get_python_variables(python_exe, variables, imports=['import sys']):
 115         """Run a python interpreter and print some variables"""
 116         program = list(imports)
 117         program.append('')
 118         for v in variables:
 119                 program.append("print(repr(%s))" % v)
 120         os_env = dict(os.environ)
 121         try:
 122                 del os_env['MACOSX_DEPLOYMENT_TARGET'] # see comments in the OSX tool
 123         except KeyError:
 124                 pass
 125         proc = Utils.pproc.Popen([python_exe, "-c", '\n'.join(program)], stdout=Utils.pproc.PIPE, env=os_env)
 126         output = proc.communicate()[0].split("\n") # do not touch, python3
 127         if proc.returncode:
 128                 if Options.options.verbose:
 129                         warn("Python program to extract python configuration variables failed:\n%s"
 130                                        % '\n'.join(["line %03i: %s" % (lineno+1, line) for lineno, line in enumerate(program)]))
 131                 raise RuntimeError
 132         return_values = []
 133         for s in output:
 134                 s = s.strip()
 135                 if not s:
 136                         continue
 137                 if s == 'None':
 138                         return_values.append(None)
 139                 elif s[0] == "'" and s[-1] == "'":
 140                         return_values.append(s[1:-1])
 141                 elif s[0].isdigit():
 142                         return_values.append(int(s))
 143                 else: break
 144         return return_values
 145
 146 @conf
 147 def check_python_headers(conf, mandatory=True):
 148         """Check for headers and libraries necessary to extend or embed python.
 149
 150         On success the environment variables xxx_PYEXT and xxx_PYEMBED are added for uselib
 151
 152         PYEXT: for compiling python extensions
 153         PYEMBED: for embedding a python interpreter"""
 154
 155         if not conf.env['CC_NAME'] and not conf.env['CXX_NAME']:
 156                 conf.fatal('load a compiler first (gcc, g++, ..)')
 157
 158         if not conf.env['PYTHON_VERSION']:
 159                 conf.check_python_version()
 160
 161         env = conf.env
 162         python = env['PYTHON']
 163         if not python:
 164                 conf.fatal('could not find the python executable')
 165
 166         ## On Mac OSX we need to use mac bundles for python plugins
 167         if Options.platform == 'darwin':
 168                 conf.check_tool('osx')
 169
 170         try:
 171                 # Get some python configuration variables using distutils
 172                 v = 'prefix SO SYSLIBS LDFLAGS SHLIBS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET'.split()
 173                 (python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 174                  python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED,
 175                  python_MACOSX_DEPLOYMENT_TARGET) = \
 176                         _get_python_variables(python, ["get_config_var('%s') or ''" % x for x in v],
 177                                               ['from distutils.sysconfig import get_config_var'])
 178         except RuntimeError:
 179                 conf.fatal("Python development headers not found (-v for details).")
 180
 181         conf.log.write("""Configuration returned from %r:
 182 python_prefix = %r
 183 python_SO = %r
 184 python_SYSLIBS = %r
 185 python_LDFLAGS = %r
 186 python_SHLIBS = %r
 187 python_LIBDIR = %r
 188 python_LIBPL = %r
 189 INCLUDEPY = %r
 190 Py_ENABLE_SHARED = %r
 191 MACOSX_DEPLOYMENT_TARGET = %r
 192 """ % (python, python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 193         python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED, python_MACOSX_DEPLOYMENT_TARGET))
 194
 195         if python_MACOSX_DEPLOYMENT_TARGET:
 196                 conf.env['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 197                 conf.environ['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 198
 199         env['pyext_PATTERN'] = '%s'+python_SO
 200
 201         # Check for python libraries for embedding
 202         if python_SYSLIBS is not None:
 203                 for lib in python_SYSLIBS.split():
 204                         if lib.startswith('-l'):
 205                                 lib = lib[2:] # strip '-l'
 206                         env.append_value('LIB_PYEMBED', lib)
 207
 208         if python_SHLIBS is not None:
 209                 for lib in python_SHLIBS.split():
 210                         if lib.startswith('-l'):
 211                                 env.append_value('LIB_PYEMBED', lib[2:]) # strip '-l'
 212                         else:
 213                                 env.append_value('LINKFLAGS_PYEMBED', lib)
 214
 215         if Options.platform != 'darwin' and python_LDFLAGS:
 216                 env.append_value('LINKFLAGS_PYEMBED', python_LDFLAGS.split())
 217
 218         result = False
 219         name = 'python' + env['PYTHON_VERSION']
 220
 221         if python_LIBDIR is not None:
 222                 path = [python_LIBDIR]
 223                 conf.log.write("\n\n# Trying LIBDIR: %r\n" % path)
 224                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 225
 226         if not result and python_LIBPL is not None:
 227                 conf.log.write("\n\n# try again with -L$python_LIBPL (some systems don't install the python library in $prefix/lib)\n")
 228                 path = [python_LIBPL]
 229                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 230
 231         if not result:
 232                 conf.log.write("\n\n# try again with -L$prefix/libs, and pythonXY name rather than pythonX.Y (win32)\n")
 233                 path = [os.path.join(python_prefix, "libs")]
 234                 name = 'python' + env['PYTHON_VERSION'].replace('.', '')
 235                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 236
 237         if result:
 238                 env['LIBPATH_PYEMBED'] = path
 239                 env.append_value('LIB_PYEMBED', name)
 240         else:
 241                 conf.log.write("\n\n### LIB NOT FOUND\n")
 242
 243         # under certain conditions, python extensions must link to
 244         # python libraries, not just python embedding programs.
 245         if (sys.platform == 'win32' or sys.platform.startswith('os2')
 246                 or sys.platform == 'darwin' or Py_ENABLE_SHARED):
 247                 env['LIBPATH_PYEXT'] = env['LIBPATH_PYEMBED']
 248                 env['LIB_PYEXT'] = env['LIB_PYEMBED']
 249
 250         # We check that pythonX.Y-config exists, and if it exists we
 251         # use it to get only the includes, else fall back to distutils.
 252         python_config = conf.find_program(
 253                 'python%s-config' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 254                 var='PYTHON_CONFIG')
 255         if not python_config:
 256                 python_config = conf.find_program(
 257                         'python-config-%s' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 258                         var='PYTHON_CONFIG')
 259
 260         includes = []
 261         if python_config:
 262                 for incstr in Utils.cmd_output("%s %s --includes" % (python, python_config)).strip().split():
 263                         # strip the -I or /I
 264                         if (incstr.startswith('-I')
 265                             or incstr.startswith('/I')):
 266                                 incstr = incstr[2:]
 267                         # append include path, unless already given
 268                         if incstr not in includes:
 269                                 includes.append(incstr)
 270                 conf.log.write("Include path for Python extensions "
 271                                "(found via python-config --includes): %r\n" % (includes,))
 272                 env['CPPPATH_PYEXT'] = includes
 273                 env['CPPPATH_PYEMBED'] = includes
 274         else:
 275                 conf.log.write("Include path for Python extensions "
 276                                "(found via distutils module): %r\n" % (INCLUDEPY,))
 277                 env['CPPPATH_PYEXT'] = [INCLUDEPY]
 278                 env['CPPPATH_PYEMBED'] = [INCLUDEPY]
 279
 280         # Code using the Python API needs to be compiled with -fno-strict-aliasing
 281         if env['CC_NAME'] == 'gcc':
 282                 env.append_value('CCFLAGS_PYEMBED', '-fno-strict-aliasing')
 283                 env.append_value('CCFLAGS_PYEXT', '-fno-strict-aliasing')
 284         if env['CXX_NAME'] == 'gcc':
 285                 env.append_value('CXXFLAGS_PYEMBED', '-fno-strict-aliasing')
 286                 env.append_value('CXXFLAGS_PYEXT', '-fno-strict-aliasing')
 287
 288         # See if it compiles
 289         conf.check(define_name='HAVE_PYTHON_H',
 290                    uselib='PYEMBED', fragment=FRAG_2,
 291                    errmsg='Could not find the python development headers', mandatory=mandatory)
 292
 293 @conf
 294 def check_python_version(conf, minver=None):
 295         """
 296         Check if the python interpreter is found matching a given minimum version.
 297         minver should be a tuple, eg. to check for python >= 2.4.2 pass (2,4,2) as minver.
 298
 299         If successful, PYTHON_VERSION is defined as 'MAJOR.MINOR'
 300         (eg. '2.4') of the actual python version found, and PYTHONDIR is
 301         defined, pointing to the site-packages directory appropriate for
 302         this python version, where modules/packages/extensions should be
 303         installed.
 304         """
 305         assert minver is None or isinstance(minver, tuple)
 306         python = conf.env['PYTHON']
 307         if not python:
 308                 conf.fatal('could not find the python executable')
 309
 310         # Get python version string
 311         cmd = [python, "-c", "import sys\nfor x in sys.version_info: print(str(x))"]
 312         debug('python: Running python command %r' % cmd)
 313         proc = Utils.pproc.Popen(cmd, stdout=Utils.pproc.PIPE, shell=False)
 314         lines = proc.communicate()[0].split()
 315         assert len(lines) == 5, "found %i lines, expected 5: %r" % (len(lines), lines)
 316         pyver_tuple = (int(lines[0]), int(lines[1]), int(lines[2]), lines[3], int(lines[4]))
 317
 318         # compare python version with the minimum required
 319         result = (minver is None) or (pyver_tuple >= minver)
 320
 321         if result:
 322                 # define useful environment variables
 323                 pyver = '.'.join([str(x) for x in pyver_tuple[:2]])
 324                 conf.env['PYTHON_VERSION'] = pyver
 325
 326                 if 'PYTHONDIR' in conf.environ:
 327                         pydir = conf.environ['PYTHONDIR']
 328                 else:
 329                         if sys.platform == 'win32':
 330                                 (python_LIBDEST, pydir) = \
 331                                                 _get_python_variables(python,
 332                                                                                           ["get_config_var('LIBDEST') or ''",
 333                                                                                            "get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 334                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 335                         else:
 336                                 python_LIBDEST = None
 337                                 (pydir,) = \
 338                                                 _get_python_variables(python,
 339                                                                                           ["get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 340                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 341                         if python_LIBDEST is None:
 342                                 if conf.env['LIBDIR']:
 343                                         python_LIBDEST = os.path.join(conf.env['LIBDIR'], "python" + pyver)
 344                                 else:
 345                                         python_LIBDEST = os.path.join(conf.env['PREFIX'], "lib", "python" + pyver)
 346
 347                 if 'PYTHONARCHDIR' in conf.environ:
 348                         pyarchdir = conf.environ['PYTHONARCHDIR']
 349                 else:
 350                         (pyarchdir,) = _get_python_variables(python,
 351                                                                                         ["get_python_lib(plat_specific=1, standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 352                                                                                         ['from distutils.sysconfig import get_config_var, get_python_lib'])
 353                         if not pyarchdir:
 354                                 pyarchdir = pydir
 355
 356                 if hasattr(conf, 'define'): # conf.define is added by the C tool, so may not exist
 357                         conf.define('PYTHONDIR', pydir)
 358                         conf.define('PYTHONARCHDIR', pyarchdir)
 359
 360                 conf.env['PYTHONDIR'] = pydir
 361
 362         # Feedback
 363         pyver_full = '.'.join(map(str, pyver_tuple[:3]))
 364         if minver is None:
 365                 conf.check_message_custom('Python version', '', pyver_full)
 366         else:
 367                 minver_str = '.'.join(map(str, minver))
 368                 conf.check_message('Python version', ">= %s" % minver_str, result, option=pyver_full)
 369
 370         if not result:
 371                 conf.fatal('The python version is too old (%r)' % pyver_full)
 372
 373 @conf
 374 def check_python_module(conf, module_name):
 375         """
 376         Check if the selected python interpreter can import the given python module.
 377         """
 378         result = not Utils.pproc.Popen([conf.env['PYTHON'], "-c", "import %s" % module_name],
 379                            stderr=Utils.pproc.PIPE, stdout=Utils.pproc.PIPE).wait()
 380         conf.check_message('Python module', module_name, result)
 381         if not result:
 382                 conf.fatal('Could not find the python module %r' % module_name)
 383
 384 def detect(conf):
 385
 386         if not conf.env.PYTHON:
 387                 conf.env.PYTHON = sys.executable
 388
 389         python = conf.find_program('python', var='PYTHON')
 390         if not python:
 391                 conf.fatal('Could not find the path of the python executable')
 392
 393         if conf.env.PYTHON != sys.executable:
 394                 warn("python executable '%s' different from sys.executable '%s'" % (conf.env.PYTHON, sys.executable))
 395
 396         v = conf.env
 397         v['PYCMD'] = '"import sys, py_compile;py_compile.compile(sys.argv[1], sys.argv[2])"'
 398         v['PYFLAGS'] = ''
 399         v['PYFLAGS_OPT'] = '-O'
 400
 401         v['PYC'] = getattr(Options.options, 'pyc', 1)
 402         v['PYO'] = getattr(Options.options, 'pyo', 1)
 403
 404 def set_options(opt):
 405         opt.add_option('--nopyc',
 406                         action='store_false',
 407                         default=1,
 408                         help = 'Do not install bytecode compiled .pyc files (configuration) [Default:install]',
 409                         dest = 'pyc')
 410         opt.add_option('--nopyo',
 411                         action='store_false',
 412                         default=1,
 413                         help='Do not install optimised compiled .pyo files (configuration) [Default:install]',
 414                         dest='pyo')
 415