buildtools/wafadmin/Tools/python.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2007 (ita)
   4 # Gustavo Carneiro (gjc), 2007
   5
   6 "Python support"
   7
   8 import os, sys
   9 import TaskGen, Utils, Options
  10 from Logs import debug, warn, info
  11 from TaskGen import extension, before, after, feature
  12 from Configure import conf
  13 from config_c import parse_flags
  14 import ccroot
  15
  16 EXT_PY = ['.py']
  17 FRAG_2 = '''
  18 #include "Python.h"
  19 #ifdef __cplusplus
  20 extern "C" {
  21 #endif
  22         void Py_Initialize(void);
  23         void Py_Finalize(void);
  24 #ifdef __cplusplus
  25 }
  26 #endif
  27 int main()
  28 {
  29    Py_Initialize();
  30    Py_Finalize();
  31    return 0;
  32 }
  33 '''
  34
  35 @feature('pyext')
  36 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars', 'apply_bundle')
  37 @after('vars_target_cshlib')
  38 def init_pyext(self):
  39         self.default_install_path = '${PYTHONARCHDIR}'
  40         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  41         if not 'PYEXT' in self.uselib:
  42                 self.uselib.append('PYEXT')
  43         self.env['MACBUNDLE'] = True
  44
  45 @before('apply_link', 'apply_lib_vars', 'apply_type_vars')
  46 @after('apply_bundle')
  47 @feature('pyext')
  48 def pyext_shlib_ext(self):
  49         # override shlib_PATTERN set by the osx module
  50         self.env['shlib_PATTERN'] = self.env['pyext_PATTERN']
  51
  52 @before('apply_incpaths', 'apply_lib_vars', 'apply_type_vars')
  53 @feature('pyembed')
  54 def init_pyembed(self):
  55         self.uselib = self.to_list(getattr(self, 'uselib', ''))
  56         if not 'PYEMBED' in self.uselib:
  57                 self.uselib.append('PYEMBED')
  58
  59 @extension(EXT_PY)
  60 def process_py(self, node):
  61         if not (self.bld.is_install and self.install_path):
  62                 return
  63         def inst_py(ctx):
  64                 install_pyfile(self, node)
  65         self.bld.add_post_fun(inst_py)
  66
  67 def install_pyfile(self, node):
  68         path = self.bld.get_install_path(self.install_path + os.sep + node.name, self.env)
  69
  70         self.bld.install_files(self.install_path, [node], self.env, self.chmod, postpone=False)
  71         if self.bld.is_install < 0:
  72                 info("* removing byte compiled python files")
  73                 for x in 'co':
  74                         try:
  75                                 os.remove(path + x)
  76                         except OSError:
  77                                 pass
  78
  79         if self.bld.is_install > 0:
  80                 if self.env['PYC'] or self.env['PYO']:
  81                         info("* byte compiling %r" % path)
  82
  83                 if self.env['PYC']:
  84                         program = ("""
  85 import sys, py_compile
  86 for pyfile in sys.argv[1:]:
  87         py_compile.compile(pyfile, pyfile + 'c')
  88 """)
  89                         argv = [self.env['PYTHON'], '-c', program, path]
  90                         ret = Utils.pproc.Popen(argv).wait()
  91                         if ret:
  92                                 raise Utils.WafError('bytecode compilation failed %r' % path)
  93
  94                 if self.env['PYO']:
  95                         program = ("""
  96 import sys, py_compile
  97 for pyfile in sys.argv[1:]:
  98         py_compile.compile(pyfile, pyfile + 'o')
  99 """)
 100                         argv = [self.env['PYTHON'], self.env['PYFLAGS_OPT'], '-c', program, path]
 101                         ret = Utils.pproc.Popen(argv).wait()
 102                         if ret:
 103                                 raise Utils.WafError('bytecode compilation failed %r' % path)
 104
 105 # COMPAT
 106 class py_taskgen(TaskGen.task_gen):
 107         def __init__(self, *k, **kw):
 108                 TaskGen.task_gen.__init__(self, *k, **kw)
 109
 110 @before('apply_core')
 111 @after('vars_target_cprogram', 'vars_target_cshlib')
 112 @feature('py')
 113 def init_py(self):
 114         self.default_install_path = '${PYTHONDIR}'
 115
 116 def _get_python_variables(python_exe, variables, imports=['import sys']):
 117         """Run a python interpreter and print some variables"""
 118         program = list(imports)
 119         program.append('')
 120         for v in variables:
 121                 program.append("print(repr(%s))" % v)
 122         os_env = dict(os.environ)
 123         try:
 124                 del os_env['MACOSX_DEPLOYMENT_TARGET'] # see comments in the OSX tool
 125         except KeyError:
 126                 pass
 127         proc = Utils.pproc.Popen([python_exe, "-c", '\n'.join(program)], stdout=Utils.pproc.PIPE, env=os_env)
 128         output = proc.communicate()[0].split("\n") # do not touch, python3
 129         if proc.returncode:
 130                 if Options.options.verbose:
 131                         warn("Python program to extract python configuration variables failed:\n%s"
 132                                        % '\n'.join(["line %03i: %s" % (lineno+1, line) for lineno, line in enumerate(program)]))
 133                 raise RuntimeError
 134         return_values = []
 135         for s in output:
 136                 s = s.strip()
 137                 if not s:
 138                         continue
 139                 if s == 'None':
 140                         return_values.append(None)
 141                 elif (s[0] == "'" and s[-1] == "'") or (s[0] == '"' and s[-1] == '"'):
 142                         return_values.append(eval(s))
 143                 elif s[0].isdigit():
 144                         return_values.append(int(s))
 145                 else: break
 146         return return_values
 147
 148 @conf
 149 def check_python_headers(conf, mandatory=True):
 150         """Check for headers and libraries necessary to extend or embed python.
 151
 152         On success the environment variables xxx_PYEXT and xxx_PYEMBED are added for uselib
 153
 154         PYEXT: for compiling python extensions
 155         PYEMBED: for embedding a python interpreter"""
 156
 157         if not conf.env['CC_NAME'] and not conf.env['CXX_NAME']:
 158                 conf.fatal('load a compiler first (gcc, g++, ..)')
 159
 160         if not conf.env['PYTHON_VERSION']:
 161                 conf.check_python_version()
 162
 163         env = conf.env
 164         python = env['PYTHON']
 165         if not python:
 166                 conf.fatal('could not find the python executable')
 167
 168         ## On Mac OSX we need to use mac bundles for python plugins
 169         if Options.platform == 'darwin':
 170                 conf.check_tool('osx')
 171
 172         try:
 173                 # Get some python configuration variables using distutils
 174                 v = 'prefix SO SYSLIBS LDFLAGS SHLIBS LIBDIR LIBPL INCLUDEPY Py_ENABLE_SHARED MACOSX_DEPLOYMENT_TARGET'.split()
 175                 (python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 176                  python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED,
 177                  python_MACOSX_DEPLOYMENT_TARGET) = \
 178                         _get_python_variables(python, ["get_config_var('%s') or ''" % x for x in v],
 179                                               ['from distutils.sysconfig import get_config_var'])
 180         except RuntimeError:
 181                 conf.fatal("Python development headers not found (-v for details).")
 182
 183         conf.log.write("""Configuration returned from %r:
 184 python_prefix = %r
 185 python_SO = %r
 186 python_SYSLIBS = %r
 187 python_LDFLAGS = %r
 188 python_SHLIBS = %r
 189 python_LIBDIR = %r
 190 python_LIBPL = %r
 191 INCLUDEPY = %r
 192 Py_ENABLE_SHARED = %r
 193 MACOSX_DEPLOYMENT_TARGET = %r
 194 """ % (python, python_prefix, python_SO, python_SYSLIBS, python_LDFLAGS, python_SHLIBS,
 195         python_LIBDIR, python_LIBPL, INCLUDEPY, Py_ENABLE_SHARED, python_MACOSX_DEPLOYMENT_TARGET))
 196
 197         # Allow some python overrides from env vars for cross-compiling
 198         os_env = dict(os.environ)
 199
 200         override_python_LDFLAGS = os_env.get('python_LDFLAGS', None)
 201         if override_python_LDFLAGS is not None:
 202                 conf.log.write("python_LDFLAGS override from environment = %r\n" % (override_python_LDFLAGS))
 203                 python_LDFLAGS = override_python_LDFLAGS
 204
 205         override_python_LIBDIR = os_env.get('python_LIBDIR', None)
 206         if override_python_LIBDIR is not None:
 207                 conf.log.write("python_LIBDIR override from environment = %r\n" % (override_python_LIBDIR))
 208                 python_LIBDIR = override_python_LIBDIR
 209
 210         if python_MACOSX_DEPLOYMENT_TARGET:
 211                 conf.env['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 212                 conf.environ['MACOSX_DEPLOYMENT_TARGET'] = python_MACOSX_DEPLOYMENT_TARGET
 213
 214         env['pyext_PATTERN'] = '%s'+python_SO
 215
 216         # Check for python libraries for embedding
 217         if python_SYSLIBS is not None:
 218                 for lib in python_SYSLIBS.split():
 219                         if lib.startswith('-l'):
 220                                 lib = lib[2:] # strip '-l'
 221                         env.append_value('LIB_PYEMBED', lib)
 222
 223         if python_SHLIBS is not None:
 224                 for lib in python_SHLIBS.split():
 225                         if lib.startswith('-l'):
 226                                 env.append_value('LIB_PYEMBED', lib[2:]) # strip '-l'
 227                         else:
 228                                 env.append_value('LINKFLAGS_PYEMBED', lib)
 229
 230         if Options.platform != 'darwin' and python_LDFLAGS:
 231                 parse_flags(python_LDFLAGS, 'PYEMBED', env)
 232
 233         result = False
 234         name = 'python' + env['PYTHON_VERSION']
 235
 236         if python_LIBDIR is not None:
 237                 path = [python_LIBDIR]
 238                 conf.log.write("\n\n# Trying LIBDIR: %r\n" % path)
 239                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 240
 241         if not result and python_LIBPL is not None:
 242                 conf.log.write("\n\n# try again with -L$python_LIBPL (some systems don't install the python library in $prefix/lib)\n")
 243                 path = [python_LIBPL]
 244                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 245
 246         if not result:
 247                 conf.log.write("\n\n# try again with -L$prefix/libs, and pythonXY name rather than pythonX.Y (win32)\n")
 248                 path = [os.path.join(python_prefix, "libs")]
 249                 name = 'python' + env['PYTHON_VERSION'].replace('.', '')
 250                 result = conf.check(lib=name, uselib='PYEMBED', libpath=path)
 251
 252         if result:
 253                 if not ccroot.is_standard_libpath(env, path[0]):
 254                         env['LIBPATH_PYEMBED'] = path
 255                 env.append_value('LIB_PYEMBED', name)
 256         else:
 257                 conf.log.write("\n\n### LIB NOT FOUND\n")
 258
 259         # under certain conditions, python extensions must link to
 260         # python libraries, not just python embedding programs.
 261         if (sys.platform == 'win32' or sys.platform.startswith('os2')
 262                 or sys.platform == 'darwin' or Py_ENABLE_SHARED):
 263                 env['LIBPATH_PYEXT'] = env['LIBPATH_PYEMBED']
 264                 env['LIB_PYEXT'] = env['LIB_PYEMBED']
 265
 266         # We check that pythonX.Y-config exists, and if it exists we
 267         # use it to get only the includes, else fall back to distutils.
 268         python_config = conf.find_program(
 269                 'python%s-config' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 270                 var='PYTHON_CONFIG')
 271         if not python_config:
 272                 python_config = conf.find_program(
 273                         'python-config-%s' % ('.'.join(env['PYTHON_VERSION'].split('.')[:2])),
 274                         var='PYTHON_CONFIG')
 275
 276         includes = []
 277         if python_config:
 278                 for incstr in Utils.cmd_output("%s --includes" % (python_config,)).strip().split():
 279                         # strip the -I or /I
 280                         if (incstr.startswith('-I')
 281                             or incstr.startswith('/I')):
 282                                 incstr = incstr[2:]
 283                         # append include path, unless already given
 284                         if incstr not in includes:
 285                                 includes.append(incstr)
 286                 conf.log.write("Include path for Python extensions "
 287                                "(found via python-config --includes): %r\n" % (includes,))
 288                 env['CPPPATH_PYEXT'] = includes
 289                 env['CPPPATH_PYEMBED'] = includes
 290         else:
 291                 conf.log.write("Include path for Python extensions "
 292                                "(found via distutils module): %r\n" % (INCLUDEPY,))
 293                 env['CPPPATH_PYEXT'] = [INCLUDEPY]
 294                 env['CPPPATH_PYEMBED'] = [INCLUDEPY]
 295
 296         # Code using the Python API needs to be compiled with -fno-strict-aliasing
 297         if env['CC_NAME'] == 'gcc':
 298                 env.append_value('CCFLAGS_PYEMBED', '-fno-strict-aliasing')
 299                 env.append_value('CCFLAGS_PYEXT', '-fno-strict-aliasing')
 300         if env['CXX_NAME'] == 'gcc':
 301                 env.append_value('CXXFLAGS_PYEMBED', '-fno-strict-aliasing')
 302                 env.append_value('CXXFLAGS_PYEXT', '-fno-strict-aliasing')
 303
 304         # See if it compiles
 305         conf.check(define_name='HAVE_PYTHON_H',
 306                    uselib='PYEMBED', fragment=FRAG_2,
 307                    errmsg='Could not find the python development headers', mandatory=mandatory)
 308
 309 @conf
 310 def check_python_version(conf, minver=None):
 311         """
 312         Check if the python interpreter is found matching a given minimum version.
 313         minver should be a tuple, eg. to check for python >= 2.4.2 pass (2,4,2) as minver.
 314
 315         If successful, PYTHON_VERSION is defined as 'MAJOR.MINOR'
 316         (eg. '2.4') of the actual python version found, and PYTHONDIR is
 317         defined, pointing to the site-packages directory appropriate for
 318         this python version, where modules/packages/extensions should be
 319         installed.
 320         """
 321         assert minver is None or isinstance(minver, tuple)
 322         python = conf.env['PYTHON']
 323         if not python:
 324                 conf.fatal('could not find the python executable')
 325
 326         # Get python version string
 327         cmd = [python, "-c", "import sys\nfor x in sys.version_info: print(str(x))"]
 328         debug('python: Running python command %r' % cmd)
 329         proc = Utils.pproc.Popen(cmd, stdout=Utils.pproc.PIPE, shell=False)
 330         lines = proc.communicate()[0].split()
 331         assert len(lines) == 5, "found %i lines, expected 5: %r" % (len(lines), lines)
 332         pyver_tuple = (int(lines[0]), int(lines[1]), int(lines[2]), lines[3], int(lines[4]))
 333
 334         # compare python version with the minimum required
 335         result = (minver is None) or (pyver_tuple >= minver)
 336
 337         if result:
 338                 # define useful environment variables
 339                 pyver = '.'.join([str(x) for x in pyver_tuple[:2]])
 340                 conf.env['PYTHON_VERSION'] = pyver
 341
 342                 if 'PYTHONDIR' in conf.environ:
 343                         pydir = conf.environ['PYTHONDIR']
 344                 else:
 345                         if sys.platform == 'win32':
 346                                 (python_LIBDEST, pydir) = \
 347                                                 _get_python_variables(python,
 348                                                                                           ["get_config_var('LIBDEST') or ''",
 349                                                                                            "get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 350                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 351                         else:
 352                                 python_LIBDEST = None
 353                                 (pydir,) = \
 354                                                 _get_python_variables(python,
 355                                                                                           ["get_python_lib(standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 356                                                                                           ['from distutils.sysconfig import get_config_var, get_python_lib'])
 357                         if python_LIBDEST is None:
 358                                 if conf.env['LIBDIR']:
 359                                         python_LIBDEST = os.path.join(conf.env['LIBDIR'], "python" + pyver)
 360                                 else:
 361                                         python_LIBDEST = os.path.join(conf.env['PREFIX'], "lib", "python" + pyver)
 362
 363                 if 'PYTHONARCHDIR' in conf.environ:
 364                         pyarchdir = conf.environ['PYTHONARCHDIR']
 365                 else:
 366                         (pyarchdir,) = _get_python_variables(python,
 367                                                                                         ["get_python_lib(plat_specific=1, standard_lib=0, prefix=%r) or ''" % conf.env['PREFIX']],
 368                                                                                         ['from distutils.sysconfig import get_config_var, get_python_lib'])
 369                         if not pyarchdir:
 370                                 pyarchdir = pydir
 371
 372                 if hasattr(conf, 'define'): # conf.define is added by the C tool, so may not exist
 373                         conf.define('PYTHONDIR', pydir)
 374                         conf.define('PYTHONARCHDIR', pyarchdir)
 375
 376                 conf.env['PYTHONDIR'] = pydir
 377
 378         # Feedback
 379         pyver_full = '.'.join(map(str, pyver_tuple[:3]))
 380         if minver is None:
 381                 conf.check_message_custom('Python version', '', pyver_full)
 382         else:
 383                 minver_str = '.'.join(map(str, minver))
 384                 conf.check_message('Python version', ">= %s" % minver_str, result, option=pyver_full)
 385
 386         if not result:
 387                 conf.fatal('The python version is too old (%r)' % pyver_full)
 388
 389 @conf
 390 def check_python_module(conf, module_name):
 391         """
 392         Check if the selected python interpreter can import the given python module.
 393         """
 394         result = not Utils.pproc.Popen([conf.env['PYTHON'], "-c", "import %s" % module_name],
 395                            stderr=Utils.pproc.PIPE, stdout=Utils.pproc.PIPE).wait()
 396         conf.check_message('Python module', module_name, result)
 397         if not result:
 398                 conf.fatal('Could not find the python module %r' % module_name)
 399
 400 def detect(conf):
 401
 402         if not conf.env.PYTHON:
 403                 conf.env.PYTHON = sys.executable
 404
 405         python = conf.find_program('python', var='PYTHON')
 406         if not python:
 407                 conf.fatal('Could not find the path of the python executable')
 408
 409         if conf.env.PYTHON != sys.executable:
 410                 warn("python executable '%s' different from sys.executable '%s'" % (conf.env.PYTHON, sys.executable))
 411
 412         v = conf.env
 413         v['PYCMD'] = '"import sys, py_compile;py_compile.compile(sys.argv[1], sys.argv[2])"'
 414         v['PYFLAGS'] = ''
 415         v['PYFLAGS_OPT'] = '-O'
 416
 417         v['PYC'] = getattr(Options.options, 'pyc', 1)
 418         v['PYO'] = getattr(Options.options, 'pyo', 1)
 419
 420 def set_options(opt):
 421         opt.add_option('--nopyc',
 422                         action='store_false',
 423                         default=1,
 424                         help = 'Do not install bytecode compiled .pyc files (configuration) [Default:install]',
 425                         dest = 'pyc')
 426         opt.add_option('--nopyo',
 427                         action='store_false',
 428                         default=1,
 429                         help='Do not install optimised compiled .pyo files (configuration) [Default:install]',
 430                         dest='pyo')
 431