gnu.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556
  1. import re
  2. import os
  3. import sys
  4. import warnings
  5. import platform
  6. import tempfile
  7. import hashlib
  8. import base64
  9. import subprocess
  10. from subprocess import Popen, PIPE, STDOUT
  11. from numpy.distutils.exec_command import filepath_from_subprocess_output
  12. from numpy.distutils.fcompiler import FCompiler
  13. from distutils.version import LooseVersion
  14. compilers = ['GnuFCompiler', 'Gnu95FCompiler']
  15. TARGET_R = re.compile(r"Target: ([a-zA-Z0-9_\-]*)")
  16. # XXX: handle cross compilation
  17. def is_win64():
  18. return sys.platform == "win32" and platform.architecture()[0] == "64bit"
  19. if is_win64():
  20. #_EXTRAFLAGS = ["-fno-leading-underscore"]
  21. _EXTRAFLAGS = []
  22. else:
  23. _EXTRAFLAGS = []
  24. class GnuFCompiler(FCompiler):
  25. compiler_type = 'gnu'
  26. compiler_aliases = ('g77', )
  27. description = 'GNU Fortran 77 compiler'
  28. def gnu_version_match(self, version_string):
  29. """Handle the different versions of GNU fortran compilers"""
  30. # Strip warning(s) that may be emitted by gfortran
  31. while version_string.startswith('gfortran: warning'):
  32. version_string = version_string[version_string.find('\n') + 1:]
  33. # Gfortran versions from after 2010 will output a simple string
  34. # (usually "x.y", "x.y.z" or "x.y.z-q") for ``-dumpversion``; older
  35. # gfortrans may still return long version strings (``-dumpversion`` was
  36. # an alias for ``--version``)
  37. if len(version_string) <= 20:
  38. # Try to find a valid version string
  39. m = re.search(r'([0-9.]+)', version_string)
  40. if m:
  41. # g77 provides a longer version string that starts with GNU
  42. # Fortran
  43. if version_string.startswith('GNU Fortran'):
  44. return ('g77', m.group(1))
  45. # gfortran only outputs a version string such as #.#.#, so check
  46. # if the match is at the start of the string
  47. elif m.start() == 0:
  48. return ('gfortran', m.group(1))
  49. else:
  50. # Output probably from --version, try harder:
  51. m = re.search(r'GNU Fortran\s+95.*?([0-9-.]+)', version_string)
  52. if m:
  53. return ('gfortran', m.group(1))
  54. m = re.search(
  55. r'GNU Fortran.*?\-?([0-9-.]+\.[0-9-.]+)', version_string)
  56. if m:
  57. v = m.group(1)
  58. if v.startswith('0') or v.startswith('2') or v.startswith('3'):
  59. # the '0' is for early g77's
  60. return ('g77', v)
  61. else:
  62. # at some point in the 4.x series, the ' 95' was dropped
  63. # from the version string
  64. return ('gfortran', v)
  65. # If still nothing, raise an error to make the problem easy to find.
  66. err = 'A valid Fortran version was not found in this string:\n'
  67. raise ValueError(err + version_string)
  68. def version_match(self, version_string):
  69. v = self.gnu_version_match(version_string)
  70. if not v or v[0] != 'g77':
  71. return None
  72. return v[1]
  73. possible_executables = ['g77', 'f77']
  74. executables = {
  75. 'version_cmd' : [None, "-dumpversion"],
  76. 'compiler_f77' : [None, "-g", "-Wall", "-fno-second-underscore"],
  77. 'compiler_f90' : None, # Use --fcompiler=gnu95 for f90 codes
  78. 'compiler_fix' : None,
  79. 'linker_so' : [None, "-g", "-Wall"],
  80. 'archiver' : ["ar", "-cr"],
  81. 'ranlib' : ["ranlib"],
  82. 'linker_exe' : [None, "-g", "-Wall"]
  83. }
  84. module_dir_switch = None
  85. module_include_switch = None
  86. # Cygwin: f771: warning: -fPIC ignored for target (all code is
  87. # position independent)
  88. if os.name != 'nt' and sys.platform != 'cygwin':
  89. pic_flags = ['-fPIC']
  90. # use -mno-cygwin for g77 when Python is not Cygwin-Python
  91. if sys.platform == 'win32':
  92. for key in ['version_cmd', 'compiler_f77', 'linker_so', 'linker_exe']:
  93. executables[key].append('-mno-cygwin')
  94. g2c = 'g2c'
  95. suggested_f90_compiler = 'gnu95'
  96. def get_flags_linker_so(self):
  97. opt = self.linker_so[1:]
  98. if sys.platform == 'darwin':
  99. target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
  100. # If MACOSX_DEPLOYMENT_TARGET is set, we simply trust the value
  101. # and leave it alone. But, distutils will complain if the
  102. # environment's value is different from the one in the Python
  103. # Makefile used to build Python. We let disutils handle this
  104. # error checking.
  105. if not target:
  106. # If MACOSX_DEPLOYMENT_TARGET is not set in the environment,
  107. # we try to get it first from sysconfig and then
  108. # fall back to setting it to 10.9 This is a reasonable default
  109. # even when using the official Python dist and those derived
  110. # from it.
  111. import sysconfig
  112. target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
  113. if not target:
  114. target = '10.9'
  115. s = f'Env. variable MACOSX_DEPLOYMENT_TARGET set to {target}'
  116. warnings.warn(s, stacklevel=2)
  117. os.environ['MACOSX_DEPLOYMENT_TARGET'] = target
  118. opt.extend(['-undefined', 'dynamic_lookup', '-bundle'])
  119. else:
  120. opt.append("-shared")
  121. if sys.platform.startswith('sunos'):
  122. # SunOS often has dynamically loaded symbols defined in the
  123. # static library libg2c.a The linker doesn't like this. To
  124. # ignore the problem, use the -mimpure-text flag. It isn't
  125. # the safest thing, but seems to work. 'man gcc' says:
  126. # ".. Instead of using -mimpure-text, you should compile all
  127. # source code with -fpic or -fPIC."
  128. opt.append('-mimpure-text')
  129. return opt
  130. def get_libgcc_dir(self):
  131. try:
  132. output = subprocess.check_output(self.compiler_f77 +
  133. ['-print-libgcc-file-name'])
  134. except (OSError, subprocess.CalledProcessError):
  135. pass
  136. else:
  137. output = filepath_from_subprocess_output(output)
  138. return os.path.dirname(output)
  139. return None
  140. def get_libgfortran_dir(self):
  141. if sys.platform[:5] == 'linux':
  142. libgfortran_name = 'libgfortran.so'
  143. elif sys.platform == 'darwin':
  144. libgfortran_name = 'libgfortran.dylib'
  145. else:
  146. libgfortran_name = None
  147. libgfortran_dir = None
  148. if libgfortran_name:
  149. find_lib_arg = ['-print-file-name={0}'.format(libgfortran_name)]
  150. try:
  151. output = subprocess.check_output(
  152. self.compiler_f77 + find_lib_arg)
  153. except (OSError, subprocess.CalledProcessError):
  154. pass
  155. else:
  156. output = filepath_from_subprocess_output(output)
  157. libgfortran_dir = os.path.dirname(output)
  158. return libgfortran_dir
  159. def get_library_dirs(self):
  160. opt = []
  161. if sys.platform[:5] != 'linux':
  162. d = self.get_libgcc_dir()
  163. if d:
  164. # if windows and not cygwin, libg2c lies in a different folder
  165. if sys.platform == 'win32' and not d.startswith('/usr/lib'):
  166. d = os.path.normpath(d)
  167. path = os.path.join(d, "lib%s.a" % self.g2c)
  168. if not os.path.exists(path):
  169. root = os.path.join(d, *((os.pardir, ) * 4))
  170. d2 = os.path.abspath(os.path.join(root, 'lib'))
  171. path = os.path.join(d2, "lib%s.a" % self.g2c)
  172. if os.path.exists(path):
  173. opt.append(d2)
  174. opt.append(d)
  175. # For Macports / Linux, libgfortran and libgcc are not co-located
  176. lib_gfortran_dir = self.get_libgfortran_dir()
  177. if lib_gfortran_dir:
  178. opt.append(lib_gfortran_dir)
  179. return opt
  180. def get_libraries(self):
  181. opt = []
  182. d = self.get_libgcc_dir()
  183. if d is not None:
  184. g2c = self.g2c + '-pic'
  185. f = self.static_lib_format % (g2c, self.static_lib_extension)
  186. if not os.path.isfile(os.path.join(d, f)):
  187. g2c = self.g2c
  188. else:
  189. g2c = self.g2c
  190. if g2c is not None:
  191. opt.append(g2c)
  192. c_compiler = self.c_compiler
  193. if sys.platform == 'win32' and c_compiler and \
  194. c_compiler.compiler_type == 'msvc':
  195. opt.append('gcc')
  196. if sys.platform == 'darwin':
  197. opt.append('cc_dynamic')
  198. return opt
  199. def get_flags_debug(self):
  200. return ['-g']
  201. def get_flags_opt(self):
  202. v = self.get_version()
  203. if v and v <= '3.3.3':
  204. # With this compiler version building Fortran BLAS/LAPACK
  205. # with -O3 caused failures in lib.lapack heevr,syevr tests.
  206. opt = ['-O2']
  207. else:
  208. opt = ['-O3']
  209. opt.append('-funroll-loops')
  210. return opt
  211. def _c_arch_flags(self):
  212. """ Return detected arch flags from CFLAGS """
  213. from distutils import sysconfig
  214. try:
  215. cflags = sysconfig.get_config_vars()['CFLAGS']
  216. except KeyError:
  217. return []
  218. arch_re = re.compile(r"-arch\s+(\w+)")
  219. arch_flags = []
  220. for arch in arch_re.findall(cflags):
  221. arch_flags += ['-arch', arch]
  222. return arch_flags
  223. def get_flags_arch(self):
  224. return []
  225. def runtime_library_dir_option(self, dir):
  226. if sys.platform == 'win32':
  227. # Linux/Solaris/Unix support RPATH, Windows does not
  228. raise NotImplementedError
  229. # TODO: could use -Xlinker here, if it's supported
  230. assert "," not in dir
  231. if sys.platform == 'darwin':
  232. return f'-Wl,-rpath,{dir}'
  233. elif sys.platform[:3] == 'aix':
  234. # AIX RPATH is called LIBPATH
  235. return f'-Wl,-blibpath:{dir}'
  236. else:
  237. return f'-Wl,-rpath={dir}'
  238. class Gnu95FCompiler(GnuFCompiler):
  239. compiler_type = 'gnu95'
  240. compiler_aliases = ('gfortran', )
  241. description = 'GNU Fortran 95 compiler'
  242. def version_match(self, version_string):
  243. v = self.gnu_version_match(version_string)
  244. if not v or v[0] != 'gfortran':
  245. return None
  246. v = v[1]
  247. if LooseVersion(v) >= "4":
  248. # gcc-4 series releases do not support -mno-cygwin option
  249. pass
  250. else:
  251. # use -mno-cygwin flag for gfortran when Python is not
  252. # Cygwin-Python
  253. if sys.platform == 'win32':
  254. for key in [
  255. 'version_cmd', 'compiler_f77', 'compiler_f90',
  256. 'compiler_fix', 'linker_so', 'linker_exe'
  257. ]:
  258. self.executables[key].append('-mno-cygwin')
  259. return v
  260. possible_executables = ['gfortran', 'f95']
  261. executables = {
  262. 'version_cmd' : ["<F90>", "-dumpversion"],
  263. 'compiler_f77' : [None, "-Wall", "-g", "-ffixed-form",
  264. "-fno-second-underscore"] + _EXTRAFLAGS,
  265. 'compiler_f90' : [None, "-Wall", "-g",
  266. "-fno-second-underscore"] + _EXTRAFLAGS,
  267. 'compiler_fix' : [None, "-Wall", "-g","-ffixed-form",
  268. "-fno-second-underscore"] + _EXTRAFLAGS,
  269. 'linker_so' : ["<F90>", "-Wall", "-g"],
  270. 'archiver' : ["ar", "-cr"],
  271. 'ranlib' : ["ranlib"],
  272. 'linker_exe' : [None, "-Wall"]
  273. }
  274. module_dir_switch = '-J'
  275. module_include_switch = '-I'
  276. if sys.platform[:3] == 'aix':
  277. executables['linker_so'].append('-lpthread')
  278. if platform.architecture()[0][:2] == '64':
  279. for key in ['compiler_f77', 'compiler_f90','compiler_fix','linker_so', 'linker_exe']:
  280. executables[key].append('-maix64')
  281. g2c = 'gfortran'
  282. def _universal_flags(self, cmd):
  283. """Return a list of -arch flags for every supported architecture."""
  284. if not sys.platform == 'darwin':
  285. return []
  286. arch_flags = []
  287. # get arches the C compiler gets.
  288. c_archs = self._c_arch_flags()
  289. if "i386" in c_archs:
  290. c_archs[c_archs.index("i386")] = "i686"
  291. # check the arches the Fortran compiler supports, and compare with
  292. # arch flags from C compiler
  293. for arch in ["ppc", "i686", "x86_64", "ppc64"]:
  294. if _can_target(cmd, arch) and arch in c_archs:
  295. arch_flags.extend(["-arch", arch])
  296. return arch_flags
  297. def get_flags(self):
  298. flags = GnuFCompiler.get_flags(self)
  299. arch_flags = self._universal_flags(self.compiler_f90)
  300. if arch_flags:
  301. flags[:0] = arch_flags
  302. return flags
  303. def get_flags_linker_so(self):
  304. flags = GnuFCompiler.get_flags_linker_so(self)
  305. arch_flags = self._universal_flags(self.linker_so)
  306. if arch_flags:
  307. flags[:0] = arch_flags
  308. return flags
  309. def get_library_dirs(self):
  310. opt = GnuFCompiler.get_library_dirs(self)
  311. if sys.platform == 'win32':
  312. c_compiler = self.c_compiler
  313. if c_compiler and c_compiler.compiler_type == "msvc":
  314. target = self.get_target()
  315. if target:
  316. d = os.path.normpath(self.get_libgcc_dir())
  317. root = os.path.join(d, *((os.pardir, ) * 4))
  318. path = os.path.join(root, "lib")
  319. mingwdir = os.path.normpath(path)
  320. if os.path.exists(os.path.join(mingwdir, "libmingwex.a")):
  321. opt.append(mingwdir)
  322. # For Macports / Linux, libgfortran and libgcc are not co-located
  323. lib_gfortran_dir = self.get_libgfortran_dir()
  324. if lib_gfortran_dir:
  325. opt.append(lib_gfortran_dir)
  326. return opt
  327. def get_libraries(self):
  328. opt = GnuFCompiler.get_libraries(self)
  329. if sys.platform == 'darwin':
  330. opt.remove('cc_dynamic')
  331. if sys.platform == 'win32':
  332. c_compiler = self.c_compiler
  333. if c_compiler and c_compiler.compiler_type == "msvc":
  334. if "gcc" in opt:
  335. i = opt.index("gcc")
  336. opt.insert(i + 1, "mingwex")
  337. opt.insert(i + 1, "mingw32")
  338. c_compiler = self.c_compiler
  339. if c_compiler and c_compiler.compiler_type == "msvc":
  340. return []
  341. else:
  342. pass
  343. return opt
  344. def get_target(self):
  345. try:
  346. output = subprocess.check_output(self.compiler_f77 + ['-v'])
  347. except (OSError, subprocess.CalledProcessError):
  348. pass
  349. else:
  350. output = filepath_from_subprocess_output(output)
  351. m = TARGET_R.search(output)
  352. if m:
  353. return m.group(1)
  354. return ""
  355. def _hash_files(self, filenames):
  356. h = hashlib.sha1()
  357. for fn in filenames:
  358. with open(fn, 'rb') as f:
  359. while True:
  360. block = f.read(131072)
  361. if not block:
  362. break
  363. h.update(block)
  364. text = base64.b32encode(h.digest())
  365. text = text.decode('ascii')
  366. return text.rstrip('=')
  367. def _link_wrapper_lib(self, objects, output_dir, extra_dll_dir,
  368. chained_dlls, is_archive):
  369. """Create a wrapper shared library for the given objects
  370. Return an MSVC-compatible lib
  371. """
  372. c_compiler = self.c_compiler
  373. if c_compiler.compiler_type != "msvc":
  374. raise ValueError("This method only supports MSVC")
  375. object_hash = self._hash_files(list(objects) + list(chained_dlls))
  376. if is_win64():
  377. tag = 'win_amd64'
  378. else:
  379. tag = 'win32'
  380. basename = 'lib' + os.path.splitext(
  381. os.path.basename(objects[0]))[0][:8]
  382. root_name = basename + '.' + object_hash + '.gfortran-' + tag
  383. dll_name = root_name + '.dll'
  384. def_name = root_name + '.def'
  385. lib_name = root_name + '.lib'
  386. dll_path = os.path.join(extra_dll_dir, dll_name)
  387. def_path = os.path.join(output_dir, def_name)
  388. lib_path = os.path.join(output_dir, lib_name)
  389. if os.path.isfile(lib_path):
  390. # Nothing to do
  391. return lib_path, dll_path
  392. if is_archive:
  393. objects = (["-Wl,--whole-archive"] + list(objects) +
  394. ["-Wl,--no-whole-archive"])
  395. self.link_shared_object(
  396. objects,
  397. dll_name,
  398. output_dir=extra_dll_dir,
  399. extra_postargs=list(chained_dlls) + [
  400. '-Wl,--allow-multiple-definition',
  401. '-Wl,--output-def,' + def_path,
  402. '-Wl,--export-all-symbols',
  403. '-Wl,--enable-auto-import',
  404. '-static',
  405. '-mlong-double-64',
  406. ])
  407. # No PowerPC!
  408. if is_win64():
  409. specifier = '/MACHINE:X64'
  410. else:
  411. specifier = '/MACHINE:X86'
  412. # MSVC specific code
  413. lib_args = ['/def:' + def_path, '/OUT:' + lib_path, specifier]
  414. if not c_compiler.initialized:
  415. c_compiler.initialize()
  416. c_compiler.spawn([c_compiler.lib] + lib_args)
  417. return lib_path, dll_path
  418. def can_ccompiler_link(self, compiler):
  419. # MSVC cannot link objects compiled by GNU fortran
  420. return compiler.compiler_type not in ("msvc", )
  421. def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
  422. """
  423. Convert a set of object files that are not compatible with the default
  424. linker, to a file that is compatible.
  425. """
  426. if self.c_compiler.compiler_type == "msvc":
  427. # Compile a DLL and return the lib for the DLL as
  428. # the object. Also keep track of previous DLLs that
  429. # we have compiled so that we can link against them.
  430. # If there are .a archives, assume they are self-contained
  431. # static libraries, and build separate DLLs for each
  432. archives = []
  433. plain_objects = []
  434. for obj in objects:
  435. if obj.lower().endswith('.a'):
  436. archives.append(obj)
  437. else:
  438. plain_objects.append(obj)
  439. chained_libs = []
  440. chained_dlls = []
  441. for archive in archives[::-1]:
  442. lib, dll = self._link_wrapper_lib(
  443. [archive],
  444. output_dir,
  445. extra_dll_dir,
  446. chained_dlls=chained_dlls,
  447. is_archive=True)
  448. chained_libs.insert(0, lib)
  449. chained_dlls.insert(0, dll)
  450. if not plain_objects:
  451. return chained_libs
  452. lib, dll = self._link_wrapper_lib(
  453. plain_objects,
  454. output_dir,
  455. extra_dll_dir,
  456. chained_dlls=chained_dlls,
  457. is_archive=False)
  458. return [lib] + chained_libs
  459. else:
  460. raise ValueError("Unsupported C compiler")
  461. def _can_target(cmd, arch):
  462. """Return true if the architecture supports the -arch flag"""
  463. newcmd = cmd[:]
  464. fid, filename = tempfile.mkstemp(suffix=".f")
  465. os.close(fid)
  466. try:
  467. d = os.path.dirname(filename)
  468. output = os.path.splitext(filename)[0] + ".o"
  469. try:
  470. newcmd.extend(["-arch", arch, "-c", filename])
  471. p = Popen(newcmd, stderr=STDOUT, stdout=PIPE, cwd=d)
  472. p.communicate()
  473. return p.returncode == 0
  474. finally:
  475. if os.path.exists(output):
  476. os.remove(output)
  477. finally:
  478. os.remove(filename)
  479. return False
  480. if __name__ == '__main__':
  481. from distutils import log
  482. from numpy.distutils import customized_fcompiler
  483. log.set_verbosity(2)
  484. print(customized_fcompiler('gnu').get_version())
  485. try:
  486. print(customized_fcompiler('g95').get_version())
  487. except Exception as e:
  488. print(e)