util.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361
  1. """
  2. Utility functions for
  3. - building and importing modules on test time, using a temporary location
  4. - detecting if compilers are present
  5. """
  6. import os
  7. import sys
  8. import subprocess
  9. import tempfile
  10. import shutil
  11. import atexit
  12. import textwrap
  13. import re
  14. import pytest
  15. from numpy.compat import asbytes, asstr
  16. from numpy.testing import temppath
  17. from importlib import import_module
  18. from hashlib import md5
  19. #
  20. # Maintaining a temporary module directory
  21. #
  22. _module_dir = None
  23. _module_num = 5403
  24. def _cleanup():
  25. global _module_dir
  26. if _module_dir is not None:
  27. try:
  28. sys.path.remove(_module_dir)
  29. except ValueError:
  30. pass
  31. try:
  32. shutil.rmtree(_module_dir)
  33. except (IOError, OSError):
  34. pass
  35. _module_dir = None
  36. def get_module_dir():
  37. global _module_dir
  38. if _module_dir is None:
  39. _module_dir = tempfile.mkdtemp()
  40. atexit.register(_cleanup)
  41. if _module_dir not in sys.path:
  42. sys.path.insert(0, _module_dir)
  43. return _module_dir
  44. def get_temp_module_name():
  45. # Assume single-threaded, and the module dir usable only by this thread
  46. global _module_num
  47. d = get_module_dir()
  48. name = "_test_ext_module_%d" % _module_num
  49. _module_num += 1
  50. if name in sys.modules:
  51. # this should not be possible, but check anyway
  52. raise RuntimeError("Temporary module name already in use.")
  53. return name
  54. def _memoize(func):
  55. memo = {}
  56. def wrapper(*a, **kw):
  57. key = repr((a, kw))
  58. if key not in memo:
  59. try:
  60. memo[key] = func(*a, **kw)
  61. except Exception as e:
  62. memo[key] = e
  63. raise
  64. ret = memo[key]
  65. if isinstance(ret, Exception):
  66. raise ret
  67. return ret
  68. wrapper.__name__ = func.__name__
  69. return wrapper
  70. #
  71. # Building modules
  72. #
  73. @_memoize
  74. def build_module(source_files, options=[], skip=[], only=[], module_name=None):
  75. """
  76. Compile and import a f2py module, built from the given files.
  77. """
  78. code = ("import sys; sys.path = %s; import numpy.f2py as f2py2e; "
  79. "f2py2e.main()" % repr(sys.path))
  80. d = get_module_dir()
  81. # Copy files
  82. dst_sources = []
  83. f2py_sources = []
  84. for fn in source_files:
  85. if not os.path.isfile(fn):
  86. raise RuntimeError("%s is not a file" % fn)
  87. dst = os.path.join(d, os.path.basename(fn))
  88. shutil.copyfile(fn, dst)
  89. dst_sources.append(dst)
  90. base, ext = os.path.splitext(dst)
  91. if ext in ('.f90', '.f', '.c', '.pyf'):
  92. f2py_sources.append(dst)
  93. # Prepare options
  94. if module_name is None:
  95. module_name = get_temp_module_name()
  96. f2py_opts = ['-c', '-m', module_name] + options + f2py_sources
  97. if skip:
  98. f2py_opts += ['skip:'] + skip
  99. if only:
  100. f2py_opts += ['only:'] + only
  101. # Build
  102. cwd = os.getcwd()
  103. try:
  104. os.chdir(d)
  105. cmd = [sys.executable, '-c', code] + f2py_opts
  106. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  107. stderr=subprocess.STDOUT)
  108. out, err = p.communicate()
  109. if p.returncode != 0:
  110. raise RuntimeError("Running f2py failed: %s\n%s"
  111. % (cmd[4:], asstr(out)))
  112. finally:
  113. os.chdir(cwd)
  114. # Partial cleanup
  115. for fn in dst_sources:
  116. os.unlink(fn)
  117. # Import
  118. return import_module(module_name)
  119. @_memoize
  120. def build_code(source_code, options=[], skip=[], only=[], suffix=None,
  121. module_name=None):
  122. """
  123. Compile and import Fortran code using f2py.
  124. """
  125. if suffix is None:
  126. suffix = '.f'
  127. with temppath(suffix=suffix) as path:
  128. with open(path, 'w') as f:
  129. f.write(source_code)
  130. return build_module([path], options=options, skip=skip, only=only,
  131. module_name=module_name)
  132. #
  133. # Check if compilers are available at all...
  134. #
  135. _compiler_status = None
  136. def _get_compiler_status():
  137. global _compiler_status
  138. if _compiler_status is not None:
  139. return _compiler_status
  140. _compiler_status = (False, False, False)
  141. # XXX: this is really ugly. But I don't know how to invoke Distutils
  142. # in a safer way...
  143. code = textwrap.dedent("""\
  144. import os
  145. import sys
  146. sys.path = %(syspath)s
  147. def configuration(parent_name='',top_path=None):
  148. global config
  149. from numpy.distutils.misc_util import Configuration
  150. config = Configuration('', parent_name, top_path)
  151. return config
  152. from numpy.distutils.core import setup
  153. setup(configuration=configuration)
  154. config_cmd = config.get_config_cmd()
  155. have_c = config_cmd.try_compile('void foo() {}')
  156. print('COMPILERS:%%d,%%d,%%d' %% (have_c,
  157. config.have_f77c(),
  158. config.have_f90c()))
  159. sys.exit(99)
  160. """)
  161. code = code % dict(syspath=repr(sys.path))
  162. tmpdir = tempfile.mkdtemp()
  163. try:
  164. script = os.path.join(tmpdir, 'setup.py')
  165. with open(script, 'w') as f:
  166. f.write(code)
  167. cmd = [sys.executable, 'setup.py', 'config']
  168. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  169. stderr=subprocess.STDOUT,
  170. cwd=tmpdir)
  171. out, err = p.communicate()
  172. finally:
  173. shutil.rmtree(tmpdir)
  174. m = re.search(br'COMPILERS:(\d+),(\d+),(\d+)', out)
  175. if m:
  176. _compiler_status = (bool(int(m.group(1))), bool(int(m.group(2))),
  177. bool(int(m.group(3))))
  178. # Finished
  179. return _compiler_status
  180. def has_c_compiler():
  181. return _get_compiler_status()[0]
  182. def has_f77_compiler():
  183. return _get_compiler_status()[1]
  184. def has_f90_compiler():
  185. return _get_compiler_status()[2]
  186. #
  187. # Building with distutils
  188. #
  189. @_memoize
  190. def build_module_distutils(source_files, config_code, module_name, **kw):
  191. """
  192. Build a module via distutils and import it.
  193. """
  194. from numpy.distutils.misc_util import Configuration
  195. from numpy.distutils.core import setup
  196. d = get_module_dir()
  197. # Copy files
  198. dst_sources = []
  199. for fn in source_files:
  200. if not os.path.isfile(fn):
  201. raise RuntimeError("%s is not a file" % fn)
  202. dst = os.path.join(d, os.path.basename(fn))
  203. shutil.copyfile(fn, dst)
  204. dst_sources.append(dst)
  205. # Build script
  206. config_code = textwrap.dedent(config_code).replace("\n", "\n ")
  207. code = textwrap.dedent("""\
  208. import os
  209. import sys
  210. sys.path = %(syspath)s
  211. def configuration(parent_name='',top_path=None):
  212. from numpy.distutils.misc_util import Configuration
  213. config = Configuration('', parent_name, top_path)
  214. %(config_code)s
  215. return config
  216. if __name__ == "__main__":
  217. from numpy.distutils.core import setup
  218. setup(configuration=configuration)
  219. """) % dict(config_code=config_code, syspath=repr(sys.path))
  220. script = os.path.join(d, get_temp_module_name() + '.py')
  221. dst_sources.append(script)
  222. with open(script, 'wb') as f:
  223. f.write(asbytes(code))
  224. # Build
  225. cwd = os.getcwd()
  226. try:
  227. os.chdir(d)
  228. cmd = [sys.executable, script, 'build_ext', '-i']
  229. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  230. stderr=subprocess.STDOUT)
  231. out, err = p.communicate()
  232. if p.returncode != 0:
  233. raise RuntimeError("Running distutils build failed: %s\n%s"
  234. % (cmd[4:], asstr(out)))
  235. finally:
  236. os.chdir(cwd)
  237. # Partial cleanup
  238. for fn in dst_sources:
  239. os.unlink(fn)
  240. # Import
  241. __import__(module_name)
  242. return sys.modules[module_name]
  243. #
  244. # Unittest convenience
  245. #
  246. class F2PyTest:
  247. code = None
  248. sources = None
  249. options = []
  250. skip = []
  251. only = []
  252. suffix = '.f'
  253. module = None
  254. module_name = None
  255. def setup(self):
  256. if sys.platform == 'win32':
  257. pytest.skip('Fails with MinGW64 Gfortran (Issue #9673)')
  258. if self.module is not None:
  259. return
  260. # Check compiler availability first
  261. if not has_c_compiler():
  262. pytest.skip("No C compiler available")
  263. codes = []
  264. if self.sources:
  265. codes.extend(self.sources)
  266. if self.code is not None:
  267. codes.append(self.suffix)
  268. needs_f77 = False
  269. needs_f90 = False
  270. for fn in codes:
  271. if fn.endswith('.f'):
  272. needs_f77 = True
  273. elif fn.endswith('.f90'):
  274. needs_f90 = True
  275. if needs_f77 and not has_f77_compiler():
  276. pytest.skip("No Fortran 77 compiler available")
  277. if needs_f90 and not has_f90_compiler():
  278. pytest.skip("No Fortran 90 compiler available")
  279. # Build the module
  280. if self.code is not None:
  281. self.module = build_code(self.code, options=self.options,
  282. skip=self.skip, only=self.only,
  283. suffix=self.suffix,
  284. module_name=self.module_name)
  285. if self.sources is not None:
  286. self.module = build_module(self.sources, options=self.options,
  287. skip=self.skip, only=self.only,
  288. module_name=self.module_name)