setup_common.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. # Code common to build tools
  2. import sys
  3. import warnings
  4. import copy
  5. import textwrap
  6. from numpy.distutils.misc_util import mingw32
  7. #-------------------
  8. # Versioning support
  9. #-------------------
  10. # How to change C_API_VERSION ?
  11. # - increase C_API_VERSION value
  12. # - record the hash for the new C API with the cversions.py script
  13. # and add the hash to cversions.txt
  14. # The hash values are used to remind developers when the C API number was not
  15. # updated - generates a MismatchCAPIWarning warning which is turned into an
  16. # exception for released version.
  17. # Binary compatibility version number. This number is increased whenever the
  18. # C-API is changed such that binary compatibility is broken, i.e. whenever a
  19. # recompile of extension modules is needed.
  20. C_ABI_VERSION = 0x01000009
  21. # Minor API version. This number is increased whenever a change is made to the
  22. # C-API -- whether it breaks binary compatibility or not. Some changes, such
  23. # as adding a function pointer to the end of the function table, can be made
  24. # without breaking binary compatibility. In this case, only the C_API_VERSION
  25. # (*not* C_ABI_VERSION) would be increased. Whenever binary compatibility is
  26. # broken, both C_API_VERSION and C_ABI_VERSION should be increased.
  27. #
  28. # 0x00000008 - 1.7.x
  29. # 0x00000009 - 1.8.x
  30. # 0x00000009 - 1.9.x
  31. # 0x0000000a - 1.10.x
  32. # 0x0000000a - 1.11.x
  33. # 0x0000000a - 1.12.x
  34. # 0x0000000b - 1.13.x
  35. # 0x0000000c - 1.14.x
  36. # 0x0000000c - 1.15.x
  37. # 0x0000000d - 1.16.x
  38. C_API_VERSION = 0x0000000d
  39. class MismatchCAPIWarning(Warning):
  40. pass
  41. def is_released(config):
  42. """Return True if a released version of numpy is detected."""
  43. from distutils.version import LooseVersion
  44. v = config.get_version('../version.py')
  45. if v is None:
  46. raise ValueError("Could not get version")
  47. pv = LooseVersion(vstring=v).version
  48. if len(pv) > 3:
  49. return False
  50. return True
  51. def get_api_versions(apiversion, codegen_dir):
  52. """
  53. Return current C API checksum and the recorded checksum.
  54. Return current C API checksum and the recorded checksum for the given
  55. version of the C API version.
  56. """
  57. # Compute the hash of the current API as defined in the .txt files in
  58. # code_generators
  59. sys.path.insert(0, codegen_dir)
  60. try:
  61. m = __import__('genapi')
  62. numpy_api = __import__('numpy_api')
  63. curapi_hash = m.fullapi_hash(numpy_api.full_api)
  64. apis_hash = m.get_versions_hash()
  65. finally:
  66. del sys.path[0]
  67. return curapi_hash, apis_hash[apiversion]
  68. def check_api_version(apiversion, codegen_dir):
  69. """Emits a MismatchCAPIWarning if the C API version needs updating."""
  70. curapi_hash, api_hash = get_api_versions(apiversion, codegen_dir)
  71. # If different hash, it means that the api .txt files in
  72. # codegen_dir have been updated without the API version being
  73. # updated. Any modification in those .txt files should be reflected
  74. # in the api and eventually abi versions.
  75. # To compute the checksum of the current API, use numpy/core/cversions.py
  76. if not curapi_hash == api_hash:
  77. msg = ("API mismatch detected, the C API version "
  78. "numbers have to be updated. Current C api version is %d, "
  79. "with checksum %s, but recorded checksum for C API version %d "
  80. "in core/codegen_dir/cversions.txt is %s. If functions were "
  81. "added in the C API, you have to update C_API_VERSION in %s."
  82. )
  83. warnings.warn(msg % (apiversion, curapi_hash, apiversion, api_hash,
  84. __file__),
  85. MismatchCAPIWarning, stacklevel=2)
  86. # Mandatory functions: if not found, fail the build
  87. MANDATORY_FUNCS = ["sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs",
  88. "floor", "ceil", "sqrt", "log10", "log", "exp", "asin",
  89. "acos", "atan", "fmod", 'modf', 'frexp', 'ldexp']
  90. # Standard functions which may not be available and for which we have a
  91. # replacement implementation. Note that some of these are C99 functions.
  92. OPTIONAL_STDFUNCS = ["expm1", "log1p", "acosh", "asinh", "atanh",
  93. "rint", "trunc", "exp2", "log2", "hypot", "atan2", "pow",
  94. "copysign", "nextafter", "ftello", "fseeko",
  95. "strtoll", "strtoull", "cbrt", "strtold_l", "fallocate",
  96. "backtrace", "madvise"]
  97. OPTIONAL_HEADERS = [
  98. # sse headers only enabled automatically on amd64/x32 builds
  99. "xmmintrin.h", # SSE
  100. "emmintrin.h", # SSE2
  101. "immintrin.h", # AVX
  102. "features.h", # for glibc version linux
  103. "xlocale.h", # see GH#8367
  104. "dlfcn.h", # dladdr
  105. "sys/mman.h", #madvise
  106. ]
  107. # optional gcc compiler builtins and their call arguments and optional a
  108. # required header and definition name (HAVE_ prepended)
  109. # call arguments are required as the compiler will do strict signature checking
  110. OPTIONAL_INTRINSICS = [("__builtin_isnan", '5.'),
  111. ("__builtin_isinf", '5.'),
  112. ("__builtin_isfinite", '5.'),
  113. ("__builtin_bswap32", '5u'),
  114. ("__builtin_bswap64", '5u'),
  115. ("__builtin_expect", '5, 0'),
  116. ("__builtin_mul_overflow", '5, 5, (int*)5'),
  117. # MMX only needed for icc, but some clangs don't have it
  118. ("_m_from_int64", '0', "emmintrin.h"),
  119. ("_mm_load_ps", '(float*)0', "xmmintrin.h"), # SSE
  120. ("_mm_prefetch", '(float*)0, _MM_HINT_NTA',
  121. "xmmintrin.h"), # SSE
  122. ("_mm_load_pd", '(double*)0', "emmintrin.h"), # SSE2
  123. ("__builtin_prefetch", "(float*)0, 0, 3"),
  124. # check that the linker can handle avx
  125. ("__asm__ volatile", '"vpand %xmm1, %xmm2, %xmm3"',
  126. "stdio.h", "LINK_AVX"),
  127. ("__asm__ volatile", '"vpand %ymm1, %ymm2, %ymm3"',
  128. "stdio.h", "LINK_AVX2"),
  129. ("__asm__ volatile", '"vpaddd %zmm1, %zmm2, %zmm3"',
  130. "stdio.h", "LINK_AVX512F"),
  131. ("__asm__ volatile", '"xgetbv"', "stdio.h", "XGETBV"),
  132. ]
  133. # function attributes
  134. # tested via "int %s %s(void *);" % (attribute, name)
  135. # function name will be converted to HAVE_<upper-case-name> preprocessor macro
  136. OPTIONAL_FUNCTION_ATTRIBUTES = [('__attribute__((optimize("unroll-loops")))',
  137. 'attribute_optimize_unroll_loops'),
  138. ('__attribute__((optimize("O3")))',
  139. 'attribute_optimize_opt_3'),
  140. ('__attribute__((nonnull (1)))',
  141. 'attribute_nonnull'),
  142. ('__attribute__((target ("avx")))',
  143. 'attribute_target_avx'),
  144. ('__attribute__((target ("avx2")))',
  145. 'attribute_target_avx2'),
  146. ('__attribute__((target ("avx512f")))',
  147. 'attribute_target_avx512f'),
  148. ]
  149. # function attributes with intrinsics
  150. # To ensure your compiler can compile avx intrinsics with just the attributes
  151. # gcc 4.8.4 support attributes but not with intrisics
  152. # tested via "#include<%s> int %s %s(void *){code; return 0;};" % (header, attribute, name, code)
  153. # function name will be converted to HAVE_<upper-case-name> preprocessor macro
  154. OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS = [('__attribute__((target("avx2,fma")))',
  155. 'attribute_target_avx2_with_intrinsics',
  156. '__m256 temp = _mm256_set1_ps(1.0); temp = \
  157. _mm256_fmadd_ps(temp, temp, temp)',
  158. 'immintrin.h'),
  159. ('__attribute__((target("avx512f")))',
  160. 'attribute_target_avx512f_with_intrinsics',
  161. '__m512 temp = _mm512_set1_ps(1.0)',
  162. 'immintrin.h'),
  163. ]
  164. # variable attributes tested via "int %s a" % attribute
  165. OPTIONAL_VARIABLE_ATTRIBUTES = ["__thread", "__declspec(thread)"]
  166. # Subset of OPTIONAL_STDFUNCS which may already have HAVE_* defined by Python.h
  167. OPTIONAL_STDFUNCS_MAYBE = [
  168. "expm1", "log1p", "acosh", "atanh", "asinh", "hypot", "copysign",
  169. "ftello", "fseeko"
  170. ]
  171. # C99 functions: float and long double versions
  172. C99_FUNCS = [
  173. "sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs", "floor", "ceil",
  174. "rint", "trunc", "sqrt", "log10", "log", "log1p", "exp", "expm1",
  175. "asin", "acos", "atan", "asinh", "acosh", "atanh", "hypot", "atan2",
  176. "pow", "fmod", "modf", 'frexp', 'ldexp', "exp2", "log2", "copysign",
  177. "nextafter", "cbrt"
  178. ]
  179. C99_FUNCS_SINGLE = [f + 'f' for f in C99_FUNCS]
  180. C99_FUNCS_EXTENDED = [f + 'l' for f in C99_FUNCS]
  181. C99_COMPLEX_TYPES = [
  182. 'complex double', 'complex float', 'complex long double'
  183. ]
  184. C99_COMPLEX_FUNCS = [
  185. "cabs", "cacos", "cacosh", "carg", "casin", "casinh", "catan",
  186. "catanh", "ccos", "ccosh", "cexp", "cimag", "clog", "conj", "cpow",
  187. "cproj", "creal", "csin", "csinh", "csqrt", "ctan", "ctanh"
  188. ]
  189. def fname2def(name):
  190. return "HAVE_%s" % name.upper()
  191. def sym2def(symbol):
  192. define = symbol.replace(' ', '')
  193. return define.upper()
  194. def type2def(symbol):
  195. define = symbol.replace(' ', '_')
  196. return define.upper()
  197. # Code to detect long double representation taken from MPFR m4 macro
  198. def check_long_double_representation(cmd):
  199. cmd._check_compiler()
  200. body = LONG_DOUBLE_REPRESENTATION_SRC % {'type': 'long double'}
  201. # Disable whole program optimization (the default on vs2015, with python 3.5+)
  202. # which generates intermediary object files and prevents checking the
  203. # float representation.
  204. if sys.platform == "win32" and not mingw32():
  205. try:
  206. cmd.compiler.compile_options.remove("/GL")
  207. except (AttributeError, ValueError):
  208. pass
  209. # Disable multi-file interprocedural optimization in the Intel compiler on Linux
  210. # which generates intermediary object files and prevents checking the
  211. # float representation.
  212. elif (sys.platform != "win32"
  213. and cmd.compiler.compiler_type.startswith('intel')
  214. and '-ipo' in cmd.compiler.cc_exe):
  215. newcompiler = cmd.compiler.cc_exe.replace(' -ipo', '')
  216. cmd.compiler.set_executables(
  217. compiler=newcompiler,
  218. compiler_so=newcompiler,
  219. compiler_cxx=newcompiler,
  220. linker_exe=newcompiler,
  221. linker_so=newcompiler + ' -shared'
  222. )
  223. # We need to use _compile because we need the object filename
  224. src, obj = cmd._compile(body, None, None, 'c')
  225. try:
  226. ltype = long_double_representation(pyod(obj))
  227. return ltype
  228. except ValueError:
  229. # try linking to support CC="gcc -flto" or icc -ipo
  230. # struct needs to be volatile so it isn't optimized away
  231. # additionally "clang -flto" requires the foo struct to be used
  232. body = body.replace('struct', 'volatile struct')
  233. body += "int main(void) { return foo.before[0]; }\n"
  234. src, obj = cmd._compile(body, None, None, 'c')
  235. cmd.temp_files.append("_configtest")
  236. cmd.compiler.link_executable([obj], "_configtest")
  237. ltype = long_double_representation(pyod("_configtest"))
  238. return ltype
  239. finally:
  240. cmd._clean()
  241. LONG_DOUBLE_REPRESENTATION_SRC = r"""
  242. /* "before" is 16 bytes to ensure there's no padding between it and "x".
  243. * We're not expecting any "long double" bigger than 16 bytes or with
  244. * alignment requirements stricter than 16 bytes. */
  245. typedef %(type)s test_type;
  246. struct {
  247. char before[16];
  248. test_type x;
  249. char after[8];
  250. } foo = {
  251. { '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0',
  252. '\001', '\043', '\105', '\147', '\211', '\253', '\315', '\357' },
  253. -123456789.0,
  254. { '\376', '\334', '\272', '\230', '\166', '\124', '\062', '\020' }
  255. };
  256. """
  257. def pyod(filename):
  258. """Python implementation of the od UNIX utility (od -b, more exactly).
  259. Parameters
  260. ----------
  261. filename : str
  262. name of the file to get the dump from.
  263. Returns
  264. -------
  265. out : seq
  266. list of lines of od output
  267. Note
  268. ----
  269. We only implement enough to get the necessary information for long double
  270. representation, this is not intended as a compatible replacement for od.
  271. """
  272. out = []
  273. with open(filename, 'rb') as fid:
  274. yo2 = [oct(o)[2:] for o in fid.read()]
  275. for i in range(0, len(yo2), 16):
  276. line = ['%07d' % int(oct(i)[2:])]
  277. line.extend(['%03d' % int(c) for c in yo2[i:i+16]])
  278. out.append(" ".join(line))
  279. return out
  280. _BEFORE_SEQ = ['000', '000', '000', '000', '000', '000', '000', '000',
  281. '001', '043', '105', '147', '211', '253', '315', '357']
  282. _AFTER_SEQ = ['376', '334', '272', '230', '166', '124', '062', '020']
  283. _IEEE_DOUBLE_BE = ['301', '235', '157', '064', '124', '000', '000', '000']
  284. _IEEE_DOUBLE_LE = _IEEE_DOUBLE_BE[::-1]
  285. _INTEL_EXTENDED_12B = ['000', '000', '000', '000', '240', '242', '171', '353',
  286. '031', '300', '000', '000']
  287. _INTEL_EXTENDED_16B = ['000', '000', '000', '000', '240', '242', '171', '353',
  288. '031', '300', '000', '000', '000', '000', '000', '000']
  289. _MOTOROLA_EXTENDED_12B = ['300', '031', '000', '000', '353', '171',
  290. '242', '240', '000', '000', '000', '000']
  291. _IEEE_QUAD_PREC_BE = ['300', '031', '326', '363', '105', '100', '000', '000',
  292. '000', '000', '000', '000', '000', '000', '000', '000']
  293. _IEEE_QUAD_PREC_LE = _IEEE_QUAD_PREC_BE[::-1]
  294. _IBM_DOUBLE_DOUBLE_BE = (['301', '235', '157', '064', '124', '000', '000', '000'] +
  295. ['000'] * 8)
  296. _IBM_DOUBLE_DOUBLE_LE = (['000', '000', '000', '124', '064', '157', '235', '301'] +
  297. ['000'] * 8)
  298. def long_double_representation(lines):
  299. """Given a binary dump as given by GNU od -b, look for long double
  300. representation."""
  301. # Read contains a list of 32 items, each item is a byte (in octal
  302. # representation, as a string). We 'slide' over the output until read is of
  303. # the form before_seq + content + after_sequence, where content is the long double
  304. # representation:
  305. # - content is 12 bytes: 80 bits Intel representation
  306. # - content is 16 bytes: 80 bits Intel representation (64 bits) or quad precision
  307. # - content is 8 bytes: same as double (not implemented yet)
  308. read = [''] * 32
  309. saw = None
  310. for line in lines:
  311. # we skip the first word, as od -b output an index at the beginning of
  312. # each line
  313. for w in line.split()[1:]:
  314. read.pop(0)
  315. read.append(w)
  316. # If the end of read is equal to the after_sequence, read contains
  317. # the long double
  318. if read[-8:] == _AFTER_SEQ:
  319. saw = copy.copy(read)
  320. # if the content was 12 bytes, we only have 32 - 8 - 12 = 12
  321. # "before" bytes. In other words the first 4 "before" bytes went
  322. # past the sliding window.
  323. if read[:12] == _BEFORE_SEQ[4:]:
  324. if read[12:-8] == _INTEL_EXTENDED_12B:
  325. return 'INTEL_EXTENDED_12_BYTES_LE'
  326. if read[12:-8] == _MOTOROLA_EXTENDED_12B:
  327. return 'MOTOROLA_EXTENDED_12_BYTES_BE'
  328. # if the content was 16 bytes, we are left with 32-8-16 = 16
  329. # "before" bytes, so 8 went past the sliding window.
  330. elif read[:8] == _BEFORE_SEQ[8:]:
  331. if read[8:-8] == _INTEL_EXTENDED_16B:
  332. return 'INTEL_EXTENDED_16_BYTES_LE'
  333. elif read[8:-8] == _IEEE_QUAD_PREC_BE:
  334. return 'IEEE_QUAD_BE'
  335. elif read[8:-8] == _IEEE_QUAD_PREC_LE:
  336. return 'IEEE_QUAD_LE'
  337. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_LE:
  338. return 'IBM_DOUBLE_DOUBLE_LE'
  339. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_BE:
  340. return 'IBM_DOUBLE_DOUBLE_BE'
  341. # if the content was 8 bytes, left with 32-8-8 = 16 bytes
  342. elif read[:16] == _BEFORE_SEQ:
  343. if read[16:-8] == _IEEE_DOUBLE_LE:
  344. return 'IEEE_DOUBLE_LE'
  345. elif read[16:-8] == _IEEE_DOUBLE_BE:
  346. return 'IEEE_DOUBLE_BE'
  347. if saw is not None:
  348. raise ValueError("Unrecognized format (%s)" % saw)
  349. else:
  350. # We never detected the after_sequence
  351. raise ValueError("Could not lock sequences (%s)" % saw)
  352. def check_for_right_shift_internal_compiler_error(cmd):
  353. """
  354. On our arm CI, this fails with an internal compilation error
  355. The failure looks like the following, and can be reproduced on ARM64 GCC 5.4:
  356. <source>: In function 'right_shift':
  357. <source>:4:20: internal compiler error: in expand_shift_1, at expmed.c:2349
  358. ip1[i] = ip1[i] >> in2;
  359. ^
  360. Please submit a full bug report,
  361. with preprocessed source if appropriate.
  362. See <http://gcc.gnu.org/bugs.html> for instructions.
  363. Compiler returned: 1
  364. This function returns True if this compiler bug is present, and we need to
  365. turn off optimization for the function
  366. """
  367. cmd._check_compiler()
  368. has_optimize = cmd.try_compile(textwrap.dedent("""\
  369. __attribute__((optimize("O3"))) void right_shift() {}
  370. """), None, None)
  371. if not has_optimize:
  372. return False
  373. no_err = cmd.try_compile(textwrap.dedent("""\
  374. typedef long the_type; /* fails also for unsigned and long long */
  375. __attribute__((optimize("O3"))) void right_shift(the_type in2, the_type *ip1, int n) {
  376. for (int i = 0; i < n; i++) {
  377. if (in2 < (the_type)sizeof(the_type) * 8) {
  378. ip1[i] = ip1[i] >> in2;
  379. }
  380. }
  381. }
  382. """), None, None)
  383. return not no_err