build_py.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. """distutils.command.build_py
  2. Implements the Distutils 'build_py' command."""
  3. import os
  4. import importlib.util
  5. import sys
  6. import glob
  7. from distutils.core import Command
  8. from distutils.errors import *
  9. from distutils.util import convert_path, Mixin2to3
  10. from distutils import log
  11. class build_py (Command):
  12. description = "\"build\" pure Python modules (copy to build directory)"
  13. user_options = [
  14. ('build-lib=', 'd', "directory to \"build\" (copy) to"),
  15. ('compile', 'c', "compile .py to .pyc"),
  16. ('no-compile', None, "don't compile .py files [default]"),
  17. ('optimize=', 'O',
  18. "also compile with optimization: -O1 for \"python -O\", "
  19. "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
  20. ('force', 'f', "forcibly build everything (ignore file timestamps)"),
  21. ]
  22. boolean_options = ['compile', 'force']
  23. negative_opt = {'no-compile' : 'compile'}
  24. def initialize_options(self):
  25. self.build_lib = None
  26. self.py_modules = None
  27. self.package = None
  28. self.package_data = None
  29. self.package_dir = None
  30. self.compile = 0
  31. self.optimize = 0
  32. self.force = None
  33. def finalize_options(self):
  34. self.set_undefined_options('build',
  35. ('build_lib', 'build_lib'),
  36. ('force', 'force'))
  37. # Get the distribution options that are aliases for build_py
  38. # options -- list of packages and list of modules.
  39. self.packages = self.distribution.packages
  40. self.py_modules = self.distribution.py_modules
  41. self.package_data = self.distribution.package_data
  42. self.package_dir = {}
  43. if self.distribution.package_dir:
  44. for name, path in self.distribution.package_dir.items():
  45. self.package_dir[name] = convert_path(path)
  46. self.data_files = self.get_data_files()
  47. # Ick, copied straight from install_lib.py (fancy_getopt needs a
  48. # type system! Hell, *everything* needs a type system!!!)
  49. if not isinstance(self.optimize, int):
  50. try:
  51. self.optimize = int(self.optimize)
  52. assert 0 <= self.optimize <= 2
  53. except (ValueError, AssertionError):
  54. raise DistutilsOptionError("optimize must be 0, 1, or 2")
  55. def run(self):
  56. # XXX copy_file by default preserves atime and mtime. IMHO this is
  57. # the right thing to do, but perhaps it should be an option -- in
  58. # particular, a site administrator might want installed files to
  59. # reflect the time of installation rather than the last
  60. # modification time before the installed release.
  61. # XXX copy_file by default preserves mode, which appears to be the
  62. # wrong thing to do: if a file is read-only in the working
  63. # directory, we want it to be installed read/write so that the next
  64. # installation of the same module distribution can overwrite it
  65. # without problems. (This might be a Unix-specific issue.) Thus
  66. # we turn off 'preserve_mode' when copying to the build directory,
  67. # since the build directory is supposed to be exactly what the
  68. # installation will look like (ie. we preserve mode when
  69. # installing).
  70. # Two options control which modules will be installed: 'packages'
  71. # and 'py_modules'. The former lets us work with whole packages, not
  72. # specifying individual modules at all; the latter is for
  73. # specifying modules one-at-a-time.
  74. if self.py_modules:
  75. self.build_modules()
  76. if self.packages:
  77. self.build_packages()
  78. self.build_package_data()
  79. self.byte_compile(self.get_outputs(include_bytecode=0))
  80. def get_data_files(self):
  81. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  82. data = []
  83. if not self.packages:
  84. return data
  85. for package in self.packages:
  86. # Locate package source directory
  87. src_dir = self.get_package_dir(package)
  88. # Compute package build directory
  89. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  90. # Length of path to strip from found files
  91. plen = 0
  92. if src_dir:
  93. plen = len(src_dir)+1
  94. # Strip directory from globbed filenames
  95. filenames = [
  96. file[plen:] for file in self.find_data_files(package, src_dir)
  97. ]
  98. data.append((package, src_dir, build_dir, filenames))
  99. return data
  100. def find_data_files(self, package, src_dir):
  101. """Return filenames for package's data files in 'src_dir'"""
  102. globs = (self.package_data.get('', [])
  103. + self.package_data.get(package, []))
  104. files = []
  105. for pattern in globs:
  106. # Each pattern has to be converted to a platform-specific path
  107. filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern)))
  108. # Files that match more than one pattern are only added once
  109. files.extend([fn for fn in filelist if fn not in files
  110. and os.path.isfile(fn)])
  111. return files
  112. def build_package_data(self):
  113. """Copy data files into build directory"""
  114. lastdir = None
  115. for package, src_dir, build_dir, filenames in self.data_files:
  116. for filename in filenames:
  117. target = os.path.join(build_dir, filename)
  118. self.mkpath(os.path.dirname(target))
  119. self.copy_file(os.path.join(src_dir, filename), target,
  120. preserve_mode=False)
  121. def get_package_dir(self, package):
  122. """Return the directory, relative to the top of the source
  123. distribution, where package 'package' should be found
  124. (at least according to the 'package_dir' option, if any)."""
  125. path = package.split('.')
  126. if not self.package_dir:
  127. if path:
  128. return os.path.join(*path)
  129. else:
  130. return ''
  131. else:
  132. tail = []
  133. while path:
  134. try:
  135. pdir = self.package_dir['.'.join(path)]
  136. except KeyError:
  137. tail.insert(0, path[-1])
  138. del path[-1]
  139. else:
  140. tail.insert(0, pdir)
  141. return os.path.join(*tail)
  142. else:
  143. # Oops, got all the way through 'path' without finding a
  144. # match in package_dir. If package_dir defines a directory
  145. # for the root (nameless) package, then fallback on it;
  146. # otherwise, we might as well have not consulted
  147. # package_dir at all, as we just use the directory implied
  148. # by 'tail' (which should be the same as the original value
  149. # of 'path' at this point).
  150. pdir = self.package_dir.get('')
  151. if pdir is not None:
  152. tail.insert(0, pdir)
  153. if tail:
  154. return os.path.join(*tail)
  155. else:
  156. return ''
  157. def check_package(self, package, package_dir):
  158. # Empty dir name means current directory, which we can probably
  159. # assume exists. Also, os.path.exists and isdir don't know about
  160. # my "empty string means current dir" convention, so we have to
  161. # circumvent them.
  162. if package_dir != "":
  163. if not os.path.exists(package_dir):
  164. raise DistutilsFileError(
  165. "package directory '%s' does not exist" % package_dir)
  166. if not os.path.isdir(package_dir):
  167. raise DistutilsFileError(
  168. "supposed package directory '%s' exists, "
  169. "but is not a directory" % package_dir)
  170. # Require __init__.py for all but the "root package"
  171. if package:
  172. init_py = os.path.join(package_dir, "__init__.py")
  173. if os.path.isfile(init_py):
  174. return init_py
  175. else:
  176. log.warn(("package init file '%s' not found " +
  177. "(or not a regular file)"), init_py)
  178. # Either not in a package at all (__init__.py not expected), or
  179. # __init__.py doesn't exist -- so don't return the filename.
  180. return None
  181. def check_module(self, module, module_file):
  182. if not os.path.isfile(module_file):
  183. log.warn("file %s (for module %s) not found", module_file, module)
  184. return False
  185. else:
  186. return True
  187. def find_package_modules(self, package, package_dir):
  188. self.check_package(package, package_dir)
  189. module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py"))
  190. modules = []
  191. setup_script = os.path.abspath(self.distribution.script_name)
  192. for f in module_files:
  193. abs_f = os.path.abspath(f)
  194. if abs_f != setup_script:
  195. module = os.path.splitext(os.path.basename(f))[0]
  196. modules.append((package, module, f))
  197. else:
  198. self.debug_print("excluding %s" % setup_script)
  199. return modules
  200. def find_modules(self):
  201. """Finds individually-specified Python modules, ie. those listed by
  202. module name in 'self.py_modules'. Returns a list of tuples (package,
  203. module_base, filename): 'package' is a tuple of the path through
  204. package-space to the module; 'module_base' is the bare (no
  205. packages, no dots) module name, and 'filename' is the path to the
  206. ".py" file (relative to the distribution root) that implements the
  207. module.
  208. """
  209. # Map package names to tuples of useful info about the package:
  210. # (package_dir, checked)
  211. # package_dir - the directory where we'll find source files for
  212. # this package
  213. # checked - true if we have checked that the package directory
  214. # is valid (exists, contains __init__.py, ... ?)
  215. packages = {}
  216. # List of (package, module, filename) tuples to return
  217. modules = []
  218. # We treat modules-in-packages almost the same as toplevel modules,
  219. # just the "package" for a toplevel is empty (either an empty
  220. # string or empty list, depending on context). Differences:
  221. # - don't check for __init__.py in directory for empty package
  222. for module in self.py_modules:
  223. path = module.split('.')
  224. package = '.'.join(path[0:-1])
  225. module_base = path[-1]
  226. try:
  227. (package_dir, checked) = packages[package]
  228. except KeyError:
  229. package_dir = self.get_package_dir(package)
  230. checked = 0
  231. if not checked:
  232. init_py = self.check_package(package, package_dir)
  233. packages[package] = (package_dir, 1)
  234. if init_py:
  235. modules.append((package, "__init__", init_py))
  236. # XXX perhaps we should also check for just .pyc files
  237. # (so greedy closed-source bastards can distribute Python
  238. # modules too)
  239. module_file = os.path.join(package_dir, module_base + ".py")
  240. if not self.check_module(module, module_file):
  241. continue
  242. modules.append((package, module_base, module_file))
  243. return modules
  244. def find_all_modules(self):
  245. """Compute the list of all modules that will be built, whether
  246. they are specified one-module-at-a-time ('self.py_modules') or
  247. by whole packages ('self.packages'). Return a list of tuples
  248. (package, module, module_file), just like 'find_modules()' and
  249. 'find_package_modules()' do."""
  250. modules = []
  251. if self.py_modules:
  252. modules.extend(self.find_modules())
  253. if self.packages:
  254. for package in self.packages:
  255. package_dir = self.get_package_dir(package)
  256. m = self.find_package_modules(package, package_dir)
  257. modules.extend(m)
  258. return modules
  259. def get_source_files(self):
  260. return [module[-1] for module in self.find_all_modules()]
  261. def get_module_outfile(self, build_dir, package, module):
  262. outfile_path = [build_dir] + list(package) + [module + ".py"]
  263. return os.path.join(*outfile_path)
  264. def get_outputs(self, include_bytecode=1):
  265. modules = self.find_all_modules()
  266. outputs = []
  267. for (package, module, module_file) in modules:
  268. package = package.split('.')
  269. filename = self.get_module_outfile(self.build_lib, package, module)
  270. outputs.append(filename)
  271. if include_bytecode:
  272. if self.compile:
  273. outputs.append(importlib.util.cache_from_source(
  274. filename, optimization=''))
  275. if self.optimize > 0:
  276. outputs.append(importlib.util.cache_from_source(
  277. filename, optimization=self.optimize))
  278. outputs += [
  279. os.path.join(build_dir, filename)
  280. for package, src_dir, build_dir, filenames in self.data_files
  281. for filename in filenames
  282. ]
  283. return outputs
  284. def build_module(self, module, module_file, package):
  285. if isinstance(package, str):
  286. package = package.split('.')
  287. elif not isinstance(package, (list, tuple)):
  288. raise TypeError(
  289. "'package' must be a string (dot-separated), list, or tuple")
  290. # Now put the module source file into the "build" area -- this is
  291. # easy, we just copy it somewhere under self.build_lib (the build
  292. # directory for Python source).
  293. outfile = self.get_module_outfile(self.build_lib, package, module)
  294. dir = os.path.dirname(outfile)
  295. self.mkpath(dir)
  296. return self.copy_file(module_file, outfile, preserve_mode=0)
  297. def build_modules(self):
  298. modules = self.find_modules()
  299. for (package, module, module_file) in modules:
  300. # Now "build" the module -- ie. copy the source file to
  301. # self.build_lib (the build directory for Python source).
  302. # (Actually, it gets copied to the directory for this package
  303. # under self.build_lib.)
  304. self.build_module(module, module_file, package)
  305. def build_packages(self):
  306. for package in self.packages:
  307. # Get list of (package, module, module_file) tuples based on
  308. # scanning the package directory. 'package' is only included
  309. # in the tuple so that 'find_modules()' and
  310. # 'find_package_tuples()' have a consistent interface; it's
  311. # ignored here (apart from a sanity check). Also, 'module' is
  312. # the *unqualified* module name (ie. no dots, no package -- we
  313. # already know its package!), and 'module_file' is the path to
  314. # the .py file, relative to the current directory
  315. # (ie. including 'package_dir').
  316. package_dir = self.get_package_dir(package)
  317. modules = self.find_package_modules(package, package_dir)
  318. # Now loop over the modules we found, "building" each one (just
  319. # copy it to self.build_lib).
  320. for (package_, module, module_file) in modules:
  321. assert package == package_
  322. self.build_module(module, module_file, package)
  323. def byte_compile(self, files):
  324. if sys.dont_write_bytecode:
  325. self.warn('byte-compiling is disabled, skipping.')
  326. return
  327. from distutils.util import byte_compile
  328. prefix = self.build_lib
  329. if prefix[-1] != os.sep:
  330. prefix = prefix + os.sep
  331. # XXX this code is essentially the same as the 'byte_compile()
  332. # method of the "install_lib" command, except for the determination
  333. # of the 'prefix' string. Hmmm.
  334. if self.compile:
  335. byte_compile(files, optimize=0,
  336. force=self.force, prefix=prefix, dry_run=self.dry_run)
  337. if self.optimize > 0:
  338. byte_compile(files, optimize=self.optimize,
  339. force=self.force, prefix=prefix, dry_run=self.dry_run)
  340. class build_py_2to3(build_py, Mixin2to3):
  341. def run(self):
  342. self.updated_files = []
  343. # Base class code
  344. if self.py_modules:
  345. self.build_modules()
  346. if self.packages:
  347. self.build_packages()
  348. self.build_package_data()
  349. # 2to3
  350. self.run_2to3(self.updated_files)
  351. # Remaining base class code
  352. self.byte_compile(self.get_outputs(include_bytecode=0))
  353. def build_module(self, module, module_file, package):
  354. res = build_py.build_module(self, module, module_file, package)
  355. if res[1]:
  356. # file was copied
  357. self.updated_files.append(res[0])
  358. return res