test_sdist.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474
  1. """sdist tests"""
  2. import os
  3. import sys
  4. import tempfile
  5. import unicodedata
  6. import contextlib
  7. import io
  8. import pytest
  9. import pkg_resources
  10. from setuptools.command.sdist import sdist
  11. from setuptools.command.egg_info import manifest_maker
  12. from setuptools.dist import Distribution
  13. from setuptools.tests import fail_on_ascii
  14. from .text import Filenames
  15. SETUP_ATTRS = {
  16. 'name': 'sdist_test',
  17. 'version': '0.0',
  18. 'packages': ['sdist_test'],
  19. 'package_data': {'sdist_test': ['*.txt']},
  20. 'data_files': [("data", [os.path.join("d", "e.dat")])],
  21. }
  22. SETUP_PY = """\
  23. from setuptools import setup
  24. setup(**%r)
  25. """ % SETUP_ATTRS
  26. @contextlib.contextmanager
  27. def quiet():
  28. old_stdout, old_stderr = sys.stdout, sys.stderr
  29. sys.stdout, sys.stderr = io.StringIO(), io.StringIO()
  30. try:
  31. yield
  32. finally:
  33. sys.stdout, sys.stderr = old_stdout, old_stderr
  34. # Convert to POSIX path
  35. def posix(path):
  36. if not isinstance(path, str):
  37. return path.replace(os.sep.encode('ascii'), b'/')
  38. else:
  39. return path.replace(os.sep, '/')
  40. # HFS Plus uses decomposed UTF-8
  41. def decompose(path):
  42. if isinstance(path, str):
  43. return unicodedata.normalize('NFD', path)
  44. try:
  45. path = path.decode('utf-8')
  46. path = unicodedata.normalize('NFD', path)
  47. path = path.encode('utf-8')
  48. except UnicodeError:
  49. pass # Not UTF-8
  50. return path
  51. def read_all_bytes(filename):
  52. with io.open(filename, 'rb') as fp:
  53. return fp.read()
  54. def latin1_fail():
  55. try:
  56. desc, filename = tempfile.mkstemp(suffix=Filenames.latin_1)
  57. os.close(desc)
  58. os.remove(filename)
  59. except Exception:
  60. return True
  61. fail_on_latin1_encoded_filenames = pytest.mark.xfail(
  62. latin1_fail(),
  63. reason="System does not support latin-1 filenames",
  64. )
  65. def touch(path):
  66. path.write_text('', encoding='utf-8')
  67. class TestSdistTest:
  68. @pytest.fixture(autouse=True)
  69. def source_dir(self, tmpdir):
  70. (tmpdir / 'setup.py').write_text(SETUP_PY, encoding='utf-8')
  71. # Set up the rest of the test package
  72. test_pkg = tmpdir / 'sdist_test'
  73. test_pkg.mkdir()
  74. data_folder = tmpdir / 'd'
  75. data_folder.mkdir()
  76. # *.rst was not included in package_data, so c.rst should not be
  77. # automatically added to the manifest when not under version control
  78. for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
  79. touch(test_pkg / fname)
  80. touch(data_folder / 'e.dat')
  81. with tmpdir.as_cwd():
  82. yield
  83. def test_package_data_in_sdist(self):
  84. """Regression test for pull request #4: ensures that files listed in
  85. package_data are included in the manifest even if they're not added to
  86. version control.
  87. """
  88. dist = Distribution(SETUP_ATTRS)
  89. dist.script_name = 'setup.py'
  90. cmd = sdist(dist)
  91. cmd.ensure_finalized()
  92. with quiet():
  93. cmd.run()
  94. manifest = cmd.filelist.files
  95. assert os.path.join('sdist_test', 'a.txt') in manifest
  96. assert os.path.join('sdist_test', 'b.txt') in manifest
  97. assert os.path.join('sdist_test', 'c.rst') not in manifest
  98. assert os.path.join('d', 'e.dat') in manifest
  99. def test_setup_py_exists(self):
  100. dist = Distribution(SETUP_ATTRS)
  101. dist.script_name = 'foo.py'
  102. cmd = sdist(dist)
  103. cmd.ensure_finalized()
  104. with quiet():
  105. cmd.run()
  106. manifest = cmd.filelist.files
  107. assert 'setup.py' in manifest
  108. def test_setup_py_missing(self):
  109. dist = Distribution(SETUP_ATTRS)
  110. dist.script_name = 'foo.py'
  111. cmd = sdist(dist)
  112. cmd.ensure_finalized()
  113. if os.path.exists("setup.py"):
  114. os.remove("setup.py")
  115. with quiet():
  116. cmd.run()
  117. manifest = cmd.filelist.files
  118. assert 'setup.py' not in manifest
  119. def test_setup_py_excluded(self):
  120. with open("MANIFEST.in", "w") as manifest_file:
  121. manifest_file.write("exclude setup.py")
  122. dist = Distribution(SETUP_ATTRS)
  123. dist.script_name = 'foo.py'
  124. cmd = sdist(dist)
  125. cmd.ensure_finalized()
  126. with quiet():
  127. cmd.run()
  128. manifest = cmd.filelist.files
  129. assert 'setup.py' not in manifest
  130. def test_defaults_case_sensitivity(self, tmpdir):
  131. """
  132. Make sure default files (README.*, etc.) are added in a case-sensitive
  133. way to avoid problems with packages built on Windows.
  134. """
  135. touch(tmpdir / 'readme.rst')
  136. touch(tmpdir / 'SETUP.cfg')
  137. dist = Distribution(SETUP_ATTRS)
  138. # the extension deliberately capitalized for this test
  139. # to make sure the actual filename (not capitalized) gets added
  140. # to the manifest
  141. dist.script_name = 'setup.PY'
  142. cmd = sdist(dist)
  143. cmd.ensure_finalized()
  144. with quiet():
  145. cmd.run()
  146. # lowercase all names so we can test in a
  147. # case-insensitive way to make sure the files
  148. # are not included.
  149. manifest = map(lambda x: x.lower(), cmd.filelist.files)
  150. assert 'readme.rst' not in manifest, manifest
  151. assert 'setup.py' not in manifest, manifest
  152. assert 'setup.cfg' not in manifest, manifest
  153. @fail_on_ascii
  154. def test_manifest_is_written_with_utf8_encoding(self):
  155. # Test for #303.
  156. dist = Distribution(SETUP_ATTRS)
  157. dist.script_name = 'setup.py'
  158. mm = manifest_maker(dist)
  159. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  160. os.mkdir('sdist_test.egg-info')
  161. # UTF-8 filename
  162. filename = os.path.join('sdist_test', 'smörbröd.py')
  163. # Must create the file or it will get stripped.
  164. open(filename, 'w').close()
  165. # Add UTF-8 filename and write manifest
  166. with quiet():
  167. mm.run()
  168. mm.filelist.append(filename)
  169. mm.write_manifest()
  170. contents = read_all_bytes(mm.manifest)
  171. # The manifest should be UTF-8 encoded
  172. u_contents = contents.decode('UTF-8')
  173. # The manifest should contain the UTF-8 filename
  174. assert posix(filename) in u_contents
  175. @fail_on_ascii
  176. def test_write_manifest_allows_utf8_filenames(self):
  177. # Test for #303.
  178. dist = Distribution(SETUP_ATTRS)
  179. dist.script_name = 'setup.py'
  180. mm = manifest_maker(dist)
  181. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  182. os.mkdir('sdist_test.egg-info')
  183. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  184. # Must touch the file or risk removal
  185. open(filename, "w").close()
  186. # Add filename and write manifest
  187. with quiet():
  188. mm.run()
  189. u_filename = filename.decode('utf-8')
  190. mm.filelist.files.append(u_filename)
  191. # Re-write manifest
  192. mm.write_manifest()
  193. contents = read_all_bytes(mm.manifest)
  194. # The manifest should be UTF-8 encoded
  195. contents.decode('UTF-8')
  196. # The manifest should contain the UTF-8 filename
  197. assert posix(filename) in contents
  198. # The filelist should have been updated as well
  199. assert u_filename in mm.filelist.files
  200. def test_write_manifest_skips_non_utf8_filenames(self):
  201. """
  202. Files that cannot be encoded to UTF-8 (specifically, those that
  203. weren't originally successfully decoded and have surrogate
  204. escapes) should be omitted from the manifest.
  205. See https://bitbucket.org/tarek/distribute/issue/303 for history.
  206. """
  207. dist = Distribution(SETUP_ATTRS)
  208. dist.script_name = 'setup.py'
  209. mm = manifest_maker(dist)
  210. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  211. os.mkdir('sdist_test.egg-info')
  212. # Latin-1 filename
  213. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  214. # Add filename with surrogates and write manifest
  215. with quiet():
  216. mm.run()
  217. u_filename = filename.decode('utf-8', 'surrogateescape')
  218. mm.filelist.append(u_filename)
  219. # Re-write manifest
  220. mm.write_manifest()
  221. contents = read_all_bytes(mm.manifest)
  222. # The manifest should be UTF-8 encoded
  223. contents.decode('UTF-8')
  224. # The Latin-1 filename should have been skipped
  225. assert posix(filename) not in contents
  226. # The filelist should have been updated as well
  227. assert u_filename not in mm.filelist.files
  228. @fail_on_ascii
  229. def test_manifest_is_read_with_utf8_encoding(self):
  230. # Test for #303.
  231. dist = Distribution(SETUP_ATTRS)
  232. dist.script_name = 'setup.py'
  233. cmd = sdist(dist)
  234. cmd.ensure_finalized()
  235. # Create manifest
  236. with quiet():
  237. cmd.run()
  238. # Add UTF-8 filename to manifest
  239. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  240. cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  241. manifest = open(cmd.manifest, 'ab')
  242. manifest.write(b'\n' + filename)
  243. manifest.close()
  244. # The file must exist to be included in the filelist
  245. open(filename, 'w').close()
  246. # Re-read manifest
  247. cmd.filelist.files = []
  248. with quiet():
  249. cmd.read_manifest()
  250. # The filelist should contain the UTF-8 filename
  251. filename = filename.decode('utf-8')
  252. assert filename in cmd.filelist.files
  253. @fail_on_latin1_encoded_filenames
  254. def test_read_manifest_skips_non_utf8_filenames(self):
  255. # Test for #303.
  256. dist = Distribution(SETUP_ATTRS)
  257. dist.script_name = 'setup.py'
  258. cmd = sdist(dist)
  259. cmd.ensure_finalized()
  260. # Create manifest
  261. with quiet():
  262. cmd.run()
  263. # Add Latin-1 filename to manifest
  264. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  265. cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  266. manifest = open(cmd.manifest, 'ab')
  267. manifest.write(b'\n' + filename)
  268. manifest.close()
  269. # The file must exist to be included in the filelist
  270. open(filename, 'w').close()
  271. # Re-read manifest
  272. cmd.filelist.files = []
  273. with quiet():
  274. cmd.read_manifest()
  275. # The Latin-1 filename should have been skipped
  276. filename = filename.decode('latin-1')
  277. assert filename not in cmd.filelist.files
  278. @fail_on_ascii
  279. @fail_on_latin1_encoded_filenames
  280. def test_sdist_with_utf8_encoded_filename(self):
  281. # Test for #303.
  282. dist = Distribution(self.make_strings(SETUP_ATTRS))
  283. dist.script_name = 'setup.py'
  284. cmd = sdist(dist)
  285. cmd.ensure_finalized()
  286. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  287. open(filename, 'w').close()
  288. with quiet():
  289. cmd.run()
  290. if sys.platform == 'darwin':
  291. filename = decompose(filename)
  292. fs_enc = sys.getfilesystemencoding()
  293. if sys.platform == 'win32':
  294. if fs_enc == 'cp1252':
  295. # Python mangles the UTF-8 filename
  296. filename = filename.decode('cp1252')
  297. assert filename in cmd.filelist.files
  298. else:
  299. filename = filename.decode('mbcs')
  300. assert filename in cmd.filelist.files
  301. else:
  302. filename = filename.decode('utf-8')
  303. assert filename in cmd.filelist.files
  304. @classmethod
  305. def make_strings(cls, item):
  306. if isinstance(item, dict):
  307. return {
  308. key: cls.make_strings(value) for key, value in item.items()}
  309. if isinstance(item, list):
  310. return list(map(cls.make_strings, item))
  311. return str(item)
  312. @fail_on_latin1_encoded_filenames
  313. def test_sdist_with_latin1_encoded_filename(self):
  314. # Test for #303.
  315. dist = Distribution(self.make_strings(SETUP_ATTRS))
  316. dist.script_name = 'setup.py'
  317. cmd = sdist(dist)
  318. cmd.ensure_finalized()
  319. # Latin-1 filename
  320. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  321. open(filename, 'w').close()
  322. assert os.path.isfile(filename)
  323. with quiet():
  324. cmd.run()
  325. # not all windows systems have a default FS encoding of cp1252
  326. if sys.platform == 'win32':
  327. # Latin-1 is similar to Windows-1252 however
  328. # on mbcs filesys it is not in latin-1 encoding
  329. fs_enc = sys.getfilesystemencoding()
  330. if fs_enc != 'mbcs':
  331. fs_enc = 'latin-1'
  332. filename = filename.decode(fs_enc)
  333. assert filename in cmd.filelist.files
  334. else:
  335. # The Latin-1 filename should have been skipped
  336. filename = filename.decode('latin-1')
  337. filename not in cmd.filelist.files
  338. def test_pyproject_toml_in_sdist(self, tmpdir):
  339. """
  340. Check if pyproject.toml is included in source distribution if present
  341. """
  342. touch(tmpdir / 'pyproject.toml')
  343. dist = Distribution(SETUP_ATTRS)
  344. dist.script_name = 'setup.py'
  345. cmd = sdist(dist)
  346. cmd.ensure_finalized()
  347. with quiet():
  348. cmd.run()
  349. manifest = cmd.filelist.files
  350. assert 'pyproject.toml' in manifest
  351. def test_pyproject_toml_excluded(self, tmpdir):
  352. """
  353. Check that pyproject.toml can excluded even if present
  354. """
  355. touch(tmpdir / 'pyproject.toml')
  356. with open('MANIFEST.in', 'w') as mts:
  357. print('exclude pyproject.toml', file=mts)
  358. dist = Distribution(SETUP_ATTRS)
  359. dist.script_name = 'setup.py'
  360. cmd = sdist(dist)
  361. cmd.ensure_finalized()
  362. with quiet():
  363. cmd.run()
  364. manifest = cmd.filelist.files
  365. assert 'pyproject.toml' not in manifest
  366. def test_default_revctrl():
  367. """
  368. When _default_revctrl was removed from the `setuptools.command.sdist`
  369. module in 10.0, it broke some systems which keep an old install of
  370. setuptools (Distribute) around. Those old versions require that the
  371. setuptools package continue to implement that interface, so this
  372. function provides that interface, stubbed. See #320 for details.
  373. This interface must be maintained until Ubuntu 12.04 is no longer
  374. supported (by Setuptools).
  375. """
  376. ep_def = 'svn_cvs = setuptools.command.sdist:_default_revctrl'
  377. ep = pkg_resources.EntryPoint.parse(ep_def)
  378. res = ep.resolve()
  379. assert hasattr(res, '__iter__')