test_manifest.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601
  1. # -*- coding: utf-8 -*-
  2. """sdist tests"""
  3. import contextlib
  4. import os
  5. import shutil
  6. import sys
  7. import tempfile
  8. import itertools
  9. import io
  10. from distutils import log
  11. from distutils.errors import DistutilsTemplateError
  12. from setuptools.command.egg_info import FileList, egg_info, translate_pattern
  13. from setuptools.dist import Distribution
  14. from setuptools.tests.textwrap import DALS
  15. import pytest
  16. def make_local_path(s):
  17. """Converts '/' in a string to os.sep"""
  18. return s.replace('/', os.sep)
  19. SETUP_ATTRS = {
  20. 'name': 'app',
  21. 'version': '0.0',
  22. 'packages': ['app'],
  23. }
  24. SETUP_PY = """\
  25. from setuptools import setup
  26. setup(**%r)
  27. """ % SETUP_ATTRS
  28. @contextlib.contextmanager
  29. def quiet():
  30. old_stdout, old_stderr = sys.stdout, sys.stderr
  31. sys.stdout, sys.stderr = io.StringIO(), io.StringIO()
  32. try:
  33. yield
  34. finally:
  35. sys.stdout, sys.stderr = old_stdout, old_stderr
  36. def touch(filename):
  37. open(filename, 'w').close()
  38. # The set of files always in the manifest, including all files in the
  39. # .egg-info directory
  40. default_files = frozenset(map(make_local_path, [
  41. 'README.rst',
  42. 'MANIFEST.in',
  43. 'setup.py',
  44. 'app.egg-info/PKG-INFO',
  45. 'app.egg-info/SOURCES.txt',
  46. 'app.egg-info/dependency_links.txt',
  47. 'app.egg-info/top_level.txt',
  48. 'app/__init__.py',
  49. ]))
  50. translate_specs = [
  51. ('foo', ['foo'], ['bar', 'foobar']),
  52. ('foo/bar', ['foo/bar'], ['foo/bar/baz', './foo/bar', 'foo']),
  53. # Glob matching
  54. ('*.txt', ['foo.txt', 'bar.txt'], ['foo/foo.txt']),
  55. (
  56. 'dir/*.txt',
  57. ['dir/foo.txt', 'dir/bar.txt', 'dir/.txt'], ['notdir/foo.txt']),
  58. ('*/*.py', ['bin/start.py'], []),
  59. ('docs/page-?.txt', ['docs/page-9.txt'], ['docs/page-10.txt']),
  60. # Globstars change what they mean depending upon where they are
  61. (
  62. 'foo/**/bar',
  63. ['foo/bing/bar', 'foo/bing/bang/bar', 'foo/bar'],
  64. ['foo/abar'],
  65. ),
  66. (
  67. 'foo/**',
  68. ['foo/bar/bing.py', 'foo/x'],
  69. ['/foo/x'],
  70. ),
  71. (
  72. '**',
  73. ['x', 'abc/xyz', '@nything'],
  74. [],
  75. ),
  76. # Character classes
  77. (
  78. 'pre[one]post',
  79. ['preopost', 'prenpost', 'preepost'],
  80. ['prepost', 'preonepost'],
  81. ),
  82. (
  83. 'hello[!one]world',
  84. ['helloxworld', 'helloyworld'],
  85. ['hellooworld', 'helloworld', 'hellooneworld'],
  86. ),
  87. (
  88. '[]one].txt',
  89. ['o.txt', '].txt', 'e.txt'],
  90. ['one].txt'],
  91. ),
  92. (
  93. 'foo[!]one]bar',
  94. ['fooybar'],
  95. ['foo]bar', 'fooobar', 'fooebar'],
  96. ),
  97. ]
  98. """
  99. A spec of inputs for 'translate_pattern' and matches and mismatches
  100. for that input.
  101. """
  102. match_params = itertools.chain.from_iterable(
  103. zip(itertools.repeat(pattern), matches)
  104. for pattern, matches, mismatches in translate_specs
  105. )
  106. @pytest.fixture(params=match_params)
  107. def pattern_match(request):
  108. return map(make_local_path, request.param)
  109. mismatch_params = itertools.chain.from_iterable(
  110. zip(itertools.repeat(pattern), mismatches)
  111. for pattern, matches, mismatches in translate_specs
  112. )
  113. @pytest.fixture(params=mismatch_params)
  114. def pattern_mismatch(request):
  115. return map(make_local_path, request.param)
  116. def test_translated_pattern_match(pattern_match):
  117. pattern, target = pattern_match
  118. assert translate_pattern(pattern).match(target)
  119. def test_translated_pattern_mismatch(pattern_mismatch):
  120. pattern, target = pattern_mismatch
  121. assert not translate_pattern(pattern).match(target)
  122. class TempDirTestCase:
  123. def setup_method(self, method):
  124. self.temp_dir = tempfile.mkdtemp()
  125. self.old_cwd = os.getcwd()
  126. os.chdir(self.temp_dir)
  127. def teardown_method(self, method):
  128. os.chdir(self.old_cwd)
  129. shutil.rmtree(self.temp_dir)
  130. class TestManifestTest(TempDirTestCase):
  131. def setup_method(self, method):
  132. super(TestManifestTest, self).setup_method(method)
  133. f = open(os.path.join(self.temp_dir, 'setup.py'), 'w')
  134. f.write(SETUP_PY)
  135. f.close()
  136. """
  137. Create a file tree like:
  138. - LICENSE
  139. - README.rst
  140. - testing.rst
  141. - .hidden.rst
  142. - app/
  143. - __init__.py
  144. - a.txt
  145. - b.txt
  146. - c.rst
  147. - static/
  148. - app.js
  149. - app.js.map
  150. - app.css
  151. - app.css.map
  152. """
  153. for fname in ['README.rst', '.hidden.rst', 'testing.rst', 'LICENSE']:
  154. touch(os.path.join(self.temp_dir, fname))
  155. # Set up the rest of the test package
  156. test_pkg = os.path.join(self.temp_dir, 'app')
  157. os.mkdir(test_pkg)
  158. for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
  159. touch(os.path.join(test_pkg, fname))
  160. # Some compiled front-end assets to include
  161. static = os.path.join(test_pkg, 'static')
  162. os.mkdir(static)
  163. for fname in ['app.js', 'app.js.map', 'app.css', 'app.css.map']:
  164. touch(os.path.join(static, fname))
  165. def make_manifest(self, contents):
  166. """Write a MANIFEST.in."""
  167. with open(os.path.join(self.temp_dir, 'MANIFEST.in'), 'w') as f:
  168. f.write(DALS(contents))
  169. def get_files(self):
  170. """Run egg_info and get all the files to include, as a set"""
  171. dist = Distribution(SETUP_ATTRS)
  172. dist.script_name = 'setup.py'
  173. cmd = egg_info(dist)
  174. cmd.ensure_finalized()
  175. cmd.run()
  176. return set(cmd.filelist.files)
  177. def test_no_manifest(self):
  178. """Check a missing MANIFEST.in includes only the standard files."""
  179. assert (default_files - set(['MANIFEST.in'])) == self.get_files()
  180. def test_empty_files(self):
  181. """Check an empty MANIFEST.in includes only the standard files."""
  182. self.make_manifest("")
  183. assert default_files == self.get_files()
  184. def test_include(self):
  185. """Include extra rst files in the project root."""
  186. self.make_manifest("include *.rst")
  187. files = default_files | set([
  188. 'testing.rst', '.hidden.rst'])
  189. assert files == self.get_files()
  190. def test_exclude(self):
  191. """Include everything in app/ except the text files"""
  192. ml = make_local_path
  193. self.make_manifest(
  194. """
  195. include app/*
  196. exclude app/*.txt
  197. """)
  198. files = default_files | set([ml('app/c.rst')])
  199. assert files == self.get_files()
  200. def test_include_multiple(self):
  201. """Include with multiple patterns."""
  202. ml = make_local_path
  203. self.make_manifest("include app/*.txt app/static/*")
  204. files = default_files | set([
  205. ml('app/a.txt'), ml('app/b.txt'),
  206. ml('app/static/app.js'), ml('app/static/app.js.map'),
  207. ml('app/static/app.css'), ml('app/static/app.css.map')])
  208. assert files == self.get_files()
  209. def test_graft(self):
  210. """Include the whole app/static/ directory."""
  211. ml = make_local_path
  212. self.make_manifest("graft app/static")
  213. files = default_files | set([
  214. ml('app/static/app.js'), ml('app/static/app.js.map'),
  215. ml('app/static/app.css'), ml('app/static/app.css.map')])
  216. assert files == self.get_files()
  217. def test_graft_glob_syntax(self):
  218. """Include the whole app/static/ directory."""
  219. ml = make_local_path
  220. self.make_manifest("graft */static")
  221. files = default_files | set([
  222. ml('app/static/app.js'), ml('app/static/app.js.map'),
  223. ml('app/static/app.css'), ml('app/static/app.css.map')])
  224. assert files == self.get_files()
  225. def test_graft_global_exclude(self):
  226. """Exclude all *.map files in the project."""
  227. ml = make_local_path
  228. self.make_manifest(
  229. """
  230. graft app/static
  231. global-exclude *.map
  232. """)
  233. files = default_files | set([
  234. ml('app/static/app.js'), ml('app/static/app.css')])
  235. assert files == self.get_files()
  236. def test_global_include(self):
  237. """Include all *.rst, *.js, and *.css files in the whole tree."""
  238. ml = make_local_path
  239. self.make_manifest(
  240. """
  241. global-include *.rst *.js *.css
  242. """)
  243. files = default_files | set([
  244. '.hidden.rst', 'testing.rst', ml('app/c.rst'),
  245. ml('app/static/app.js'), ml('app/static/app.css')])
  246. assert files == self.get_files()
  247. def test_graft_prune(self):
  248. """Include all files in app/, except for the whole app/static/ dir."""
  249. ml = make_local_path
  250. self.make_manifest(
  251. """
  252. graft app
  253. prune app/static
  254. """)
  255. files = default_files | set([
  256. ml('app/a.txt'), ml('app/b.txt'), ml('app/c.rst')])
  257. assert files == self.get_files()
  258. class TestFileListTest(TempDirTestCase):
  259. """
  260. A copy of the relevant bits of distutils/tests/test_filelist.py,
  261. to ensure setuptools' version of FileList keeps parity with distutils.
  262. """
  263. def setup_method(self, method):
  264. super(TestFileListTest, self).setup_method(method)
  265. self.threshold = log.set_threshold(log.FATAL)
  266. self._old_log = log.Log._log
  267. log.Log._log = self._log
  268. self.logs = []
  269. def teardown_method(self, method):
  270. log.set_threshold(self.threshold)
  271. log.Log._log = self._old_log
  272. super(TestFileListTest, self).teardown_method(method)
  273. def _log(self, level, msg, args):
  274. if level not in (log.DEBUG, log.INFO, log.WARN, log.ERROR, log.FATAL):
  275. raise ValueError('%s wrong log level' % str(level))
  276. self.logs.append((level, msg, args))
  277. def get_logs(self, *levels):
  278. def _format(msg, args):
  279. if len(args) == 0:
  280. return msg
  281. return msg % args
  282. return [_format(msg, args) for level, msg, args
  283. in self.logs if level in levels]
  284. def clear_logs(self):
  285. self.logs = []
  286. def assertNoWarnings(self):
  287. assert self.get_logs(log.WARN) == []
  288. self.clear_logs()
  289. def assertWarnings(self):
  290. assert len(self.get_logs(log.WARN)) > 0
  291. self.clear_logs()
  292. def make_files(self, files):
  293. for file in files:
  294. file = os.path.join(self.temp_dir, file)
  295. dirname, basename = os.path.split(file)
  296. os.makedirs(dirname, exist_ok=True)
  297. open(file, 'w').close()
  298. def test_process_template_line(self):
  299. # testing all MANIFEST.in template patterns
  300. file_list = FileList()
  301. ml = make_local_path
  302. # simulated file list
  303. self.make_files([
  304. 'foo.tmp', 'ok', 'xo', 'four.txt',
  305. 'buildout.cfg',
  306. # filelist does not filter out VCS directories,
  307. # it's sdist that does
  308. ml('.hg/last-message.txt'),
  309. ml('global/one.txt'),
  310. ml('global/two.txt'),
  311. ml('global/files.x'),
  312. ml('global/here.tmp'),
  313. ml('f/o/f.oo'),
  314. ml('dir/graft-one'),
  315. ml('dir/dir2/graft2'),
  316. ml('dir3/ok'),
  317. ml('dir3/sub/ok.txt'),
  318. ])
  319. MANIFEST_IN = DALS("""\
  320. include ok
  321. include xo
  322. exclude xo
  323. include foo.tmp
  324. include buildout.cfg
  325. global-include *.x
  326. global-include *.txt
  327. global-exclude *.tmp
  328. recursive-include f *.oo
  329. recursive-exclude global *.x
  330. graft dir
  331. prune dir3
  332. """)
  333. for line in MANIFEST_IN.split('\n'):
  334. if not line:
  335. continue
  336. file_list.process_template_line(line)
  337. wanted = [
  338. 'buildout.cfg',
  339. 'four.txt',
  340. 'ok',
  341. ml('.hg/last-message.txt'),
  342. ml('dir/graft-one'),
  343. ml('dir/dir2/graft2'),
  344. ml('f/o/f.oo'),
  345. ml('global/one.txt'),
  346. ml('global/two.txt'),
  347. ]
  348. file_list.sort()
  349. assert file_list.files == wanted
  350. def test_exclude_pattern(self):
  351. # return False if no match
  352. file_list = FileList()
  353. assert not file_list.exclude_pattern('*.py')
  354. # return True if files match
  355. file_list = FileList()
  356. file_list.files = ['a.py', 'b.py']
  357. assert file_list.exclude_pattern('*.py')
  358. # test excludes
  359. file_list = FileList()
  360. file_list.files = ['a.py', 'a.txt']
  361. file_list.exclude_pattern('*.py')
  362. file_list.sort()
  363. assert file_list.files == ['a.txt']
  364. def test_include_pattern(self):
  365. # return False if no match
  366. file_list = FileList()
  367. self.make_files([])
  368. assert not file_list.include_pattern('*.py')
  369. # return True if files match
  370. file_list = FileList()
  371. self.make_files(['a.py', 'b.txt'])
  372. assert file_list.include_pattern('*.py')
  373. # test * matches all files
  374. file_list = FileList()
  375. self.make_files(['a.py', 'b.txt'])
  376. file_list.include_pattern('*')
  377. file_list.sort()
  378. assert file_list.files == ['a.py', 'b.txt']
  379. def test_process_template_line_invalid(self):
  380. # invalid lines
  381. file_list = FileList()
  382. for action in ('include', 'exclude', 'global-include',
  383. 'global-exclude', 'recursive-include',
  384. 'recursive-exclude', 'graft', 'prune', 'blarg'):
  385. try:
  386. file_list.process_template_line(action)
  387. except DistutilsTemplateError:
  388. pass
  389. except Exception:
  390. assert False, "Incorrect error thrown"
  391. else:
  392. assert False, "Should have thrown an error"
  393. def test_include(self):
  394. ml = make_local_path
  395. # include
  396. file_list = FileList()
  397. self.make_files(['a.py', 'b.txt', ml('d/c.py')])
  398. file_list.process_template_line('include *.py')
  399. file_list.sort()
  400. assert file_list.files == ['a.py']
  401. self.assertNoWarnings()
  402. file_list.process_template_line('include *.rb')
  403. file_list.sort()
  404. assert file_list.files == ['a.py']
  405. self.assertWarnings()
  406. def test_exclude(self):
  407. ml = make_local_path
  408. # exclude
  409. file_list = FileList()
  410. file_list.files = ['a.py', 'b.txt', ml('d/c.py')]
  411. file_list.process_template_line('exclude *.py')
  412. file_list.sort()
  413. assert file_list.files == ['b.txt', ml('d/c.py')]
  414. self.assertNoWarnings()
  415. file_list.process_template_line('exclude *.rb')
  416. file_list.sort()
  417. assert file_list.files == ['b.txt', ml('d/c.py')]
  418. self.assertWarnings()
  419. def test_global_include(self):
  420. ml = make_local_path
  421. # global-include
  422. file_list = FileList()
  423. self.make_files(['a.py', 'b.txt', ml('d/c.py')])
  424. file_list.process_template_line('global-include *.py')
  425. file_list.sort()
  426. assert file_list.files == ['a.py', ml('d/c.py')]
  427. self.assertNoWarnings()
  428. file_list.process_template_line('global-include *.rb')
  429. file_list.sort()
  430. assert file_list.files == ['a.py', ml('d/c.py')]
  431. self.assertWarnings()
  432. def test_global_exclude(self):
  433. ml = make_local_path
  434. # global-exclude
  435. file_list = FileList()
  436. file_list.files = ['a.py', 'b.txt', ml('d/c.py')]
  437. file_list.process_template_line('global-exclude *.py')
  438. file_list.sort()
  439. assert file_list.files == ['b.txt']
  440. self.assertNoWarnings()
  441. file_list.process_template_line('global-exclude *.rb')
  442. file_list.sort()
  443. assert file_list.files == ['b.txt']
  444. self.assertWarnings()
  445. def test_recursive_include(self):
  446. ml = make_local_path
  447. # recursive-include
  448. file_list = FileList()
  449. self.make_files(['a.py', ml('d/b.py'), ml('d/c.txt'), ml('d/d/e.py')])
  450. file_list.process_template_line('recursive-include d *.py')
  451. file_list.sort()
  452. assert file_list.files == [ml('d/b.py'), ml('d/d/e.py')]
  453. self.assertNoWarnings()
  454. file_list.process_template_line('recursive-include e *.py')
  455. file_list.sort()
  456. assert file_list.files == [ml('d/b.py'), ml('d/d/e.py')]
  457. self.assertWarnings()
  458. def test_recursive_exclude(self):
  459. ml = make_local_path
  460. # recursive-exclude
  461. file_list = FileList()
  462. file_list.files = ['a.py', ml('d/b.py'), ml('d/c.txt'), ml('d/d/e.py')]
  463. file_list.process_template_line('recursive-exclude d *.py')
  464. file_list.sort()
  465. assert file_list.files == ['a.py', ml('d/c.txt')]
  466. self.assertNoWarnings()
  467. file_list.process_template_line('recursive-exclude e *.py')
  468. file_list.sort()
  469. assert file_list.files == ['a.py', ml('d/c.txt')]
  470. self.assertWarnings()
  471. def test_graft(self):
  472. ml = make_local_path
  473. # graft
  474. file_list = FileList()
  475. self.make_files(['a.py', ml('d/b.py'), ml('d/d/e.py'), ml('f/f.py')])
  476. file_list.process_template_line('graft d')
  477. file_list.sort()
  478. assert file_list.files == [ml('d/b.py'), ml('d/d/e.py')]
  479. self.assertNoWarnings()
  480. file_list.process_template_line('graft e')
  481. file_list.sort()
  482. assert file_list.files == [ml('d/b.py'), ml('d/d/e.py')]
  483. self.assertWarnings()
  484. def test_prune(self):
  485. ml = make_local_path
  486. # prune
  487. file_list = FileList()
  488. file_list.files = ['a.py', ml('d/b.py'), ml('d/d/e.py'), ml('f/f.py')]
  489. file_list.process_template_line('prune d')
  490. file_list.sort()
  491. assert file_list.files == ['a.py', ml('f/f.py')]
  492. self.assertNoWarnings()
  493. file_list.process_template_line('prune e')
  494. file_list.sort()
  495. assert file_list.files == ['a.py', ml('f/f.py')]
  496. self.assertWarnings()