configuration.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407
  1. """Configuration management setup
  2. Some terminology:
  3. - name
  4. As written in config files.
  5. - value
  6. Value associated with a name
  7. - key
  8. Name combined with it's section (section.name)
  9. - variant
  10. A single word describing where the configuration key-value pair came from
  11. """
  12. import locale
  13. import logging
  14. import os
  15. import sys
  16. from pip._vendor.six.moves import configparser
  17. from pip._internal.exceptions import (
  18. ConfigurationError,
  19. ConfigurationFileCouldNotBeLoaded,
  20. )
  21. from pip._internal.utils import appdirs
  22. from pip._internal.utils.compat import WINDOWS, expanduser
  23. from pip._internal.utils.misc import ensure_dir, enum
  24. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  25. if MYPY_CHECK_RUNNING:
  26. from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
  27. RawConfigParser = configparser.RawConfigParser # Shorthand
  28. Kind = NewType("Kind", str)
  29. CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'
  30. ENV_NAMES_IGNORED = "version", "help"
  31. # The kinds of configurations there are.
  32. kinds = enum(
  33. USER="user", # User Specific
  34. GLOBAL="global", # System Wide
  35. SITE="site", # [Virtual] Environment Specific
  36. ENV="env", # from PIP_CONFIG_FILE
  37. ENV_VAR="env-var", # from Environment Variables
  38. )
  39. OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
  40. VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
  41. logger = logging.getLogger(__name__)
  42. # NOTE: Maybe use the optionx attribute to normalize keynames.
  43. def _normalize_name(name):
  44. # type: (str) -> str
  45. """Make a name consistent regardless of source (environment or file)
  46. """
  47. name = name.lower().replace('_', '-')
  48. if name.startswith('--'):
  49. name = name[2:] # only prefer long opts
  50. return name
  51. def _disassemble_key(name):
  52. # type: (str) -> List[str]
  53. if "." not in name:
  54. error_message = (
  55. "Key does not contain dot separated section and key. "
  56. "Perhaps you wanted to use 'global.{}' instead?"
  57. ).format(name)
  58. raise ConfigurationError(error_message)
  59. return name.split(".", 1)
  60. def get_configuration_files():
  61. # type: () -> Dict[Kind, List[str]]
  62. global_config_files = [
  63. os.path.join(path, CONFIG_BASENAME)
  64. for path in appdirs.site_config_dirs('pip')
  65. ]
  66. site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
  67. legacy_config_file = os.path.join(
  68. expanduser('~'),
  69. 'pip' if WINDOWS else '.pip',
  70. CONFIG_BASENAME,
  71. )
  72. new_config_file = os.path.join(
  73. appdirs.user_config_dir("pip"), CONFIG_BASENAME
  74. )
  75. return {
  76. kinds.GLOBAL: global_config_files,
  77. kinds.SITE: [site_config_file],
  78. kinds.USER: [legacy_config_file, new_config_file],
  79. }
  80. class Configuration(object):
  81. """Handles management of configuration.
  82. Provides an interface to accessing and managing configuration files.
  83. This class converts provides an API that takes "section.key-name" style
  84. keys and stores the value associated with it as "key-name" under the
  85. section "section".
  86. This allows for a clean interface wherein the both the section and the
  87. key-name are preserved in an easy to manage form in the configuration files
  88. and the data stored is also nice.
  89. """
  90. def __init__(self, isolated, load_only=None):
  91. # type: (bool, Optional[Kind]) -> None
  92. super(Configuration, self).__init__()
  93. if load_only is not None and load_only not in VALID_LOAD_ONLY:
  94. raise ConfigurationError(
  95. "Got invalid value for load_only - should be one of {}".format(
  96. ", ".join(map(repr, VALID_LOAD_ONLY))
  97. )
  98. )
  99. self.isolated = isolated
  100. self.load_only = load_only
  101. # Because we keep track of where we got the data from
  102. self._parsers = {
  103. variant: [] for variant in OVERRIDE_ORDER
  104. } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
  105. self._config = {
  106. variant: {} for variant in OVERRIDE_ORDER
  107. } # type: Dict[Kind, Dict[str, Any]]
  108. self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]]
  109. def load(self):
  110. # type: () -> None
  111. """Loads configuration from configuration files and environment
  112. """
  113. self._load_config_files()
  114. if not self.isolated:
  115. self._load_environment_vars()
  116. def get_file_to_edit(self):
  117. # type: () -> Optional[str]
  118. """Returns the file with highest priority in configuration
  119. """
  120. assert self.load_only is not None, \
  121. "Need to be specified a file to be editing"
  122. try:
  123. return self._get_parser_to_modify()[0]
  124. except IndexError:
  125. return None
  126. def items(self):
  127. # type: () -> Iterable[Tuple[str, Any]]
  128. """Returns key-value pairs like dict.items() representing the loaded
  129. configuration
  130. """
  131. return self._dictionary.items()
  132. def get_value(self, key):
  133. # type: (str) -> Any
  134. """Get a value from the configuration.
  135. """
  136. try:
  137. return self._dictionary[key]
  138. except KeyError:
  139. raise ConfigurationError("No such key - {}".format(key))
  140. def set_value(self, key, value):
  141. # type: (str, Any) -> None
  142. """Modify a value in the configuration.
  143. """
  144. self._ensure_have_load_only()
  145. assert self.load_only
  146. fname, parser = self._get_parser_to_modify()
  147. if parser is not None:
  148. section, name = _disassemble_key(key)
  149. # Modify the parser and the configuration
  150. if not parser.has_section(section):
  151. parser.add_section(section)
  152. parser.set(section, name, value)
  153. self._config[self.load_only][key] = value
  154. self._mark_as_modified(fname, parser)
  155. def unset_value(self, key):
  156. # type: (str) -> None
  157. """Unset a value in the configuration."""
  158. self._ensure_have_load_only()
  159. assert self.load_only
  160. if key not in self._config[self.load_only]:
  161. raise ConfigurationError("No such key - {}".format(key))
  162. fname, parser = self._get_parser_to_modify()
  163. if parser is not None:
  164. section, name = _disassemble_key(key)
  165. if not (parser.has_section(section)
  166. and parser.remove_option(section, name)):
  167. # The option was not removed.
  168. raise ConfigurationError(
  169. "Fatal Internal error [id=1]. Please report as a bug."
  170. )
  171. # The section may be empty after the option was removed.
  172. if not parser.items(section):
  173. parser.remove_section(section)
  174. self._mark_as_modified(fname, parser)
  175. del self._config[self.load_only][key]
  176. def save(self):
  177. # type: () -> None
  178. """Save the current in-memory state.
  179. """
  180. self._ensure_have_load_only()
  181. for fname, parser in self._modified_parsers:
  182. logger.info("Writing to %s", fname)
  183. # Ensure directory exists.
  184. ensure_dir(os.path.dirname(fname))
  185. with open(fname, "w") as f:
  186. parser.write(f)
  187. #
  188. # Private routines
  189. #
  190. def _ensure_have_load_only(self):
  191. # type: () -> None
  192. if self.load_only is None:
  193. raise ConfigurationError("Needed a specific file to be modifying.")
  194. logger.debug("Will be working with %s variant only", self.load_only)
  195. @property
  196. def _dictionary(self):
  197. # type: () -> Dict[str, Any]
  198. """A dictionary representing the loaded configuration.
  199. """
  200. # NOTE: Dictionaries are not populated if not loaded. So, conditionals
  201. # are not needed here.
  202. retval = {}
  203. for variant in OVERRIDE_ORDER:
  204. retval.update(self._config[variant])
  205. return retval
  206. def _load_config_files(self):
  207. # type: () -> None
  208. """Loads configuration from configuration files
  209. """
  210. config_files = dict(self.iter_config_files())
  211. if config_files[kinds.ENV][0:1] == [os.devnull]:
  212. logger.debug(
  213. "Skipping loading configuration files due to "
  214. "environment's PIP_CONFIG_FILE being os.devnull"
  215. )
  216. return
  217. for variant, files in config_files.items():
  218. for fname in files:
  219. # If there's specific variant set in `load_only`, load only
  220. # that variant, not the others.
  221. if self.load_only is not None and variant != self.load_only:
  222. logger.debug(
  223. "Skipping file '%s' (variant: %s)", fname, variant
  224. )
  225. continue
  226. parser = self._load_file(variant, fname)
  227. # Keeping track of the parsers used
  228. self._parsers[variant].append((fname, parser))
  229. def _load_file(self, variant, fname):
  230. # type: (Kind, str) -> RawConfigParser
  231. logger.debug("For variant '%s', will try loading '%s'", variant, fname)
  232. parser = self._construct_parser(fname)
  233. for section in parser.sections():
  234. items = parser.items(section)
  235. self._config[variant].update(self._normalized_keys(section, items))
  236. return parser
  237. def _construct_parser(self, fname):
  238. # type: (str) -> RawConfigParser
  239. parser = configparser.RawConfigParser()
  240. # If there is no such file, don't bother reading it but create the
  241. # parser anyway, to hold the data.
  242. # Doing this is useful when modifying and saving files, where we don't
  243. # need to construct a parser.
  244. if os.path.exists(fname):
  245. try:
  246. parser.read(fname)
  247. except UnicodeDecodeError:
  248. # See https://github.com/pypa/pip/issues/4963
  249. raise ConfigurationFileCouldNotBeLoaded(
  250. reason="contains invalid {} characters".format(
  251. locale.getpreferredencoding(False)
  252. ),
  253. fname=fname,
  254. )
  255. except configparser.Error as error:
  256. # See https://github.com/pypa/pip/issues/4893
  257. raise ConfigurationFileCouldNotBeLoaded(error=error)
  258. return parser
  259. def _load_environment_vars(self):
  260. # type: () -> None
  261. """Loads configuration from environment variables
  262. """
  263. self._config[kinds.ENV_VAR].update(
  264. self._normalized_keys(":env:", self.get_environ_vars())
  265. )
  266. def _normalized_keys(self, section, items):
  267. # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
  268. """Normalizes items to construct a dictionary with normalized keys.
  269. This routine is where the names become keys and are made the same
  270. regardless of source - configuration files or environment.
  271. """
  272. normalized = {}
  273. for name, val in items:
  274. key = section + "." + _normalize_name(name)
  275. normalized[key] = val
  276. return normalized
  277. def get_environ_vars(self):
  278. # type: () -> Iterable[Tuple[str, str]]
  279. """Returns a generator with all environmental vars with prefix PIP_"""
  280. for key, val in os.environ.items():
  281. if key.startswith("PIP_"):
  282. name = key[4:].lower()
  283. if name not in ENV_NAMES_IGNORED:
  284. yield name, val
  285. # XXX: This is patched in the tests.
  286. def iter_config_files(self):
  287. # type: () -> Iterable[Tuple[Kind, List[str]]]
  288. """Yields variant and configuration files associated with it.
  289. This should be treated like items of a dictionary.
  290. """
  291. # SMELL: Move the conditions out of this function
  292. # environment variables have the lowest priority
  293. config_file = os.environ.get('PIP_CONFIG_FILE', None)
  294. if config_file is not None:
  295. yield kinds.ENV, [config_file]
  296. else:
  297. yield kinds.ENV, []
  298. config_files = get_configuration_files()
  299. # at the base we have any global configuration
  300. yield kinds.GLOBAL, config_files[kinds.GLOBAL]
  301. # per-user configuration next
  302. should_load_user_config = not self.isolated and not (
  303. config_file and os.path.exists(config_file)
  304. )
  305. if should_load_user_config:
  306. # The legacy config file is overridden by the new config file
  307. yield kinds.USER, config_files[kinds.USER]
  308. # finally virtualenv configuration first trumping others
  309. yield kinds.SITE, config_files[kinds.SITE]
  310. def get_values_in_config(self, variant):
  311. # type: (Kind) -> Dict[str, Any]
  312. """Get values present in a config file"""
  313. return self._config[variant]
  314. def _get_parser_to_modify(self):
  315. # type: () -> Tuple[str, RawConfigParser]
  316. # Determine which parser to modify
  317. assert self.load_only
  318. parsers = self._parsers[self.load_only]
  319. if not parsers:
  320. # This should not happen if everything works correctly.
  321. raise ConfigurationError(
  322. "Fatal Internal error [id=2]. Please report as a bug."
  323. )
  324. # Use the highest priority parser.
  325. return parsers[-1]
  326. # XXX: This is patched in the tests.
  327. def _mark_as_modified(self, fname, parser):
  328. # type: (str, RawConfigParser) -> None
  329. file_parser_tuple = (fname, parser)
  330. if file_parser_tuple not in self._modified_parsers:
  331. self._modified_parsers.append(file_parser_tuple)
  332. def __repr__(self):
  333. # type: () -> str
  334. return "{}({!r})".format(self.__class__.__name__, self._dictionary)