ordered_set.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. """
  2. An OrderedSet is a custom MutableSet that remembers its order, so that every
  3. entry has an index that can be looked up.
  4. Based on a recipe originally posted to ActiveState Recipes by Raymond Hettiger,
  5. and released under the MIT license.
  6. """
  7. import itertools as it
  8. from collections import deque
  9. try:
  10. # Python 3
  11. from collections.abc import MutableSet, Sequence
  12. except ImportError:
  13. # Python 2.7
  14. from collections import MutableSet, Sequence
  15. SLICE_ALL = slice(None)
  16. __version__ = "3.1"
  17. def is_iterable(obj):
  18. """
  19. Are we being asked to look up a list of things, instead of a single thing?
  20. We check for the `__iter__` attribute so that this can cover types that
  21. don't have to be known by this module, such as NumPy arrays.
  22. Strings, however, should be considered as atomic values to look up, not
  23. iterables. The same goes for tuples, since they are immutable and therefore
  24. valid entries.
  25. We don't need to check for the Python 2 `unicode` type, because it doesn't
  26. have an `__iter__` attribute anyway.
  27. """
  28. return (
  29. hasattr(obj, "__iter__")
  30. and not isinstance(obj, str)
  31. and not isinstance(obj, tuple)
  32. )
  33. class OrderedSet(MutableSet, Sequence):
  34. """
  35. An OrderedSet is a custom MutableSet that remembers its order, so that
  36. every entry has an index that can be looked up.
  37. Example:
  38. >>> OrderedSet([1, 1, 2, 3, 2])
  39. OrderedSet([1, 2, 3])
  40. """
  41. def __init__(self, iterable=None):
  42. self.items = []
  43. self.map = {}
  44. if iterable is not None:
  45. self |= iterable
  46. def __len__(self):
  47. """
  48. Returns the number of unique elements in the ordered set
  49. Example:
  50. >>> len(OrderedSet([]))
  51. 0
  52. >>> len(OrderedSet([1, 2]))
  53. 2
  54. """
  55. return len(self.items)
  56. def __getitem__(self, index):
  57. """
  58. Get the item at a given index.
  59. If `index` is a slice, you will get back that slice of items, as a
  60. new OrderedSet.
  61. If `index` is a list or a similar iterable, you'll get a list of
  62. items corresponding to those indices. This is similar to NumPy's
  63. "fancy indexing". The result is not an OrderedSet because you may ask
  64. for duplicate indices, and the number of elements returned should be
  65. the number of elements asked for.
  66. Example:
  67. >>> oset = OrderedSet([1, 2, 3])
  68. >>> oset[1]
  69. 2
  70. """
  71. if isinstance(index, slice) and index == SLICE_ALL:
  72. return self.copy()
  73. elif is_iterable(index):
  74. return [self.items[i] for i in index]
  75. elif hasattr(index, "__index__") or isinstance(index, slice):
  76. result = self.items[index]
  77. if isinstance(result, list):
  78. return self.__class__(result)
  79. else:
  80. return result
  81. else:
  82. raise TypeError("Don't know how to index an OrderedSet by %r" % index)
  83. def copy(self):
  84. """
  85. Return a shallow copy of this object.
  86. Example:
  87. >>> this = OrderedSet([1, 2, 3])
  88. >>> other = this.copy()
  89. >>> this == other
  90. True
  91. >>> this is other
  92. False
  93. """
  94. return self.__class__(self)
  95. def __getstate__(self):
  96. if len(self) == 0:
  97. # The state can't be an empty list.
  98. # We need to return a truthy value, or else __setstate__ won't be run.
  99. #
  100. # This could have been done more gracefully by always putting the state
  101. # in a tuple, but this way is backwards- and forwards- compatible with
  102. # previous versions of OrderedSet.
  103. return (None,)
  104. else:
  105. return list(self)
  106. def __setstate__(self, state):
  107. if state == (None,):
  108. self.__init__([])
  109. else:
  110. self.__init__(state)
  111. def __contains__(self, key):
  112. """
  113. Test if the item is in this ordered set
  114. Example:
  115. >>> 1 in OrderedSet([1, 3, 2])
  116. True
  117. >>> 5 in OrderedSet([1, 3, 2])
  118. False
  119. """
  120. return key in self.map
  121. def add(self, key):
  122. """
  123. Add `key` as an item to this OrderedSet, then return its index.
  124. If `key` is already in the OrderedSet, return the index it already
  125. had.
  126. Example:
  127. >>> oset = OrderedSet()
  128. >>> oset.append(3)
  129. 0
  130. >>> print(oset)
  131. OrderedSet([3])
  132. """
  133. if key not in self.map:
  134. self.map[key] = len(self.items)
  135. self.items.append(key)
  136. return self.map[key]
  137. append = add
  138. def update(self, sequence):
  139. """
  140. Update the set with the given iterable sequence, then return the index
  141. of the last element inserted.
  142. Example:
  143. >>> oset = OrderedSet([1, 2, 3])
  144. >>> oset.update([3, 1, 5, 1, 4])
  145. 4
  146. >>> print(oset)
  147. OrderedSet([1, 2, 3, 5, 4])
  148. """
  149. item_index = None
  150. try:
  151. for item in sequence:
  152. item_index = self.add(item)
  153. except TypeError:
  154. raise ValueError(
  155. "Argument needs to be an iterable, got %s" % type(sequence)
  156. )
  157. return item_index
  158. def index(self, key):
  159. """
  160. Get the index of a given entry, raising an IndexError if it's not
  161. present.
  162. `key` can be an iterable of entries that is not a string, in which case
  163. this returns a list of indices.
  164. Example:
  165. >>> oset = OrderedSet([1, 2, 3])
  166. >>> oset.index(2)
  167. 1
  168. """
  169. if is_iterable(key):
  170. return [self.index(subkey) for subkey in key]
  171. return self.map[key]
  172. # Provide some compatibility with pd.Index
  173. get_loc = index
  174. get_indexer = index
  175. def pop(self):
  176. """
  177. Remove and return the last element from the set.
  178. Raises KeyError if the set is empty.
  179. Example:
  180. >>> oset = OrderedSet([1, 2, 3])
  181. >>> oset.pop()
  182. 3
  183. """
  184. if not self.items:
  185. raise KeyError("Set is empty")
  186. elem = self.items[-1]
  187. del self.items[-1]
  188. del self.map[elem]
  189. return elem
  190. def discard(self, key):
  191. """
  192. Remove an element. Do not raise an exception if absent.
  193. The MutableSet mixin uses this to implement the .remove() method, which
  194. *does* raise an error when asked to remove a non-existent item.
  195. Example:
  196. >>> oset = OrderedSet([1, 2, 3])
  197. >>> oset.discard(2)
  198. >>> print(oset)
  199. OrderedSet([1, 3])
  200. >>> oset.discard(2)
  201. >>> print(oset)
  202. OrderedSet([1, 3])
  203. """
  204. if key in self:
  205. i = self.map[key]
  206. del self.items[i]
  207. del self.map[key]
  208. for k, v in self.map.items():
  209. if v >= i:
  210. self.map[k] = v - 1
  211. def clear(self):
  212. """
  213. Remove all items from this OrderedSet.
  214. """
  215. del self.items[:]
  216. self.map.clear()
  217. def __iter__(self):
  218. """
  219. Example:
  220. >>> list(iter(OrderedSet([1, 2, 3])))
  221. [1, 2, 3]
  222. """
  223. return iter(self.items)
  224. def __reversed__(self):
  225. """
  226. Example:
  227. >>> list(reversed(OrderedSet([1, 2, 3])))
  228. [3, 2, 1]
  229. """
  230. return reversed(self.items)
  231. def __repr__(self):
  232. if not self:
  233. return "%s()" % (self.__class__.__name__,)
  234. return "%s(%r)" % (self.__class__.__name__, list(self))
  235. def __eq__(self, other):
  236. """
  237. Returns true if the containers have the same items. If `other` is a
  238. Sequence, then order is checked, otherwise it is ignored.
  239. Example:
  240. >>> oset = OrderedSet([1, 3, 2])
  241. >>> oset == [1, 3, 2]
  242. True
  243. >>> oset == [1, 2, 3]
  244. False
  245. >>> oset == [2, 3]
  246. False
  247. >>> oset == OrderedSet([3, 2, 1])
  248. False
  249. """
  250. # In Python 2 deque is not a Sequence, so treat it as one for
  251. # consistent behavior with Python 3.
  252. if isinstance(other, (Sequence, deque)):
  253. # Check that this OrderedSet contains the same elements, in the
  254. # same order, as the other object.
  255. return list(self) == list(other)
  256. try:
  257. other_as_set = set(other)
  258. except TypeError:
  259. # If `other` can't be converted into a set, it's not equal.
  260. return False
  261. else:
  262. return set(self) == other_as_set
  263. def union(self, *sets):
  264. """
  265. Combines all unique items.
  266. Each items order is defined by its first appearance.
  267. Example:
  268. >>> oset = OrderedSet.union(OrderedSet([3, 1, 4, 1, 5]), [1, 3], [2, 0])
  269. >>> print(oset)
  270. OrderedSet([3, 1, 4, 5, 2, 0])
  271. >>> oset.union([8, 9])
  272. OrderedSet([3, 1, 4, 5, 2, 0, 8, 9])
  273. >>> oset | {10}
  274. OrderedSet([3, 1, 4, 5, 2, 0, 10])
  275. """
  276. cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
  277. containers = map(list, it.chain([self], sets))
  278. items = it.chain.from_iterable(containers)
  279. return cls(items)
  280. def __and__(self, other):
  281. # the parent implementation of this is backwards
  282. return self.intersection(other)
  283. def intersection(self, *sets):
  284. """
  285. Returns elements in common between all sets. Order is defined only
  286. by the first set.
  287. Example:
  288. >>> oset = OrderedSet.intersection(OrderedSet([0, 1, 2, 3]), [1, 2, 3])
  289. >>> print(oset)
  290. OrderedSet([1, 2, 3])
  291. >>> oset.intersection([2, 4, 5], [1, 2, 3, 4])
  292. OrderedSet([2])
  293. >>> oset.intersection()
  294. OrderedSet([1, 2, 3])
  295. """
  296. cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
  297. if sets:
  298. common = set.intersection(*map(set, sets))
  299. items = (item for item in self if item in common)
  300. else:
  301. items = self
  302. return cls(items)
  303. def difference(self, *sets):
  304. """
  305. Returns all elements that are in this set but not the others.
  306. Example:
  307. >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2]))
  308. OrderedSet([1, 3])
  309. >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2]), OrderedSet([3]))
  310. OrderedSet([1])
  311. >>> OrderedSet([1, 2, 3]) - OrderedSet([2])
  312. OrderedSet([1, 3])
  313. >>> OrderedSet([1, 2, 3]).difference()
  314. OrderedSet([1, 2, 3])
  315. """
  316. cls = self.__class__
  317. if sets:
  318. other = set.union(*map(set, sets))
  319. items = (item for item in self if item not in other)
  320. else:
  321. items = self
  322. return cls(items)
  323. def issubset(self, other):
  324. """
  325. Report whether another set contains this set.
  326. Example:
  327. >>> OrderedSet([1, 2, 3]).issubset({1, 2})
  328. False
  329. >>> OrderedSet([1, 2, 3]).issubset({1, 2, 3, 4})
  330. True
  331. >>> OrderedSet([1, 2, 3]).issubset({1, 4, 3, 5})
  332. False
  333. """
  334. if len(self) > len(other): # Fast check for obvious cases
  335. return False
  336. return all(item in other for item in self)
  337. def issuperset(self, other):
  338. """
  339. Report whether this set contains another set.
  340. Example:
  341. >>> OrderedSet([1, 2]).issuperset([1, 2, 3])
  342. False
  343. >>> OrderedSet([1, 2, 3, 4]).issuperset({1, 2, 3})
  344. True
  345. >>> OrderedSet([1, 4, 3, 5]).issuperset({1, 2, 3})
  346. False
  347. """
  348. if len(self) < len(other): # Fast check for obvious cases
  349. return False
  350. return all(item in self for item in other)
  351. def symmetric_difference(self, other):
  352. """
  353. Return the symmetric difference of two OrderedSets as a new set.
  354. That is, the new set will contain all elements that are in exactly
  355. one of the sets.
  356. Their order will be preserved, with elements from `self` preceding
  357. elements from `other`.
  358. Example:
  359. >>> this = OrderedSet([1, 4, 3, 5, 7])
  360. >>> other = OrderedSet([9, 7, 1, 3, 2])
  361. >>> this.symmetric_difference(other)
  362. OrderedSet([4, 5, 9, 2])
  363. """
  364. cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
  365. diff1 = cls(self).difference(other)
  366. diff2 = cls(other).difference(self)
  367. return diff1.union(diff2)
  368. def _update_items(self, items):
  369. """
  370. Replace the 'items' list of this OrderedSet with a new one, updating
  371. self.map accordingly.
  372. """
  373. self.items = items
  374. self.map = {item: idx for (idx, item) in enumerate(items)}
  375. def difference_update(self, *sets):
  376. """
  377. Update this OrderedSet to remove items from one or more other sets.
  378. Example:
  379. >>> this = OrderedSet([1, 2, 3])
  380. >>> this.difference_update(OrderedSet([2, 4]))
  381. >>> print(this)
  382. OrderedSet([1, 3])
  383. >>> this = OrderedSet([1, 2, 3, 4, 5])
  384. >>> this.difference_update(OrderedSet([2, 4]), OrderedSet([1, 4, 6]))
  385. >>> print(this)
  386. OrderedSet([3, 5])
  387. """
  388. items_to_remove = set()
  389. for other in sets:
  390. items_to_remove |= set(other)
  391. self._update_items([item for item in self.items if item not in items_to_remove])
  392. def intersection_update(self, other):
  393. """
  394. Update this OrderedSet to keep only items in another set, preserving
  395. their order in this set.
  396. Example:
  397. >>> this = OrderedSet([1, 4, 3, 5, 7])
  398. >>> other = OrderedSet([9, 7, 1, 3, 2])
  399. >>> this.intersection_update(other)
  400. >>> print(this)
  401. OrderedSet([1, 3, 7])
  402. """
  403. other = set(other)
  404. self._update_items([item for item in self.items if item in other])
  405. def symmetric_difference_update(self, other):
  406. """
  407. Update this OrderedSet to remove items from another set, then
  408. add items from the other set that were not present in this set.
  409. Example:
  410. >>> this = OrderedSet([1, 4, 3, 5, 7])
  411. >>> other = OrderedSet([9, 7, 1, 3, 2])
  412. >>> this.symmetric_difference_update(other)
  413. >>> print(this)
  414. OrderedSet([4, 5, 9, 2])
  415. """
  416. items_to_add = [item for item in other if item not in self]
  417. items_to_remove = set(other)
  418. self._update_items(
  419. [item for item in self.items if item not in items_to_remove] + items_to_add
  420. )