backend_webagg_core.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. """
  2. Displays Agg images in the browser, with interactivity
  3. """
  4. # The WebAgg backend is divided into two modules:
  5. #
  6. # - `backend_webagg_core.py` contains code necessary to embed a WebAgg
  7. # plot inside of a web application, and communicate in an abstract
  8. # way over a web socket.
  9. #
  10. # - `backend_webagg.py` contains a concrete implementation of a basic
  11. # application, implemented with tornado.
  12. import datetime
  13. from io import BytesIO, StringIO
  14. import json
  15. import logging
  16. import os
  17. from pathlib import Path
  18. import numpy as np
  19. from PIL import Image
  20. import tornado
  21. from matplotlib import backend_bases, cbook
  22. from matplotlib.backends import backend_agg
  23. from matplotlib.backend_bases import _Backend
  24. _log = logging.getLogger(__name__)
  25. # http://www.cambiaresearch.com/articles/15/javascript-char-codes-key-codes
  26. _SHIFT_LUT = {59: ':',
  27. 61: '+',
  28. 173: '_',
  29. 186: ':',
  30. 187: '+',
  31. 188: '<',
  32. 189: '_',
  33. 190: '>',
  34. 191: '?',
  35. 192: '~',
  36. 219: '{',
  37. 220: '|',
  38. 221: '}',
  39. 222: '"'}
  40. _LUT = {8: 'backspace',
  41. 9: 'tab',
  42. 13: 'enter',
  43. 16: 'shift',
  44. 17: 'control',
  45. 18: 'alt',
  46. 19: 'pause',
  47. 20: 'caps',
  48. 27: 'escape',
  49. 32: ' ',
  50. 33: 'pageup',
  51. 34: 'pagedown',
  52. 35: 'end',
  53. 36: 'home',
  54. 37: 'left',
  55. 38: 'up',
  56. 39: 'right',
  57. 40: 'down',
  58. 45: 'insert',
  59. 46: 'delete',
  60. 91: 'super',
  61. 92: 'super',
  62. 93: 'select',
  63. 106: '*',
  64. 107: '+',
  65. 109: '-',
  66. 110: '.',
  67. 111: '/',
  68. 144: 'num_lock',
  69. 145: 'scroll_lock',
  70. 186: ':',
  71. 187: '=',
  72. 188: ',',
  73. 189: '-',
  74. 190: '.',
  75. 191: '/',
  76. 192: '`',
  77. 219: '[',
  78. 220: '\\',
  79. 221: ']',
  80. 222: "'"}
  81. def _handle_key(key):
  82. """Handle key codes"""
  83. code = int(key[key.index('k') + 1:])
  84. value = chr(code)
  85. # letter keys
  86. if 65 <= code <= 90:
  87. if 'shift+' in key:
  88. key = key.replace('shift+', '')
  89. else:
  90. value = value.lower()
  91. # number keys
  92. elif 48 <= code <= 57:
  93. if 'shift+' in key:
  94. value = ')!@#$%^&*('[int(value)]
  95. key = key.replace('shift+', '')
  96. # function keys
  97. elif 112 <= code <= 123:
  98. value = 'f%s' % (code - 111)
  99. # number pad keys
  100. elif 96 <= code <= 105:
  101. value = '%s' % (code - 96)
  102. # keys with shift alternatives
  103. elif code in _SHIFT_LUT and 'shift+' in key:
  104. key = key.replace('shift+', '')
  105. value = _SHIFT_LUT[code]
  106. elif code in _LUT:
  107. value = _LUT[code]
  108. key = key[:key.index('k')] + value
  109. return key
  110. class FigureCanvasWebAggCore(backend_agg.FigureCanvasAgg):
  111. supports_blit = False
  112. def __init__(self, *args, **kwargs):
  113. backend_agg.FigureCanvasAgg.__init__(self, *args, **kwargs)
  114. # Set to True when the renderer contains data that is newer
  115. # than the PNG buffer.
  116. self._png_is_old = True
  117. # Set to True by the `refresh` message so that the next frame
  118. # sent to the clients will be a full frame.
  119. self._force_full = True
  120. # Store the current image mode so that at any point, clients can
  121. # request the information. This should be changed by calling
  122. # self.set_image_mode(mode) so that the notification can be given
  123. # to the connected clients.
  124. self._current_image_mode = 'full'
  125. # Store the DPI ratio of the browser. This is the scaling that
  126. # occurs automatically for all images on a HiDPI display.
  127. self._dpi_ratio = 1
  128. def show(self):
  129. # show the figure window
  130. from matplotlib.pyplot import show
  131. show()
  132. def draw(self):
  133. self._png_is_old = True
  134. try:
  135. super().draw()
  136. finally:
  137. self.manager.refresh_all() # Swap the frames.
  138. def draw_idle(self):
  139. self.send_event("draw")
  140. def set_image_mode(self, mode):
  141. """
  142. Set the image mode for any subsequent images which will be sent
  143. to the clients. The modes may currently be either 'full' or 'diff'.
  144. Note: diff images may not contain transparency, therefore upon
  145. draw this mode may be changed if the resulting image has any
  146. transparent component.
  147. """
  148. cbook._check_in_list(['full', 'diff'], mode=mode)
  149. if self._current_image_mode != mode:
  150. self._current_image_mode = mode
  151. self.handle_send_image_mode(None)
  152. def get_diff_image(self):
  153. if self._png_is_old:
  154. renderer = self.get_renderer()
  155. # The buffer is created as type uint32 so that entire
  156. # pixels can be compared in one numpy call, rather than
  157. # needing to compare each plane separately.
  158. buff = (np.frombuffer(renderer.buffer_rgba(), dtype=np.uint32)
  159. .reshape((renderer.height, renderer.width)))
  160. # If any pixels have transparency, we need to force a full
  161. # draw as we cannot overlay new on top of old.
  162. pixels = buff.view(dtype=np.uint8).reshape(buff.shape + (4,))
  163. if self._force_full or np.any(pixels[:, :, 3] != 255):
  164. self.set_image_mode('full')
  165. output = buff
  166. else:
  167. self.set_image_mode('diff')
  168. last_buffer = (np.frombuffer(self._last_renderer.buffer_rgba(),
  169. dtype=np.uint32)
  170. .reshape((renderer.height, renderer.width)))
  171. diff = buff != last_buffer
  172. output = np.where(diff, buff, 0)
  173. buf = BytesIO()
  174. data = output.view(dtype=np.uint8).reshape((*output.shape, 4))
  175. Image.fromarray(data).save(buf, format="png")
  176. # Swap the renderer frames
  177. self._renderer, self._last_renderer = (
  178. self._last_renderer, renderer)
  179. self._force_full = False
  180. self._png_is_old = False
  181. return buf.getvalue()
  182. def get_renderer(self, cleared=None):
  183. # Mirrors super.get_renderer, but caches the old one so that we can do
  184. # things such as produce a diff image in get_diff_image.
  185. w, h = self.figure.bbox.size.astype(int)
  186. key = w, h, self.figure.dpi
  187. try:
  188. self._lastKey, self._renderer
  189. except AttributeError:
  190. need_new_renderer = True
  191. else:
  192. need_new_renderer = (self._lastKey != key)
  193. if need_new_renderer:
  194. self._renderer = backend_agg.RendererAgg(
  195. w, h, self.figure.dpi)
  196. self._last_renderer = backend_agg.RendererAgg(
  197. w, h, self.figure.dpi)
  198. self._lastKey = key
  199. elif cleared:
  200. self._renderer.clear()
  201. return self._renderer
  202. def handle_event(self, event):
  203. e_type = event['type']
  204. handler = getattr(self, 'handle_{0}'.format(e_type),
  205. self.handle_unknown_event)
  206. return handler(event)
  207. def handle_unknown_event(self, event):
  208. _log.warning('Unhandled message type {0}. {1}'.format(
  209. event['type'], event))
  210. def handle_ack(self, event):
  211. # Network latency tends to decrease if traffic is flowing
  212. # in both directions. Therefore, the browser sends back
  213. # an "ack" message after each image frame is received.
  214. # This could also be used as a simple sanity check in the
  215. # future, but for now the performance increase is enough
  216. # to justify it, even if the server does nothing with it.
  217. pass
  218. def handle_draw(self, event):
  219. self.draw()
  220. def _handle_mouse(self, event):
  221. x = event['x']
  222. y = event['y']
  223. y = self.get_renderer().height - y
  224. # Javascript button numbers and matplotlib button numbers are
  225. # off by 1
  226. button = event['button'] + 1
  227. # The right mouse button pops up a context menu, which
  228. # doesn't work very well, so use the middle mouse button
  229. # instead. It doesn't seem that it's possible to disable
  230. # the context menu in recent versions of Chrome. If this
  231. # is resolved, please also adjust the docstring in MouseEvent.
  232. if button == 2:
  233. button = 3
  234. e_type = event['type']
  235. guiEvent = event.get('guiEvent', None)
  236. if e_type == 'button_press':
  237. self.button_press_event(x, y, button, guiEvent=guiEvent)
  238. elif e_type == 'button_release':
  239. self.button_release_event(x, y, button, guiEvent=guiEvent)
  240. elif e_type == 'motion_notify':
  241. self.motion_notify_event(x, y, guiEvent=guiEvent)
  242. elif e_type == 'figure_enter':
  243. self.enter_notify_event(xy=(x, y), guiEvent=guiEvent)
  244. elif e_type == 'figure_leave':
  245. self.leave_notify_event()
  246. elif e_type == 'scroll':
  247. self.scroll_event(x, y, event['step'], guiEvent=guiEvent)
  248. handle_button_press = handle_button_release = handle_motion_notify = \
  249. handle_figure_enter = handle_figure_leave = handle_scroll = \
  250. _handle_mouse
  251. def _handle_key(self, event):
  252. key = _handle_key(event['key'])
  253. e_type = event['type']
  254. guiEvent = event.get('guiEvent', None)
  255. if e_type == 'key_press':
  256. self.key_press_event(key, guiEvent=guiEvent)
  257. elif e_type == 'key_release':
  258. self.key_release_event(key, guiEvent=guiEvent)
  259. handle_key_press = handle_key_release = _handle_key
  260. def handle_toolbar_button(self, event):
  261. # TODO: Be more suspicious of the input
  262. getattr(self.toolbar, event['name'])()
  263. def handle_refresh(self, event):
  264. figure_label = self.figure.get_label()
  265. if not figure_label:
  266. figure_label = "Figure {0}".format(self.manager.num)
  267. self.send_event('figure_label', label=figure_label)
  268. self._force_full = True
  269. if self.toolbar:
  270. # Normal toolbar init would refresh this, but it happens before the
  271. # browser canvas is set up.
  272. self.toolbar.set_history_buttons()
  273. self.draw_idle()
  274. def handle_resize(self, event):
  275. x, y = event.get('width', 800), event.get('height', 800)
  276. x, y = int(x) * self._dpi_ratio, int(y) * self._dpi_ratio
  277. fig = self.figure
  278. # An attempt at approximating the figure size in pixels.
  279. fig.set_size_inches(x / fig.dpi, y / fig.dpi, forward=False)
  280. # Acknowledge the resize, and force the viewer to update the
  281. # canvas size to the figure's new size (which is hopefully
  282. # identical or within a pixel or so).
  283. self._png_is_old = True
  284. self.manager.resize(*fig.bbox.size, forward=False)
  285. self.resize_event()
  286. def handle_send_image_mode(self, event):
  287. # The client requests notification of what the current image mode is.
  288. self.send_event('image_mode', mode=self._current_image_mode)
  289. def handle_set_dpi_ratio(self, event):
  290. dpi_ratio = event.get('dpi_ratio', 1)
  291. if dpi_ratio != self._dpi_ratio:
  292. # We don't want to scale up the figure dpi more than once.
  293. if not hasattr(self.figure, '_original_dpi'):
  294. self.figure._original_dpi = self.figure.dpi
  295. self.figure.dpi = dpi_ratio * self.figure._original_dpi
  296. self._dpi_ratio = dpi_ratio
  297. self._force_full = True
  298. self.draw_idle()
  299. def send_event(self, event_type, **kwargs):
  300. if self.manager:
  301. self.manager._send_event(event_type, **kwargs)
  302. _ALLOWED_TOOL_ITEMS = {
  303. 'home',
  304. 'back',
  305. 'forward',
  306. 'pan',
  307. 'zoom',
  308. 'download',
  309. None,
  310. }
  311. class NavigationToolbar2WebAgg(backend_bases.NavigationToolbar2):
  312. # Use the standard toolbar items + download button
  313. toolitems = [
  314. (text, tooltip_text, image_file, name_of_method)
  315. for text, tooltip_text, image_file, name_of_method
  316. in (*backend_bases.NavigationToolbar2.toolitems,
  317. ('Download', 'Download plot', 'filesave', 'download'))
  318. if name_of_method in _ALLOWED_TOOL_ITEMS
  319. ]
  320. def __init__(self, canvas):
  321. self.message = ''
  322. self.cursor = 0
  323. super().__init__(canvas)
  324. def set_message(self, message):
  325. if message != self.message:
  326. self.canvas.send_event("message", message=message)
  327. self.message = message
  328. def set_cursor(self, cursor):
  329. if cursor != self.cursor:
  330. self.canvas.send_event("cursor", cursor=cursor)
  331. self.cursor = cursor
  332. def draw_rubberband(self, event, x0, y0, x1, y1):
  333. self.canvas.send_event(
  334. "rubberband", x0=x0, y0=y0, x1=x1, y1=y1)
  335. def release_zoom(self, event):
  336. backend_bases.NavigationToolbar2.release_zoom(self, event)
  337. self.canvas.send_event(
  338. "rubberband", x0=-1, y0=-1, x1=-1, y1=-1)
  339. def save_figure(self, *args):
  340. """Save the current figure"""
  341. self.canvas.send_event('save')
  342. def pan(self):
  343. super().pan()
  344. self.canvas.send_event('navigate_mode', mode=self.mode.name)
  345. def zoom(self):
  346. super().zoom()
  347. self.canvas.send_event('navigate_mode', mode=self.mode.name)
  348. def set_history_buttons(self):
  349. can_backward = self._nav_stack._pos > 0
  350. can_forward = self._nav_stack._pos < len(self._nav_stack._elements) - 1
  351. self.canvas.send_event('history_buttons',
  352. Back=can_backward, Forward=can_forward)
  353. class FigureManagerWebAgg(backend_bases.FigureManagerBase):
  354. ToolbarCls = NavigationToolbar2WebAgg
  355. def __init__(self, canvas, num):
  356. backend_bases.FigureManagerBase.__init__(self, canvas, num)
  357. self.web_sockets = set()
  358. self.toolbar = self._get_toolbar(canvas)
  359. def show(self):
  360. pass
  361. def _get_toolbar(self, canvas):
  362. toolbar = self.ToolbarCls(canvas)
  363. return toolbar
  364. def resize(self, w, h, forward=True):
  365. self._send_event(
  366. 'resize',
  367. size=(w / self.canvas._dpi_ratio, h / self.canvas._dpi_ratio),
  368. forward=forward)
  369. def set_window_title(self, title):
  370. self._send_event('figure_label', label=title)
  371. # The following methods are specific to FigureManagerWebAgg
  372. def add_web_socket(self, web_socket):
  373. assert hasattr(web_socket, 'send_binary')
  374. assert hasattr(web_socket, 'send_json')
  375. self.web_sockets.add(web_socket)
  376. self.resize(*self.canvas.figure.bbox.size)
  377. self._send_event('refresh')
  378. def remove_web_socket(self, web_socket):
  379. self.web_sockets.remove(web_socket)
  380. def handle_json(self, content):
  381. self.canvas.handle_event(content)
  382. def refresh_all(self):
  383. if self.web_sockets:
  384. diff = self.canvas.get_diff_image()
  385. if diff is not None:
  386. for s in self.web_sockets:
  387. s.send_binary(diff)
  388. @classmethod
  389. def get_javascript(cls, stream=None):
  390. if stream is None:
  391. output = StringIO()
  392. else:
  393. output = stream
  394. output.write((Path(__file__).parent / "web_backend/js/mpl.js")
  395. .read_text(encoding="utf-8"))
  396. toolitems = []
  397. for name, tooltip, image, method in cls.ToolbarCls.toolitems:
  398. if name is None:
  399. toolitems.append(['', '', '', ''])
  400. else:
  401. toolitems.append([name, tooltip, image, method])
  402. output.write("mpl.toolbar_items = {0};\n\n".format(
  403. json.dumps(toolitems)))
  404. extensions = []
  405. for filetype, ext in sorted(FigureCanvasWebAggCore.
  406. get_supported_filetypes_grouped().
  407. items()):
  408. if ext[0] != 'pgf': # pgf does not support BytesIO
  409. extensions.append(ext[0])
  410. output.write("mpl.extensions = {0};\n\n".format(
  411. json.dumps(extensions)))
  412. output.write("mpl.default_extension = {0};".format(
  413. json.dumps(FigureCanvasWebAggCore.get_default_filetype())))
  414. if stream is None:
  415. return output.getvalue()
  416. @classmethod
  417. def get_static_file_path(cls):
  418. return os.path.join(os.path.dirname(__file__), 'web_backend')
  419. def _send_event(self, event_type, **kwargs):
  420. payload = {'type': event_type, **kwargs}
  421. for s in self.web_sockets:
  422. s.send_json(payload)
  423. class TimerTornado(backend_bases.TimerBase):
  424. def __init__(self, *args, **kwargs):
  425. self._timer = None
  426. super().__init__(*args, **kwargs)
  427. def _timer_start(self):
  428. self._timer_stop()
  429. if self._single:
  430. ioloop = tornado.ioloop.IOLoop.instance()
  431. self._timer = ioloop.add_timeout(
  432. datetime.timedelta(milliseconds=self.interval),
  433. self._on_timer)
  434. else:
  435. self._timer = tornado.ioloop.PeriodicCallback(
  436. self._on_timer,
  437. max(self.interval, 1e-6))
  438. self._timer.start()
  439. def _timer_stop(self):
  440. if self._timer is None:
  441. return
  442. elif self._single:
  443. ioloop = tornado.ioloop.IOLoop.instance()
  444. ioloop.remove_timeout(self._timer)
  445. else:
  446. self._timer.stop()
  447. self._timer = None
  448. def _timer_set_interval(self):
  449. # Only stop and restart it if the timer has already been started
  450. if self._timer is not None:
  451. self._timer_stop()
  452. self._timer_start()
  453. @_Backend.export
  454. class _BackendWebAggCoreAgg(_Backend):
  455. FigureCanvas = FigureCanvasWebAggCore
  456. FigureManager = FigureManagerWebAgg