Skip to content

Commit

Permalink
Capture stdout and stderr in pyodide execution
Browse files Browse the repository at this point in the history
  • Loading branch information
@philippjfr
philippjfr committed Sep 17, 2022
1 parent be08046 commit 5cd028e
Show file tree
Hide file tree
Showing 3 changed files with 148 additions and 87 deletions.
113 changes: 48 additions & 65 deletions panel/io/mime_render.py
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,30 @@
"""
Utilities for executing Python code and rendering the resulting output
using a similar MIME-type based rendering system as implemented by
IPython.
Attempts to limit the actual MIME types that have to be rendered on
to a minimum simplifying frontend implementation:
- application/bokeh: Model JSON representation
- text/plain: HTML escaped string output
- text/html: HTML code to insert into the DOM
"""

from __future__ import annotations

import ast
import base64
import copy
import io
import json
import traceback

from contextlib import redirect_stderr, redirect_stdout
from html import escape
from typing import Any, Dict, Tuple
from typing import Any, Dict

import markdown

from bokeh import __version__
from bokeh.document import Document
from bokeh.embed.util import standalone_docs_json_and_render_items
from bokeh.model import Model

from ..pane import panel
from ..viewable import Viewable, Viewer
from .state import state

UNDEFINED = object()

#---------------------------------------------------------------------
Expand Down Expand Up @@ -49,7 +54,8 @@ def exec_with_return(code: str, global_context: Dict[str, Any] = None) -> Any:
Returns
-------
The return value of the executed code.
The return value of the executed code and stdout and stederr output.
"""
global_context = global_context if global_context else globals()
code_ast = ast.parse(code)
Expand All @@ -60,18 +66,25 @@ def exec_with_return(code: str, global_context: Dict[str, Any] = None) -> Any:
last_ast = copy.deepcopy(code_ast)
last_ast.body = code_ast.body[-1:]

exec(compile(init_ast, "<ast>", "exec"), global_context)
if type(last_ast.body[0]) == ast.Expr:
return eval(compile(_convert_expr(last_ast.body[0]), "<ast>", "eval"), global_context)
else:
exec(compile(last_ast, "<ast>", "exec"), global_context)
return UNDEFINED
stdout = io.StringIO()
stderr = io.StringIO()
with redirect_stdout(stdout), redirect_stderr(stderr):
try:
exec(compile(init_ast, "<ast>", "exec"), global_context)
if type(last_ast.body[0]) == ast.Expr:
out = eval(compile(_convert_expr(last_ast.body[0]), "<ast>", "eval"), global_context)
else:
exec(compile(last_ast, "<ast>", "exec"), global_context)
out = UNDEFINED
except Exception:
out = UNDEFINED
traceback.print_exc(file=stderr)
return out, stdout.getvalue(), stderr.getvalue()

#---------------------------------------------------------------------
# MIME Render API
#---------------------------------------------------------------------


MIME_METHODS = {
"__repr__": "text/plain",
"_repr_html_": "text/html",
Expand All @@ -83,10 +96,14 @@ def exec_with_return(code: str, global_context: Dict[str, Any] = None) -> Any:
"_repr_latex": "text/latex",
"_repr_json_": "application/json",
"_repr_javascript_": "application/javascript",
"savefig": "image/png",
"servable": ""
"savefig": "image/png"
}

# Rendering function

def render_svg(value, meta, mime):
return value, 'text/html'

def render_image(value, meta, mime):
data = f"data:{mime};charset=utf-8;base64,{value}"
attrs = " ".join(['{k}="{v}"' for k, v in meta.items()])
Expand All @@ -100,56 +117,27 @@ def render_markdown(value, meta, mime):
value, extensions=["extra", "smarty", "codehilite"], output_format='html5'
), 'text/html')

def render_pdf(value, meta, mime):
data = value.encode('utf-8')
base64_pdf = base64.b64encode(data).decode("utf-8")
src = f"data:application/pdf;base64,{base64_pdf}"
return f'<embed src="{src}" width="100%" height="100%" type="application/pdf">', 'text/html'

def identity(value, meta, mime):
return value, mime

MIME_RENDERERS = {
"text/plain": identity,
"text/html": identity,
"image/png": render_image,
"image/jpeg": render_image,
"image/svg+xml": identity,
"application/json": identity,
"application/javascript": render_javascript,
"text/markdown": render_markdown
"application/pdf": render_pdf,
"text/html": identity,
"text/markdown": render_markdown,
"text/plain": identity,
}

def _model_json(model: Model, target: str) -> Tuple[Document, str]:
"""
Renders a Bokeh Model to JSON representation given a particular
DOM target and returns the Document and the serialized JSON string.
Arguments
---------
model: bokeh.model.Model
The bokeh model to render.
target: str
The id of the DOM node to render to.
Returns
-------
document: Document
The bokeh Document containing the rendered Bokeh Model.
model_json: str
The serialized JSON representation of the Bokeh Model.
"""
doc = Document()
model.server_doc(doc=doc)
model = doc.roots[0]
docs_json, _ = standalone_docs_json_and_render_items(
[model], suppress_callback_warning=True
)

doc_json = list(docs_json.values())[0]
root_id = doc_json['roots']['root_ids'][0]

return doc, json.dumps(dict(
target_id = target,
root_id = root_id,
doc = doc_json,
version = __version__,
))

def eval_formatter(obj, print_method):
"""
Evaluates a formatter method.
Expand All @@ -173,11 +161,6 @@ def format_mime(obj):
"""
if isinstance(obj, str):
return escape(obj), "text/plain"
elif isinstance(obj, (Model, Viewable, Viewer)):
doc, out = _model_json(panel(obj), 'output-${msg.id}')
state.cache['${msg.id}'] = doc
return out, 'application/bokeh'

mimebundle = eval_formatter(obj, "_repr_mimebundle_")
if isinstance(mimebundle, tuple):
format_dict, _ = mimebundle
Expand Down
89 changes: 84 additions & 5 deletions panel/io/pyodide.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,26 @@

import pyodide # isort: split

from bokeh import __version__
from bokeh.document import Document
from bokeh.embed.elements import script_for_render_items
from bokeh.embed.util import standalone_docs_json_and_render_items
from bokeh.embed.wrappers import wrap_in_script_tag
from bokeh.io.doc import set_curdoc
from bokeh.model import Model
from bokeh.protocol.messages.patch_doc import process_document_events
from js import JSON

from ..config import config
from ..util import isurl
from . import resources
from .document import MockSessionContext
from .mime_render import _model_json
from .mime_render import UNDEFINED, exec_with_return, format_mime
from .state import state

resources.RESOURCE_MODE = 'CDN'
os.environ['BOKEH_RESOURCES'] = 'cdn'


#---------------------------------------------------------------------
# Private API
#---------------------------------------------------------------------
Expand Down Expand Up @@ -92,6 +93,42 @@ def _doc_json(doc: Document, root_els=None) -> Tuple[str, str, str]:
})
return json.dumps(docs_json), json.dumps(render_items_json), json.dumps(root_ids)

def _model_json(model: Model, target: str) -> Tuple[Document, str]:
"""
Renders a Bokeh Model to JSON representation given a particular
DOM target and returns the Document and the serialized JSON string.
Arguments
---------
model: bokeh.model.Model
The bokeh model to render.
target: str
The id of the DOM node to render to.
Returns
-------
document: Document
The bokeh Document containing the rendered Bokeh Model.
model_json: str
The serialized JSON representation of the Bokeh Model.
"""
doc = Document()
model.server_doc(doc=doc)
model = doc.roots[0]
docs_json, _ = standalone_docs_json_and_render_items(
[model], suppress_callback_warning=True
)

doc_json = list(docs_json.values())[0]
root_id = doc_json['roots']['root_ids'][0]

return doc, json.dumps(dict(
target_id = target,
root_id = root_id,
doc = doc_json,
version = __version__,
))

def _link_docs(pydoc: Document, jsdoc: Any) -> None:
"""
Links Python and JS documents in Pyodide ensuring taht messages
Expand Down Expand Up @@ -128,7 +165,17 @@ def pysync(event):
def _link_docs_worker(doc: Document, dispatch_fn: Any, msg_id: str | None = None):
"""
Links the Python document to a dispatch_fn which can be used to
sync messages between a WebWorker and the main thread.
sync messages between a WebWorker and the main thread in the
browser.
Arguments
---------
doc: bokeh.document.Document
The document to dispatch messages from.
dispatch_fn: JS function
The Javascript function to dispatch messages to.
msg_id: str | None
An optional message ID to pass through to the dispatch_fn.
"""
def pysync(event):
json_patch, buffers = process_document_events([event], use_buffers=True)
Expand All @@ -150,9 +197,9 @@ async def _link_model(ref: str, doc: Document) -> None:
Arguments
---------
ref: str
The ID of the rendered Bokeh Model.
The ID of the rendered bokeh Model
doc: bokeh.document.Document
The Bokeh Document to sync the rendered Model with.
The bokeh Document to sync the rendered Model with.
"""
from js import Bokeh
rendered = Bokeh.index.object_keys()
Expand Down Expand Up @@ -265,6 +312,7 @@ async def write_doc(doc: Document | None = None) -> Tuple[str, str, str]:
Arguments
---------
doc: Document
The document to render to JSON.
Returns
-------
Expand Down Expand Up @@ -295,3 +343,34 @@ async def write_doc(doc: Document | None = None) -> Tuple[str, str, str]:
_link_docs(pydoc, jsdoc)
hide_loader()
return docs_json, render_items, root_ids

def pyrender(code: str, msg_id: str):
"""
Executes Python code and returns a MIME representation of the
return value.
Arguments
---------
code: str
Python code to execute
msg_id: str
A unique ID associated with the output being rendered.
Returns
-------
Returns an JS Map containing the content, mime_type, stdout and stderr.
"""
from ..pane import panel as as_panel
from ..viewable import Viewable, Viewer
out, stdout, stderr = exec_with_return(code)
ret = {
'stdout': stdout,
'stderr': stderr
}
if isinstance(out, (Model, Viewable, Viewer)):
doc, model_json = _model_json(as_panel(out), msg_id)
state.cache[msg_id] = doc
ret['content'], ret['mime_type'] = model_json, 'application/bokeh'
elif out is not UNDEFINED:
ret['content'], ret['mime_type'] = format_mime(out)
return pyodide.ffi.to_js(ret)
33 changes: 16 additions & 17 deletions panel/tests/io/test_mime_render.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import json
import pathlib

from bokeh.models import Slider

from panel.io.mime_render import UNDEFINED, exec_with_return, format_mime
from panel.widgets import FloatSlider


class HTML:
Expand Down Expand Up @@ -32,13 +28,26 @@ def _repr_png_(self):


def test_exec_with_return_multi_line():
assert exec_with_return('a = 1\nb = 2\na + b') == 3
assert exec_with_return('a = 1\nb = 2\na + b') == (3, '', '')

def test_exec_with_return_no_return():
assert exec_with_return('a = 1') is UNDEFINED
assert exec_with_return('a = 1') == (UNDEFINED, '', '')

def test_exec_with_return_None():
assert exec_with_return('None') is None
assert exec_with_return('None') == (None, '', '')

def test_exec_captures_print():
assert exec_with_return('print("foo")') == (None, 'foo\n', '')

def test_exec_captures_error():
exc = (
'Traceback (most recent call last):\n'
' File "/Users/philippjfr/development/panel/panel/io/mime_render.py", line 77, in exec_with_return\n'
' exec(compile(last_ast, "<ast>", "exec"), global_context)\n'
' File "<ast>", line 1, in <module>\n'
'ValueError: bar\n'
)
assert exec_with_return('raise ValueError("bar")') == (UNDEFINED, '', exc)

def test_format_mime_None():
assert format_mime(None) == ('None', 'text/plain')
Expand All @@ -62,13 +71,3 @@ def test_format_mime_repr_png():
img, mime_type = format_mime(PNG())
assert mime_type == 'text/html'
assert img.startswith('<img src="data:image/png')

def test_format_mime_panel_obj():
model_json, mime_type = format_mime(FloatSlider())
assert mime_type == 'application/bokeh'
assert 'doc' in json.loads(model_json)

def test_format_mime_bokeh_obj():
model_json, mime_type = format_mime(Slider())
assert mime_type == 'application/bokeh'
assert 'doc' in json.loads(model_json)

0 comments on commit 5cd028e

Please sign in to comment.