Skip to content
This repository has been archived by the owner on Oct 26, 2019. It is now read-only.

[Issue: 169] Frontend hardcodes python3 #211

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile.kernel
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

FROM jupyter/pyspark-notebook:0017b56d93c9
FROM jupyter/all-spark-notebook:0017b56d93c9

RUN pip install jupyter_kernel_gateway==0.5.0
# install ipywidgets 5.x
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is where work needs to be done:

  1. Declarative widgets and ipywidgets need to be installed against python2 environment.
  2. Declarative widgets JAR needs to be installed against the Toree kernel.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For first one, it's something like:

RUN /opt/conda/envs/python2/bin/pip install jupyter_declarativewidgets

Check the path (should be in the docker-stacks README) and probably add a version too.

Expand Down
2 changes: 1 addition & 1 deletion client/js/dashboard.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ if (Element && !Element.prototype.matches) {
_registerKernelErrorHandler(kernel);

// initialize an ipywidgets manager
var widgetManager = new WidgetManager(kernel, _consumeMessage);
var widgetManager = new WidgetManager(kernel, Config.kernelname, _consumeMessage);

// initialize Declarative Widgets library
var widgetsReady = _initDeclWidgets();
Expand Down
2 changes: 1 addition & 1 deletion client/js/kernel.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ var Services = require('jupyter-js-services');
var kernelOptions = {
baseUrl: kernelUrl,
wsUrl: kernelUrl.replace(/^http/, 'ws'),
name: 'python3',
name: '', // Set by API when making POST request to kernel gateway
clientId: clientId,
ajaxSettings: {
requestHeaders: {
Expand Down
28 changes: 16 additions & 12 deletions client/js/widget-manager.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
var $ = require('jquery');
var Widgets = require('jupyter-js-widgets');

var WidgetManager = function(kernel, msgHandler) {
var WidgetManager = function(kernel, kernelname, msgHandler) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IKernel (type of kernel) has a name property, but it always seems to come back as empty string in my testing.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so kernel.name is an empty string because we pass an empty string now when calling startNewKernel. Instead, I think we should do this:

diff --git a/client/js/dashboard.js b/client/js/dashboard.js
index fec0656..c88d37e 100644
--- a/client/js/dashboard.js
+++ b/client/js/dashboard.js
@@ -50,7 +50,7 @@ if (Element && !Element.prototype.matches) {
     var renderMime = _createRenderMime();

     // start a kernel
-    Kernel.start().then(function(kernel) {
+    Kernel.start(Config.kernelname).then(function(kernel) {
         // do some additional shimming
         _setKernelShims(kernel);

@@ -58,7 +58,7 @@ if (Element && !Element.prototype.matches) {
         _registerKernelErrorHandler(kernel);

         // initialize an ipywidgets manager
-        var widgetManager = new WidgetManager(kernel, Config.kernelname, _consumeMessage);
+        var widgetManager = new WidgetManager(kernel, _consumeMessage);

         // initialize Declarative Widgets library
         var widgetsReady = _initDeclWidgets();
diff --git a/client/js/kernel.js b/client/js/kernel.js
index 4444210..a15135a 100644
--- a/client/js/kernel.js
+++ b/client/js/kernel.js
@@ -18,14 +18,14 @@ var Services = require('jupyter-js-services');

     var _kernel;

-    function _startKernel() {
+    function _startKernel(kernelname) {
         var loc = window.location;
         var kernelUrl = loc.protocol + '//' + loc.host;
         var clientId = _uuid();
         var kernelOptions = {
             baseUrl: kernelUrl,
             wsUrl: kernelUrl.replace(/^http/, 'ws'),
-            name: '', // Set by API when making POST request to kernel gateway
+            name: kernelname,
             clientId: clientId,
             ajaxSettings: {
                 requestHeaders: {
diff --git a/client/js/widget-manager.js b/client/js/widget-manager.js
index 278b2e8..d3a717c 100644
--- a/client/js/widget-manager.js
+++ b/client/js/widget-manager.js
@@ -10,7 +10,7 @@
 var $ = require('jquery');
 var Widgets = require('jupyter-js-widgets');

-    var WidgetManager = function(kernel, kernelname, msgHandler) {
+    var WidgetManager = function(kernel, msgHandler) {
         //  Call the base class.
         Widgets.ManagerBase.call(this);

@@ -26,7 +26,7 @@ var Widgets = require('jupyter-js-widgets');

         // Validate the version requested by the backend -- required for ipywidgets
         // If kernel name is not defined, default to python3
-        kernelname = kernelname.toLowerCase();
+        var kernelname = kernel.name.toLowerCase();
         if (!kernelname || kernelname === 'python3' || kernelname === 'python2') {
             var validate = (function validate() {
                 this.validateVersion().then(function(valid) {

Copy link
Collaborator Author

@jameslmartin jameslmartin May 16, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if this is what we want to do here. The original bug was that the front end was hardcoding 'python3' into that field, kernel.name. @parente and I discussed that the back end should determine what type of kernel should be started, rather having the front end pass along the name.

However, we do need the kernel name for the widget manager. Thoughts?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See below.

// Call the base class.
Widgets.ManagerBase.call(this);

Expand All @@ -24,17 +24,21 @@ var Widgets = require('jupyter-js-widgets');
// Register the comm target
this.commManager.register_target(this.comm_target_name, this.handle_comm_open.bind(this));

// Validate the version requested by the backend.
var validate = (function validate() {
this.validateVersion().then(function(valid) {
if (!valid) {
console.warn('Widget frontend version does not match the backend.');
}
}).catch(function(err) {
console.error('Could not cross validate the widget frontend and backend versions.', err);
});
}).bind(this);
validate();
// Validate the version requested by the backend -- required for ipywidgets
// If kernel name is not defined, default to python3
kernelname = kernelname.toLowerCase();
if (!kernelname || kernelname === 'python3' || kernelname === 'python2') {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You'll also have to include just "python" here, but I really suspect the Python check won't be required once the other issues are resolved (e.g., properly installing the decl widgets scala jar, installing ipywidgets against python2 conda env) .

Copy link
Collaborator Author

@jameslmartin jameslmartin May 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why exactly does the widget manager need to know the language?

We need it here in this check to run the validation code.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. I still think once Scala is setup properly, that special casing for Python will not be needed. I'll get with you and or @jhpedemonte today.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed with @jhpedemonte . For now, we're not going to bother setting up decl widgets for scala since the example is only hello world. Adding whatever config is necessary for a scala decl widgets example to work can come as part of #203.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jhpedemonte also clarified that the validation check does not actually inhibit the Toree kernel from working. It's just an ugly error. We agreed that switching it to a warning that explicitly says ala "Compatible version of ipywidgets is not available" is cleaner than hardcoding python checks and still makes sense with other kernels (yes, ipywidgets is not available for R, Toree, etc.)

var validate = (function validate() {
this.validateVersion().then(function(valid) {
if (!valid) {
console.warn('Widget frontend version does not match the backend.');
}
}).catch(function(err) {
console.error('Could not cross validate the widget frontend and backend versions.', err);
});
}).bind(this);
validate();
}

this._shimWidgetsLibs();
};
Expand Down
67 changes: 67 additions & 0 deletions etc/notebooks/test/test_scala_hello_world.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false,
"urth": {
"dashboard": {
"layout": {
"col": 0,
"height": 4,
"row": 0,
"width": 4
}
}
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hello World!\n"
]
}
],
"source": [
"println(\"Hello World!\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"urth": {
"dashboard": {
"hidden": true
}
}
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Apache_Toree",
"language": "",
"name": "apache_toree"
},
"language_info": {
"name": "scala"
},
"urth": {
"dashboard": {
"cellMargin": 10,
"defaultCellHeight": 20,
"layout": "grid",
"maxColumns": 12
}
}
},
"nbformat": 4,
"nbformat_minor": 0
}
95 changes: 61 additions & 34 deletions routes/api.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ var urljoin = require('url-join');
var urlToDashboard = require('../app/url-to-dashboard');
var WsRewriter = require('../app/ws-rewriter');

var nbstore = require('../app/notebook-store');

var kgUrl = config.get('KERNEL_GATEWAY_URL');
var kgAuthToken = config.get('KG_AUTH_TOKEN');
var kgBaseUrl = config.get('KG_BASE_URL');
Expand All @@ -40,12 +42,12 @@ function initWsProxy(server) {
if (wsProxy) {
return;
}

var headers = null;
if(kgAuthToken) {
// include the kg auth token if we have one
headers = {};
headers.Authorization = 'token ' + kgAuthToken;
// include the kg auth token if we have one
headers = {};
headers.Authorization = 'token ' + kgAuthToken;
}

wsProxy = new WsRewriter({
Expand Down Expand Up @@ -130,7 +132,7 @@ var removeSession = function(sessionId) {
// configurations.
router.post('/kernels', bodyParser.json({ type: 'text/plain' }), function(req, res) {
var headers = {};
if(kgAuthToken) {
if (kgAuthToken) {
headers['Authorization'] = 'token ' + kgAuthToken;
}

Expand All @@ -139,42 +141,67 @@ router.post('/kernels', bodyParser.json({ type: 'text/plain' }), function(req, r
initWsProxy(req.connection.server);

// Forward the user object in the session to the kernel gateway.
if(config.get('KG_FORWARD_USER_AUTH') && req.user) {
if (config.get('KG_FORWARD_USER_AUTH') && req.user) {
req.body.env = {
KERNEL_USER_AUTH: JSON.stringify(req.user)
};
}

// Pass the (modified) request to the kernel gateway.
request({
url: urljoin(kgUrl, kgBaseUrl, '/api/kernels'),
method: 'POST',
headers: headers,
json: req.body
}, function(err, response, body) {
if(err) {
error('Error proxying kernel creation request:' + err.toString());
return res.status(500).end();
}
// Store the notebook path for use within the WS proxy.
var notebookPathHeader = req.headers['x-jupyter-notebook-path'];
var sessionId = req.headers['x-jupyter-session-id'];
if (!notebookPathHeader || !sessionId) {
error('Missing notebook path or session ID headers');
return res.status(500).end();
}
var matches = notebookPathHeader.match(/^\/(?:dashboards(-plain)?)?(.*)$/);
if (!matches) {
error('Invalid notebook path header');
return res.status(500).end();
}
// Get notebook path from request headers
var notebookPathHeader = req.headers['x-jupyter-notebook-path'];
var sessionId = req.headers['x-jupyter-session-id'];
if (!notebookPathHeader || !sessionId) {
error('Missing notebook path or session ID headers');
return res.status(500).end();
}

var matches = notebookPathHeader.match(/^\/(?:dashboards(-plain)?)?(.*)$/);
if (!matches) {
error('Invalid notebook path header');
return res.status(500).end();
} else {
var notebookPath = matches[2];
// Store notebook path for later use
sessions[sessionId] = matches[2];
sessions[sessionId] = notebookPath;

// Pass the kernel gateway response back to the client.
res.set(response.headers);
res.status(response.statusCode).json(body);
});
// Retrieve notebook from store to pull out kernel name
nbstore.get(notebookPath)
.then(function success(notebook){

if (notebook.metadata.kernelspec.name) {
var kernelName = notebook.metadata.kernelspec.name;
debug('Notebook kernel name found: ' + kernelName);
if (kernelName === 'apache_toree') {
kernelName = 'scala';
}
req.body.name = kernelName;
} else {
// Default to Python 3
debug('Notebook kernel name not found, defaulting to Python 3');
req.body.name = 'python3';
}
debug('Issuing request for kernel: ' + req.body);
// Pass the (modified) request to the kernel gateway.
request({
url: urljoin(kgUrl, kgBaseUrl, '/api/kernels'),
method: 'POST',
headers: headers,
json: req.body
}, function(err, response, body) {
if (err) {
error('Error proxying kernel creation request:' + err.toString());
return res.status(500).end();
}
// Pass the kernel gateway response back to the client.
res.set(response.headers);
res.status(response.statusCode).json(body);
});
})
.catch(function(err) {
error('Unknown notebook path: ' + notebookPath);
return res.status(500).end();
});
}
});

// Proxy all unhandled requests to the kernel gateway.
Expand Down
1 change: 1 addition & 0 deletions views/dashboard.handlebars
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
(function() {
var db = window.jupyter_dashboard = window.jupyter_dashboard || {};
var Config = db.Config = db.Config || {};
Config.kernelname = "{{notebook.metadata.kernelspec.name}}";
// values from notebook metadata
{{#each notebook.metadata.urth.dashboard}}
Config.{{@key}} = "{{this}}";
Expand Down