From fe82032ff446b9580ca4758955a29469b825934a Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:54:22 -0700 Subject: [PATCH 1/9] Update JupyterLab dependencies to work with newer lab --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3fad463..4e63182 100644 --- a/package.json +++ b/package.json @@ -29,17 +29,17 @@ "watch": "tsc -w" }, "dependencies": { - "@jupyterlab/application": "^0.15.0", - "@jupyterlab/apputils": "^0.15.2", - "@jupyterlab/coreutils": "^1.0.3", - "@jupyterlab/services": "^1.1.1", + "@jupyterlab/application": "^0.19.1", + "@jupyterlab/apputils": "^0.19.1", + "@jupyterlab/coreutils": "^2.2.1", + "@jupyterlab/services": "^3.2.1", "@phosphor/commands": "^1.4.0", "@phosphor/coreutils": "^1.3.0", "@phosphor/widgets": "^1.5.0" }, "devDependencies": { "rimraf": "^2.6.1", - "typescript": "~2.6.0" + "typescript": "^3.4.5" }, "jupyterlab": { "extension": true From fc9dd41a88e5b0e79ed59c504809f981db0cdc6b Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:54:39 -0700 Subject: [PATCH 2/9] Add JupyterLab as a python dependency --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f29c8b2..96da940 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,8 @@ 'Programming Language :: Python :: 3', ], install_requires=[ - 'notebook' + 'notebook', + 'jupyterlab' ], ) From c480e21ffe87a133b63edaab3095aca6079a3aa7 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:55:29 -0700 Subject: [PATCH 3/9] Use EventLog class for receiving & emitting events --- jupyterlab_telemetry/__init__.py | 46 +++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/jupyterlab_telemetry/__init__.py b/jupyterlab_telemetry/__init__.py index de63f8e..69c555a 100644 --- a/jupyterlab_telemetry/__init__.py +++ b/jupyterlab_telemetry/__init__.py @@ -1,32 +1,46 @@ """ JupyterLab LaTex : live Telemetry editing for JupyterLab """ import json +import os +from glob import glob -from tornado import gen, web +from tornado import web from notebook.utils import url_path_join from notebook.base.handlers import APIHandler, json_errors +from jupyterhub.events import EventLog from ._version import __version__ +here = os.path.dirname(__file__) -class TelemetryHandler(APIHandler): +class EventLoggingHandler(APIHandler): """ A handler that receives and stores telemetry data from the client. """ + @property + def eventlog(self) -> EventLog: + return self.settings['eventlog'] + @json_errors - @gen.coroutine @web.authenticated - def put(self, *args, **kwargs): - # Parse the data from the request body - raw = self.request.body.strip().decode(u'utf-8') + async def put(self, *args, **kwargs): try: - decoder = json.JSONDecoder() - session_log = decoder.decode(raw) + # Parse the data from the request body + raw_event = json.loads(self.request.body.strip().decode()) except Exception as e: raise web.HTTPError(400, str(e)) + + required_fields = {'schema', 'version', 'event'} + for rf in required_fields: + if rf not in raw_event: + raise web.HTTPError(400, f'{rf} is a required field') + + schema_name = raw_event['schema'] + version = raw_event['version'] + event = raw_event['event'] + self.eventlog.emit(schema_name, version, event) - self.log.info(session_log) self.set_status(204) self.finish() @@ -45,8 +59,16 @@ def load_jupyter_server_extension(nb_server_app): nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance. """ web_app = nb_server_app.web_app + + eventlog = EventLog(parent=nb_server_app) + web_app.settings['eventlog'] = eventlog + for schema_file in glob(os.path.join(here, 'event-schemas','*.json')): + with open(schema_file) as f: + eventlog.register_schema(json.load(f)) + # Prepend the base_url so that it works in a jupyterhub setting base_url = web_app.settings['base_url'] - endpoint = url_path_join(base_url, 'telemetry') - handlers = [(endpoint + '(.*)', TelemetryHandler)] - web_app.add_handlers('.*$', handlers) + endpoint = url_path_join(base_url, 'eventlog') + + handlers = [(endpoint + '(.*)', EventLoggingHandler)] + web_app.add_handlers('.*$', handlers) \ No newline at end of file From 3c6fbca59f35ff59cc3d7b742e7ede597a35ef14 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:55:49 -0700 Subject: [PATCH 4/9] Add schema for receiving command invocations --- .../event-schemas/command-invocations.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 jupyterlab_telemetry/event-schemas/command-invocations.json diff --git a/jupyterlab_telemetry/event-schemas/command-invocations.json b/jupyterlab_telemetry/event-schemas/command-invocations.json new file mode 100644 index 0000000..ed5095a --- /dev/null +++ b/jupyterlab_telemetry/event-schemas/command-invocations.json @@ -0,0 +1,23 @@ +{ + "$id": "lab.jupyter.org/command-invocations", + "version": 1, + "title": "JupyterLab command invocations", + "description": "Records each invocation of any command in JupyterLab", + "type": "object", + "properties": { + "session_id": { + "comment": "We should validate that this is a UUID, Is this PII?", + "type": "string", + "description": "Randomly generated session ID for this user session" + }, + "command_id": { + "type": "string", + "description": "ID of the command being executed" + }, + "command_args": { + "type": "object", + "description": "Arguments to the command being executed", + "pii": true + } + } +} \ No newline at end of file From 6e9b98e5488d139ec878374a3d1ef6508c9cc13a Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:56:08 -0700 Subject: [PATCH 5/9] Make labextension emit events This removes the consent GUI screen since I couldn't figure out how to make that work --- src/handler.ts | 49 ++++++++++------------- src/index.ts | 105 +++++++++++-------------------------------------- 2 files changed, 44 insertions(+), 110 deletions(-) diff --git a/src/handler.ts b/src/handler.ts index b6b6aaa..2bf512d 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -32,20 +32,26 @@ class TelemetryHandler { readonly serverSettings: ServerConnection.ISettings; /** - * Save telemetry data to the server. + * Emit an event to the server * - * @param id - The telemetry's ID. + * This may implement batching and other performance optimizations + * in the future. * - * @param telemetry - The telemetry being saved. + * @param event - The event being emitted * * @returns A promise that resolves when saving is complete or rejects with * a `ServerConnection.IError`. */ - save(telemetry: Telemetry.ISessionLog): Promise { + emit(event: Telemetry.ICommandInvocation): Promise { const { serverSettings } = this; - const url = URLExt.join(serverSettings.baseUrl, 'telemetry'); + const url = URLExt.join(serverSettings.baseUrl, 'eventlog'); + const full_event = { + 'schema': 'lab.jupyter.org/command-invocations', + 'version': 1, + 'event': event + } const init = { - body: JSON.stringify(telemetry), + body: JSON.stringify(full_event), method: 'PUT' }; const promise = ServerConnection.makeRequest(url, init, serverSettings); @@ -54,7 +60,6 @@ class TelemetryHandler { if (response.status !== 204) { throw new ServerConnection.ResponseError(response); } - return undefined; }); } @@ -85,39 +90,25 @@ namespace TelemetryHandler { export namespace Telemetry { /** - * The interface describing a telemetry resource. + * An interface describing an executed command. + * + * FIXME: Automatically generate these from the schema? */ export - interface ISessionLog { + interface ICommandInvocation { /** - * A unique identifier for the current session. + * UUID representing current session */ - id: string; - - /** - * A log of executed commands. - */ - commands: ICommandExecuted[]; - } + readonly session_id: string; - /** - * An interface describing an executed command. - */ - export - interface ICommandExecuted { /** * The id of the command. */ - readonly id: string; + readonly command_id: string; /** * The args of the command. */ - readonly args: ReadonlyJSONObject; - - /** - * The timestamp of the command. - */ - readonly date: string; + readonly command_args: ReadonlyJSONObject; } } diff --git a/src/index.ts b/src/index.ts index 4658ad7..8efa2ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,25 +4,16 @@ import { JupyterLab, JupyterLabPlugin } from '@jupyterlab/application'; - -import { - Dialog, showDialog -} from '@jupyterlab/apputils'; - -import { - uuid -} from '@jupyterlab/coreutils'; - import { - h -} from '@phosphor/virtualdom'; + UUID +} from '@phosphor/coreutils'; import { Widget } from '@phosphor/widgets'; import { - Telemetry, TelemetryHandler + TelemetryHandler } from './handler'; import '../style/index.css'; @@ -37,79 +28,31 @@ const extension: JupyterLabPlugin = { activate: (app: JupyterLab) => { const { commands } = app; const handler = new TelemetryHandler(); + // Make a uuid for this session, which will be its // key in the session data. - const id = uuid(); - // A log of executed commands. - const commandLog: Telemetry.ICommandExecuted[] = []; + const session_id = UUID.uuid4(); + console.log("WOOHOOO?") app.restored.then(() => { - // Create the disclaimer dialog - const headerLogo = h.div({className: 'jp-About-header-logo'}); - const title = h.span({className: 'jp-About-header'}, - headerLogo, - 'JupyterLab UX Survey'); - const message = 'Are you willing to participate in a ' + - 'JupyterLab user-experience survey? ' + - 'If you agree, we will log:'; - const message2 = 'We will NOT track:'; - const dos = h.ul({}, - h.li({}, 'The menu items you use'), - h.li({}, 'The command palette commands you run'), - h.li({}, 'The keyboard shortcuts you invoke'), - h.li({}, 'The filebrowser operations you use'), - ); - const donts = h.ul({}, - h.li({}, 'The contents of any notebooks or other files'), - h.li({}, 'The code you run'), - h.li({}, 'The commands you give in terminals'), - ); - const disclaimer = h.div({}, message, dos, message2, donts); - const body = h.div({ className: 'jp-About-body' }, - disclaimer - ); - - showDialog({ - title, - body, - buttons: [ - Dialog.cancelButton({ label: 'NO WAY!' }), - Dialog.okButton({ label: 'SURE!' }), - ] - }).then(result => { - if (result.button.accept) { - // Add a telemetry icon to the top bar. - // We do it after the app has been restored to place it - // at the right. - const widget = new Widget(); - widget.addClass('jp-telemetry-icon'); - widget.id = 'telemetry:icon'; - widget.node.title = 'Telemetry data is being collected'; - app.shell.addToTopArea(widget); - - // When a command is executed, store it in the log. - commands.commandExecuted.connect((registry, command) => { - const date = new Date(); - commandLog.push({ - id: command.id, - args: command.args, - date: date.toJSON(), - }); - }); - - const saveLog = () => { - if (commandLog.length === 0) { - return; - } - const outgoing = commandLog.splice(0); - handler.save({ id, commands: outgoing }).catch(() => { - // If the save fails, put the outgoing list back in the log. - commandLog.unshift(...outgoing); - }); - }; - // Save the log to the server every two minutes. - setInterval(saveLog, 120 * 1000); - } + // Add a telemetry icon to the top bar. + // We do it after the app has been restored to place it + // at the right. + const widget = new Widget(); + widget.addClass('jp-telemetry-icon'); + widget.id = 'telemetry:icon'; + widget.node.title = 'Telemetry data is being collected'; + app.shell.addToTopArea(widget); + + + // When a command is executed, emit it + commands.commandExecuted.connect((registry, command) => { + console.log("A COMMAND HAS BEEN EXECUTED YO"); + handler.emit({ + session_id: session_id, + command_id: command.id, + command_args: command.args + }) }); }); } From 60c38d3120ac3e743cb70b0de77fcbd7b76e9e6f Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:57:00 -0700 Subject: [PATCH 6/9] Enable storing telemetry about command invocations --- jupyter_notebook_config.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 jupyter_notebook_config.py diff --git a/jupyter_notebook_config.py b/jupyter_notebook_config.py new file mode 100644 index 0000000..970d11d --- /dev/null +++ b/jupyter_notebook_config.py @@ -0,0 +1,11 @@ +import logging + +c.EventLog.allowed_schemas = [ + 'lab.jupyter.org/command-invocations' +] + +def make_eventlog_sinks(eventlog): + # As an example, let's write these to local file + return [logging.FileHandler('events.log')] + +c.EventLog.handlers_maker = make_eventlog_sinks \ No newline at end of file From 50121d8244ab49afa37c6e00a904ac5785a29af7 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 15:57:16 -0700 Subject: [PATCH 7/9] Try getting this to work on mybinder --- postBuild | 6 ++++++ requirements.txt | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 postBuild create mode 100644 requirements.txt diff --git a/postBuild b/postBuild new file mode 100644 index 0000000..9d7d2f1 --- /dev/null +++ b/postBuild @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +jlpm install +jlpm run build +jupyter labextension link . \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2fda81e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# JupyterHub with PR https://github.com/jupyterhub/jupyterhub/pull/2542 +git+https://github.com/yuvipanda/jupyterhub.git@d22e5cfa25ee430fca32b64a3a527723aa0c8fe4 \ No newline at end of file From 60064c0e777537b6f297e02e74330592540a6747 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 16:07:05 -0700 Subject: [PATCH 8/9] Install current python package as editable This lets me play with event schemas without having to worry about packaging them in my package --- postBuild | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postBuild b/postBuild index 9d7d2f1..d10802f 100644 --- a/postBuild +++ b/postBuild @@ -1,6 +1,8 @@ #!/bin/bash set -euo pipefail +pip install --no-cache-dir --editable . + jlpm install jlpm run build jupyter labextension link . \ No newline at end of file From 25d9abc4d051e9e6f111c639f1140a0a4cbd45e2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 21 May 2019 16:12:33 -0700 Subject: [PATCH 9/9] Explicitly enable serverextension Not done when using an editable pip install by default --- postBuild | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postBuild b/postBuild index d10802f..31e93b0 100644 --- a/postBuild +++ b/postBuild @@ -2,7 +2,8 @@ set -euo pipefail pip install --no-cache-dir --editable . +jupyter serverextension enable --py jupyterlab_telemetry --sys-prefix jlpm install jlpm run build -jupyter labextension link . \ No newline at end of file +jupyter labextension link .