Add widget support

.gitignore

@@ -1,3 +1,6 @@
 *.elc
 .projectile
 flycheck_*
+node_modules/
+*lock*
+built/

js/emacs-jupyter.js

@@ -0,0 +1,334 @@
+var disposable = require('@phosphor/disposable');
+var coreutils = require('@jupyterlab/coreutils');
+var KernelFutureHandler = require('@jupyterlab/services/kernel/future').KernelFutureHandler;
+var CommHandler = require('@jupyterlab/services/kernel/comm').CommHandler;
+
+var EmacsJupyter = function(options) {
+    var _this = this;
+
+    this.username = options.username || '';
+    // This is the Jupyter session id
+    this.clientId = options.clientId;
+    this.isDisposed = false;
+    this.commPromises = new Map();
+    this.targetRegistry = {};
+    this.futures = new Map();
+    this.widgetManager = null;
+    this.widgetState = null;
+    this.commManager = null;
+    this.messagePromise = Promise.resolve();
+
+    window.addEventListener("unload", function(event) {
+        var XHR = window.skewerNativeXHR || XMLHttpRequest;
+        var xhr = new XHR();
+        xhr.open('POST', skewer.host + '/jupyter/widgets/state/' + _this.clientId, false);
+        xhr.setRequestHeader("Content-Type", "text/plain");
+        xhr.send(JSON.stringify(_this.widgetState));
+        return undefined;
+    });
+
+    // Kick off the message receiving process
+    var callback = function(msg) {
+        if(_this.isDisposed) {
+            return;
+        }
+
+        var p = _this.handlerPromise;
+        _this.handlerPromise = new Promise(function (resolve) {
+            if(msg.buffers && msg.buffers.length > 0) {
+                for(var i = 0; i < msg.buffers.length; i++) {
+                    var bin = atob(msg.buffers[i]);
+                    var len = bin.length;
+                    var buf = new Uint8Array(len);
+                    for(var j = 0; j < len; j++) {
+                        buf[j] = bin.charCodeAt(j);
+                    }
+                    msg.buffers[i] = buf.buffer;
+                }
+            }
+
+            resolve(Promise.all([p, _this.handleMessage(msg)]).then(function () {
+                skewer.getJSON(EmacsJupyter.baseUrl + "/recv?clientId=" + _this.clientId, callback);
+            }));
+        });
+    };
+    skewer.getJSON(EmacsJupyter.baseUrl + "/recv?clientId=" + _this.clientId, callback);
+};
+exports.EmacsJupyter = EmacsJupyter;
+
+EmacsJupyter.baseUrl = skewer.host + "/jupyter/widgets"
+
+EmacsJupyter.prototype.dispose = function () {
+    if (this.isDisposed) {
+        return;
+    }
+    this.isDisposed = true;
+    this.commPromises.forEach(function (promise, key) {
+        promise.then(function (comm) {
+            comm.dispose();
+        });
+    });
+};
+
+EmacsJupyter.prototype.registerCommTarget = function(targetName, callback) {
+    var _this = this;
+    this.targetRegistry[targetName] = callback;
+    return new disposable.DisposableDelegate(function () {
+        if (!_this.isDisposed) {
+            delete _this.targetRegistry[targetName];
+        }
+    });
+};
+
+EmacsJupyter.prototype.connectToComm = function (targetName, commId) {
+    var _this = this;
+    var id = commId || coreutils.uuid();
+    if (this.commPromises.has(id)) {
+        return this.commPromises.get(id);
+    }
+    var promise = Promise.resolve(void 0).then(function () {
+        return new CommHandler(targetName, id, _this, function () { _this._unregisterComm(id); });
+    });
+    this.commPromises.set(id, promise);
+    return promise;
+};
+
+EmacsJupyter.prototype.handleCommOpen = function (msg) {
+    var _this = this;
+    var content = msg.content;
+    if (this.isDisposed) {
+        return;
+    }
+    var promise = this.loadObject(content.target_name, content.target_module, this.targetRegistry).then(function (target) {
+        var comm = new CommHandler(content.target_name, content.comm_id, _this, function () { _this._unregisterComm(content.comm_id); });
+        var response;
+        try {
+            response = target(comm, msg);
+        }
+        catch (e) {
+            comm.close();
+            console.error('Exception opening new comm');
+            throw (e);
+        }
+        return Promise.resolve(response).then(function () {
+            if (_this.isDisposed) {
+                return;
+            }
+            return comm;
+        });
+    });
+    this.commPromises.set(content.comm_id, promise);
+    return undefined;
+};
+
+EmacsJupyter.prototype.handleCommClose = function (msg) {
+    var _this = this;
+    var content = msg.content;
+    var promise = this.commPromises.get(content.comm_id);
+    if (!promise) {
+        console.error('Comm not found for comm id ' + content.comm_id);
+        return;
+    }
+    promise.then(function (comm) {
+        if (!comm) {
+            return;
+        }
+        _this._unregisterComm(comm.commId);
+        try {
+            var onClose = comm.onClose;
+            if (onClose) {
+                onClose(msg);
+            }
+            comm.dispose();
+        }
+        catch (e) {
+            console.error('Exception closing comm: ', e, e.stack, msg);
+        }
+    });
+
+    return undefined;
+};
+
+EmacsJupyter.prototype.handleCommMsg = function (msg) {
+    var promise = this.commPromises.get(msg.content.comm_id);
+    if (!promise) {
+        // We do have a registered comm for this comm id, ignore.
+        return;
+    }
+    promise.then(function (comm) {
+        if (!comm) {
+            return;
+        }
+        try {
+            var onMsg = comm.onMsg;
+            if (onMsg) {
+                onMsg(msg);
+            }
+        }
+        catch (e) {
+            console.error('Exception handling comm msg: ', e, e.stack, msg);
+        }
+    });
+
+    return undefined;
+};
+
+EmacsJupyter.prototype.loadObject = function(name, moduleName, registry) {
+    return new Promise(function (resolve, reject) {
+        // Try loading the view module using require.js
+        if (moduleName) {
+            if (typeof window.require === 'undefined') {
+                throw new Error('requirejs not found');
+            }
+            window.require([moduleName], function (mod) {
+                if (mod[name] === void 0) {
+                    var msg = "Object '" + name + "' not found in module '" + moduleName + "'";
+                    reject(new Error(msg));
+                }
+                else {
+                    resolve(mod[name]);
+                }
+            }, reject);
+        }
+        else {
+            if (registry && registry[name]) {
+                resolve(registry[name]);
+            }
+            else {
+                reject(new Error("Object '" + name + "' not found in registry"));
+            }
+        }
+    });
+}
+
+EmacsJupyter.prototype._unregisterComm = function (commId) {
+    this.commPromises.delete(commId);
+};
+
+// It looks like widgets send messages through the callbacks of a
+// KernelFutureHandler so I will have to redirect all received messages that
+// originated from a request generated by skewer.postJSON back to the
+// JavaScript environment. Emacs then acts as an intermediary, capturing kernel
+// messages and re-packaging them to send to the Javascript environment.
+//
+// It looks like whenever the kernel receives a message it accesse the correct
+// future object using this.futures.get and calls handleMsg function of the
+// future.
+//
+// The flow of message with respect to Comm objects is that Comm object send
+// shell messages, then widgets register callbacks on the future.
+EmacsJupyter.prototype.sendShellMessage = function(msg, expectReply, disposeOnDone) {
+    var _this = this;
+    if (expectReply === void 0) { expectReply = false; }
+    if (disposeOnDone === void 0) { disposeOnDone = true; }
+
+    var future = new KernelFutureHandler(function () {
+        var msgId = msg.header.msg_id;
+        _this.futures.delete(msgId);
+    }, msg, expectReply, disposeOnDone, this);
+
+    var promise = new Promise(function (resolve) {
+        skewer.postJSON(EmacsJupyter.baseUrl + "/send/" + _this.clientId, msg, function (reply) {
+            // This is needed since Emacs will generate a new ID on every
+            // messsage sent, this is the least intrusive way of handling it.
+            var id = reply.id;
+            msg.header.msg_id = id;
+            _this.futures.set(id, future);
+            resolve(void 0);
+        });
+    });
+    if(_this.pending === void 0) {
+        _this.pending = promise;
+    } else {
+        _this.pending = Promise.all([_this.pending, promise]);
+    }
+    return future;
+};
+
+EmacsJupyter.prototype.requestCommInfo = function(targetName) {
+    var msg = {
+        channel: 'shell',
+        msg_type: 'comm_info_request',
+        // A message ID will be added by Emacs anyway
+        header: {msg_id: ''},
+        content: {target_name: targetName}
+    };
+    var future = this.sendShellMessage(msg, true);
+    return new Promise(function (resolve) {
+        future.onReply = resolve;
+    });
+};
+
+EmacsJupyter.prototype.handleMessage = function(msg) {
+    var _this = this;
+    var parentHeader = msg.parent_header;
+    var future = parentHeader && this.futures && this.futures.get(parentHeader.msg_id);
+    if (future) {
+        return new Promise(function (resolve, reject) {
+            try {
+                future.handleMsg(msg);
+                resolve(msg);
+            } catch(err) {
+                reject(err);
+            }
+        });
+    } else {
+        return new Promise(function (resolve, reject) {
+            switch(msg.msg_type) {
+                // Special messages not really a Jupyter message
+            case 'display_model':
+                _this.widgetManager.get_model(msg.content.model_id).then(function (model) {
+                    _this.widgetManager.display_model(undefined, model);
+                });
+                break;
+            case 'clear_display':
+                var widget = _this.widgetManager.area;
+                while(widget.firstChild) {
+                    widget.removeChild(widget.firstChild);
+                }
+                break;
+                // Regular Jupyter messages
+            case 'comm_open':
+                _this.handleCommOpen(msg);
+                // Periodically get the state of the widgetManager, this gets
+                // sent to the browser when its unloaded.
+                // _this.widgetManager.get_state({}).then(function (state) {
+                //     _this.widgetState = state;
+                // });
+                break;
+            case 'comm_close':
+                _this.handleCommClose(msg);
+                break;
+            case 'comm_msg':
+                _this.handleCommMsg(msg);
+                break;
+            case 'status':
+                // Comes from the comm info messages
+                break;
+            default:
+                reject(new Error('Unhandled message', msg));
+            };
+            resolve(msg);
+        });
+    }
+}
+
+// The CommHandler object handles comm interaction to/from the kernel. It takes
+// a target_name, usually jupyter.widget, and a comm id and takes care of
+// sending comm messages to the kernel and calls the callback methods when a
+// comm msg is received from the kernel.
+
+// A Comm object is just a wrapper around a CommHandler that updates its
+// callbacks
+
+// The targetRegistry is a dictionary mapping target names to target functions
+// to call whenever a new Comm is requested to be open by the kernel. The
+// target function gets called with the message data cand a comm handler.
+
+// A CommManager takes care of registering new comm targets and creating new
+// comms and holding a list of all the live comms.
+
+// It looks like I just ned to implement the IKernel interface and pass the
+// object that implements it to CommManager, this way I can create new comms
+// with CommManager.new_comm when handling comm_open messages. In the IKernel
+// interface, I'll just redirect all the message sending functions to Emacs.

js/index.js

@@ -0,0 +1,11 @@
+window.CommManager = require('@jupyter-widgets/base').shims.services.CommManager;
+window.WidgetManager = require('./manager').WidgetManager;
+window.EmacsJupyter = require('./emacs-jupyter').EmacsJupyter;
+
+require('@jupyter-widgets/controls/css/widgets.built.css');
+
+document.addEventListener("DOMContentLoaded", function(event) {
+    var widget = document.createElement("div");
+    widget.setAttribute("id", "widget");
+    document.body.appendChild(widget);
+});

js/manager.js

@@ -0,0 +1,82 @@
+var base = require('@jupyter-widgets/base');
+var output = require('@jupyter-widgets/output');
+var controls = require('@jupyter-widgets/controls');
+var PhosphorWidget = require('@phosphor/widgets').Widget;
+
+var defineWidgetModules = function () {
+    if(window.define) {
+        window.define('@jupyter-widgets/output', [], function () { return output; });
+        window.define('@jupyter-widgets/base', [], function () { return base; });
+        window.define('@jupyter-widgets/controls', [], function () { return controls; });
+    } else {
+        setTimeout(defineWidgetModules, 50);
+    }
+};
+
+// requirejs loading is async so it may not be available on this event
+window.addEventListener("DOMContentLoaded", function () {
+    defineWidgetModules();
+});
+
+var WidgetManager = exports.WidgetManager = function(kernel, area) {
+    base.ManagerBase.call(this);
+    this.kernel = kernel;
+    this.area = area;
+};
+WidgetManager.prototype = Object.create(base.ManagerBase.prototype);
+
+WidgetManager.prototype.loadClass = function(className, moduleName, moduleVersion) {
+    return new Promise(function(resolve, reject) {
+        if (moduleName === '@jupyter-widgets/controls') {
+            resolve(controls);
+        } else if (moduleName === '@jupyter-widgets/base') {
+            resolve(base);
+        } else if (moduleName === '@jupyter-widgets/output')
+            resolve(output);
+        else {
+            var fallback = function(err) {
+                let failedId = err.requireModules && err.requireModules[0];
+                if (failedId) {
+                    console.log('Falling back to unpkg.com for ' + moduleName + '@' + moduleVersion);
+                    window.require(['https://unpkg.com/' + moduleName + '@' + moduleVersion + '/dist/index.js'], resolve, reject);
+                } else {
+                    throw err;
+                }
+            };
+            window.require([moduleName + '.js'], resolve, fallback);
+        }
+    }).then(function(module) {
+        if (module[className]) {
+            return module[className];
+        } else {
+            return Promise.reject('Class ' + className + ' not found in module ' + moduleName + '@' + moduleVersion);
+        }
+    });
+}
+
+WidgetManager.prototype.display_view = function(msg, view, options) {
+    var _this = this;
+    return Promise.resolve(view).then(function(view) {
+        PhosphorWidget.attach(view.pWidget, _this.area);
+        view.on('remove', function() {
+            skewer.log('View removed', view);
+        });
+        view.trigger('displayed');
+        return view;
+    });
+};
+
+WidgetManager.prototype._get_comm_info = function() {
+    return this.kernel.requestCommInfo(this.comm_target_name).then(function(reply) {
+        return reply.content.comms;
+    });
+};
+
+WidgetManager.prototype._create_comm = function(targetName, commId, data, metadata) {
+    // Construct a comm that already exists
+    var comm = this.kernel.connectToComm(targetName, commId);
+    if(data || metadata) {
+        comm.open(data, metadata);
+    }
+    return Promise.resolve(new base.shims.services.Comm(comm));
+}

js/package.json

@@ -0,0 +1,32 @@
+{
+  "private": true,
+  "name": "emacs-jupyter",
+  "version": "0.1",
+  "description": "Integrate emacs-jupyter with widgets in a browser.",
+  "main": "index.js",
+  "scripts": {
+    "clean": "rimraf built",
+    "build": "webpack",
+    "test": "npm run test:default",
+    "test:default": "echo \"No test specified\""
+  },
+  "author": "Nathaniel Nicandro",
+  "license": "BSD-3-Clause",
+  "dependencies": {
+    "@jupyter-widgets/base": "^1.1.10",
+    "@jupyter-widgets/controls": "^1.2.1",
+    "@jupyter-widgets/output": "^1.0.10",
+    "codemirror": "^5.9.0",
+    "font-awesome": "^4.7.0",
+    "npm": "^6.0.1"
+  },
+  "devDependencies": {
+    "css-loader": "^0.28.4",
+    "file-loader": "^0.11.2",
+    "json-loader": "^0.5.4",
+    "raw-loader": "^0.5.1",
+    "style-loader": "^0.18.1",
+    "url-loader": "^0.5.9",
+    "webpack": "^3.5.5"
+  }
+}

js/webpack.config.js

@@ -0,0 +1,29 @@
+var path = require('path');
+
+module.exports = {
+    entry: "./index.js",
+    output: {
+        filename: 'index.built.js',
+        path: path.resolve(__dirname, 'built'),
+        publicPath: 'built/'
+    },
+    resolve: {
+        alias: {
+            '@jupyterlab/services/kernel/future': path.resolve(__dirname, 'node_modules/@jupyterlab/services/lib/kernel/future'),
+            '@jupyterlab/services/kernel/comm': path.resolve(__dirname, 'node_modules/@jupyterlab/services/lib/kernel/comm')
+        }
+    },
+    module: {
+        rules: [
+            { test: /\.css$/, loader: "style-loader!css-loader" },
+            // jquery-ui loads some images
+            { test: /\.(jpg|png|gif)$/, use: 'file-loader' },
+            // required to load font-awesome
+            { test: /\.woff2(\?v=\d+\.\d+\.\d+)?$/, use: 'url-loader?mimetype=application/font-woff' },
+            { test: /\.woff(\?v=\d+\.\d+\.\d+)?$/, use: 'url-loader?mimetype=application/font-woff' },
+            { test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, use: 'url-loader?mimetype=application/octet-stream' },
+            { test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, use: 'file-loader' },
+            { test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, use: 'url-loader?mimetype=image/svg+xml' }
+        ]
+    },
+}

jupyter-base.el

@@ -31,6 +31,7 @@
 
 (require 'cl-lib)
 (require 'eieio)
+(require 'eieio-base)
 (require 'json)
 (require 'zmq)
 (require 'hmac-def)
@@ -83,6 +84,9 @@ directory is where kernel connection files are written to."
   :group 'jupyter
   :type 'string)
 
+(defconst jupyter-root (file-name-directory load-file-name)
+  "Root directory containing emacs-jupyter.")
+
 (defconst jupyter-protocol-version "5.3"
   "The jupyter protocol version that is implemented.")
 
@@ -108,6 +112,9 @@ directory is where kernel connection files are written to."
         :is-complete-reply "is_complete_reply"
         :comm-info-request "comm_info_request"
         :comm-info-reply "comm_info_reply"
+        :comm-open "comm_open"
+        :comm-msg "comm_msg"
+        :comm-close "comm_close"
         :kernel-info-request "kernel_info_request"
         :kernel-info-reply "kernel_info_reply"
         :shutdown-request "shutdown_request"
@@ -192,6 +199,10 @@ following fields:
                     `jupyter-kernel-client' has received the
                     status: idle message for the request.
 
+- LAST-MESSAGE :: The raw message property list of the last
+                  message received by the kernel in response to
+                  this request.
+
 - LAST-MESSAGE-TIME :: The last time a message was received for
                        the request.
 
@@ -209,6 +220,7 @@ following fields:
   (-id)
   (time (current-time))
   (idle-received-p nil)
+  (last-message nil)
   (last-message-time nil)
   (inhibited-handlers nil)
   (callbacks))
@@ -329,6 +341,12 @@ the ROUTING-ID of the socket. Return the created socket."
         (json-false nil))
     (json-read-file file)))
 
+(defun jupyter-read-plist-from-string (string)
+  (let ((json-object-type 'plist)
+        ;; TODO: See the comment in `jupyter--decode'
+        (json-array-type 'list))
+    (json-read-from-string string)))
+
 (provide 'jupyter-base)
 
 ;;; jupyter-base.el ends here

jupyter-client.el

@@ -34,12 +34,17 @@
 (require 'jupyter-base)
 (require 'jupyter-channels)
 (require 'jupyter-messages)
+(require 'skewer-mode)
 
 (declare-function hash-table-values "subr-x" (hash-table))
 
 (defvar jupyter--debug nil
   "Set to non-nil to emit sent and received messages to *Messages*.")
 
+(defvar jupyter--clients nil
+  "A list of all live clients.
+Clients are removed from this list when their `destructor' is called.")
+
 (defvar jupyter-default-timeout 1
   "The default timeout in seconds for `jupyter-wait-until'.")
 
@@ -70,8 +75,9 @@ If set to t, disable all client handlers.")
   ((type
     :initform :stdin)))
 
-(defclass jupyter-kernel-client ()
+(defclass jupyter-kernel-client (eieio-instance-tracker)
-  ((requests
+  ((tracking-symbol :initform 'jupyter--clients)
+   (requests
     :type hash-table
     :initform (make-hash-table :test 'equal)
     :documentation "A hash table with message ID's as keys. This
@@ -130,14 +136,24 @@ buffer.")
 
 (cl-defmethod initialize-instance ((client jupyter-kernel-client) &rest _slots)
   (cl-call-next-method)
+  (push client jupyter--clients)
   (oset client -buffer (generate-new-buffer " *jupyter-kernel-client*")))
 
 (cl-defmethod destructor ((client jupyter-kernel-client) &rest _params)
   "Close CLIENT's channels and cleanup internal resources."
   (jupyter-stop-channels client)
+  (delete-instance client)
   (when (buffer-live-p (oref client -buffer))
     (kill-buffer (oref client -buffer))))
 
+(defun jupyter-find-client-for-session (session-id)
+  "Return the `jupyter-kernel-client' for SESSION-ID."
+  (or (cl-find-if
+       (lambda (client)
+         (string= (jupyter-session-id (oref client session)) session-id))
+       jupyter--clients)
+      (error "No client found for session (%s)" session-id)))
+
 (defun jupyter-initialize-connection (client info-or-session)
   "Initialize CLIENT with connection INFO-OR-SESSION.
 If INFO-OR-SESSION is the name of a file, assume the file to be a
@@ -795,11 +811,14 @@ are taken:
         (if (not req)
             (when (jupyter-get client 'jupyter-include-other-output)
               (jupyter--run-handler-maybe client channel req msg))
+          (setf (jupyter-request-last-message req) msg)
+          ;; TODO: This is redundant if we keep the last message since we can
+          ;; just access the date field of a message.
+          (setf (jupyter-request-last-message-time req) (current-time))
           (unwind-protect
               (jupyter--run-callbacks req msg)
             (unwind-protect
                 (jupyter--run-handler-maybe client channel req msg)
-              (setf (jupyter-request-last-message-time req) (current-time))
               (when (jupyter-message-status-idle-p msg)
                 (setf (jupyter-request-idle-received-p req) t))
               (jupyter--drop-idle-requests client))))))))
@@ -1038,6 +1057,38 @@ the user. Otherwise `read-from-minibuffer' is used."
               :target-name target-name)))
     (jupyter-send client channel :comm-info-request msg)))
 
+(cl-defgeneric jupyter-send-comm-open ((client jupyter-kernel-client)
+                                       &key id
+                                       target-name
+                                       data)
+  (declare (indent 1))
+  (let ((channel (oref client shell-channel))
+        (msg (jupyter-message-comm-open
+              :id id
+              :target-name target-name
+              :data data)))
+    (jupyter-send client channel :comm-open msg)))
+
+(cl-defgeneric jupyter-send-comm-msg ((client jupyter-kernel-client)
+                                      &key id
+                                      data)
+  (declare (indent 1))
+  (let ((channel (oref client shell-channel))
+        (msg (jupyter-message-comm-msg
+              :id id
+              :data data)))
+    (jupyter-send client channel :comm-msg msg)))
+
+(cl-defgeneric jupyter-send-comm-close ((client jupyter-kernel-client)
+                                        &key id
+                                        data)
+  (declare (indent 1))
+  (let ((channel (oref client shell-channel))
+        (msg (jupyter-message-comm-close
+              :id id
+              :data data)))
+    (jupyter-send client channel :comm-close msg)))
+
 (cl-defgeneric jupyter-handle-comm-info-reply ((_client jupyter-kernel-client)
                                                _req
                                                _comms)
@@ -1089,6 +1140,9 @@ If RESTART is non-nil, request a restart instead of a complete shutdown."
   (jupyter-dispatch-message-cases client req msg
     ((shutdown-reply restart)
      (stream name text)
+     (comm-open comm_id target_name target_module data)
+     (comm-msg comm_id data)
+     (comm-close comm_id data)
      (execute-input code execution_count)
      (execute-result execution_count data metadata)
      (error ename evalue traceback)
@@ -1097,6 +1151,29 @@ If RESTART is non-nil, request a restart instead of a complete shutdown."
      (display-data data metadata transient)
      (update-display-data data metadata transient))))
 
+(cl-defgeneric jupyter-handle-comm-open ((_client jupyter-kernel-client)
+                                         _req
+                                         _id
+                                         _target-name
+                                         _target-module
+                                         _data)
+  (declare (indent 1))
+  nil)
+
+(cl-defgeneric jupyter-handle-comm-msg ((_client jupyter-kernel-client)
+                                        _req
+                                        _id
+                                        _data)
+  (declare (indent 1))
+  nil)
+
+(cl-defgeneric jupyter-handle-comm-close ((_client jupyter-kernel-client)
+                                          _req
+                                          _id
+                                          _data)
+  (declare (indent 1))
+  nil)
+
 (cl-defgeneric jupyter-handle-stream ((_client jupyter-kernel-client)
                                       _req
                                       _name

jupyter-messages.el

@@ -290,6 +290,22 @@
     (cl-check-type target-name string)
     (list :target_name target-name)))
 
+(cl-defun jupyter-message-comm-open (&key id target-name data)
+  (cl-check-type id string)
+  (cl-check-type target-name string)
+  (cl-check-type data json-plist)
+  (list :comm_id id :target_name target-name :data data))
+
+(cl-defun jupyter-message-comm-msg (&key id data)
+  (cl-check-type id string)
+  (cl-check-type data json-plist)
+  (list :comm_id id :data data))
+
+(cl-defun jupyter-message-comm-close (&key id data)
+  (cl-check-type id string)
+  (cl-check-type data json-plist)
+  (list :comm_id id :data data))
+
 (cl-defun jupyter-message-shutdown-request (&key restart)
   (list :restart (if restart t jupyter--false)))
 

jupyter-repl.el

@@ -1432,10 +1432,9 @@ is used for completion."
       (let ((max-len (apply #'max (mapcar #'length matches))))
         (cl-mapc
          (lambda (match meta)
-           (let ((prefix (make-string (1+ (- max-len (length match))) ? )))
+           (let* ((prefix (make-string (1+ (- max-len (length match))) ? ))
-             (put-text-property
+                  (annot (concat prefix (plist-get meta :type))))
-              0 1 'annot (concat prefix (plist-get meta :type))
+             (put-text-property 0 1 'annot annot match)))
-              match)))
          matches types)))
     matches))
 

jupyter-widget-client.el

@@ -0,0 +1,264 @@
+;;; jupyter-widget-client.el --- Widget support -*- lexical-binding: t -*-
+
+;; Copyright (C) 2018 Nathaniel Nicandro
+
+;; Author: Nathaniel Nicandro <nathanielnicandro@gmail.com>
+;; Created: 21 May 2018
+;; Version: 0.0.1
+;; Keywords:
+
+;; This program is free software; you can redistribute it and/or
+;; modify it under the terms of the GNU General Public License as
+;; published by the Free Software Foundation; either version 2, or (at
+;; your option) any later version.
+
+;; This program is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs; see the file COPYING.  If not, write to the
+;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+;; Boston, MA 02111-1307, USA.
+
+;;; Commentary:
+
+;;
+
+;;; Code:
+
+(require 'skewer-mode)
+(require 'jupyter-client)
+
+(defvar jupyter-widgets-initialized nil)
+
+(defclass jupyter-widget-client (jupyter-kernel-client)
+  ((widget-proc
+    :initform nil
+    :documentation "The process currently requesting a message.")
+   (widget-state
+    :type string
+    :initform "null"
+    :documentation "The JSON encode string representing the
+widget state. When a browser displaying the widgets of the client
+is closed, the state of the widgets is sent back to Emacs so that
+the state can be recovred when a new browser is opened.")
+   (widget-messages
+    :type list
+    :initform nil
+    :documentation "A list of messages to send to the widget process."))
+  :abstract t)
+
+(defun jupyter-widgets-sanitize-comm-msg (msg)
+  "Ensure that a comm MSG's fields are not ambiguous before encoding.
+For example, for fields that are supposed to be arrays, ensure
+that they will be encoded as such. In addition, add fields required
+by the JupyterLab widget manager."
+  (prog1 msg
+    (let ((buffers (plist-member msg :buffers)))
+      (if (null buffers) (plist-put msg :buffers [])
+        (when (eq (cadr buffers) nil)
+          (setcar (cdr buffers) [])))
+      (unless (equal (cadr buffers) [])
+        (setq buffers (cadr buffers))
+        (while (car buffers)
+          (setcar buffers
+                  (base64-encode-string
+                   (encode-coding-string (car buffers) 'utf-8-auto t) t))
+          (setq buffers (cdr buffers))))
+      ;; Needed by WidgetManager
+      (unless (plist-get msg :metadata)
+        (plist-put msg :metadata '(:version "2.0"))))))
+
+(cl-defmethod jupyter-widgets-process-one-message ((client jupyter-widget-client))
+  "Process one message in CLIENT's `widget-messages' slot.
+A message is only processed if CLIENT's `widget-proc' slot is a
+live process requesting a message. Schedule to process another
+message if one is available."
+  (when (process-live-p (oref client widget-proc))
+    (let ((msg (pop (oref client widget-messages))))
+      (when msg
+        (with-temp-buffer
+          (let ((proc (prog1 (oref client widget-proc)
+                        (oset client widget-proc nil))))
+            ;; FIXME: This is a bottleneck. Maybe only partially decode a
+            ;; message instead of the full message.
+            (insert (jupyter--encode msg))
+            (httpd-send-header
+             proc "text/json; charset=UTF-8" 200
+             :Access-Control-Allow-Origin "*")))
+        (run-at-time 0.001 nil #'jupyter-widgets-process-one-message client)))))
+
+(cl-defmethod jupyter-widgets-queue-message ((client jupyter-widget-client) msg)
+  "Queue a MSG to be processed by CLIENT's `widget-proc' at a later time."
+  (let* ((msg (jupyter-widgets-sanitize-comm-msg msg))
+         (msg-type (jupyter-message-type msg)))
+    ;; FIXME: The :date field is an emacs time object, i.e. a 4 element list,
+    ;; convert to an actual time.
+    ;; We don't have a channel field, but KernelFutureHandler.handleMsg
+    ;; of jupyterlab requires it
+    (plist-put msg :channel
+               (cond
+                ((memq msg-type '(:status :comm-msg :comm-close :comm-open))
+                 :iopub)
+                ((memq msg-type '(:comm-info-reply))
+                 :shell)))
+    (oset client widget-messages
+          (nconc (oref client widget-messages) (list msg)))))
+
+(cl-defmethod jupyter-widgets-display-model ((client jupyter-widget-client) model-id)
+  "Display the model with MODEL-ID for the kernel CLIENT is connected to."
+  ;; NOTE: This is a message specific for this purpose and not really a
+  ;; Jupyter message
+  ;; (jupyter-widgets-clear-display client)
+  (jupyter-widgets-queue-message
+   client (list :msg_type "display_model"
+                :content (list :model_id model-id)))
+  (jupyter-widgets-process-one-message client))
+
+(cl-defmethod jupyter-widgets-clear-display ((client jupyter-widget-client))
+  "Clear the models being displayed for CLIENT."
+  ;; NOTE: This is a message specific for this purpose and not really a
+  ;; Jupyter message
+  (jupyter-widgets-queue-message
+   client (list :msg_type "clear_display")))
+
+(defservlet jupyter "text/javascript; charset=UTF-8" ()
+  (insert-file-contents
+   (expand-file-name "js/built/index.built.js" jupyter-root)))
+
+(defun httpd/jupyter/widgets/built (proc path query &rest _args)
+  (let* ((split-path (split-string (substring path 1) "/"))
+         (file (car (last split-path)))
+         (mime (pcase (file-name-extension file)
+                 ((or "woff" "woff2")
+                  "application/font-woff")
+                 ("ttf"
+                  "application/octet-stream")
+                 ("svg"
+                  "image/svg+xml")
+                 ("eot"
+                  "application/vnd.ms-fontobject"))))
+    (unless mime
+      (error "Unsupported file type"))
+    (setq file (expand-file-name (concat "js/built/" file) jupyter-root))
+    ;; TODO: Fix this, when loading the files through httpd, font awesome
+    ;; doesnt work
+    (when (file-exists-p file)
+      (error "File nonexistent (%s)" (file-name-nondirectory file)))
+    (with-temp-buffer
+      (insert-file-contents file)
+      (httpd-send-header proc mime 200
+                         :Access-Control-Allow-Origin "*"))))
+
+;; TODO: Since the path when we instantiate widgets is jupyter/widgets, all
+;; files that are trying to be loaded locally in the javascript will be
+;; referenced to this path. If we encounter a javascript file requesting to be
+;; loaded we can automatically search the jupyter --paths for notebook
+;; extension modules matching it.
+(defservlet* jupyter/widgets/:client-id "text/html; charset=UTF-8" ()
+  (let ((client (jupyter-find-client-for-session client-id)))
+    (if (not client)
+        (error "No client found for ID (%s)" client-id)
+      (insert-file-contents (expand-file-name "widget.html" jupyter-root))
+      (dolist (key-replace `(("{username}" . ,user-login-name)
+                             ("{client-id}" . ,client-id)
+                             ("{widget-state}" . ,(oref client widget-state))))
+        (goto-char (point-min))
+        (while (search-forward (car key-replace) nil t)
+          (replace-match (cdr key-replace)))))))
+
+(defservlet* jupyter/widgets/send/:client-id "text/json; charset=UTF-8" ()
+  "Send a message to the kernel whose session ID is CLIENT-ID.
+The message is sent using the `jupyter-widget-client' whose
+session ID is CLIENT-ID. Queue all messages generated by the sent
+message to be sent back to `jupyter-widget-client's WIDGET-PROC."
+  ;; TODO: How to avoid this step if it isn't needed?
+  (let* ((msg
+          (condition-case nil
+              (jupyter-read-plist-from-string
+               (cadr (assoc "Content" httpd-request)))
+            (error (message (cadr (assoc "Content" httpd-request))))))
+         (client (jupyter-find-client-for-session client-id))
+         (channel (pcase (plist-get msg :channel)
+                    ("shell" (oref client shell-channel))
+                    ("iopub" (oref client iopub-channel))
+                    ("stdin" (oref client stdin-channel))))
+         (msg-type (jupyter-message-type-as-keyword
+                    (jupyter-message-type msg)))
+         (jupyter-inhibit-handlers
+          ;; Only let the browser handle thee messages
+          (when (memq msg-type '(:comm-info-request))
+            '(:status :comm-info-reply))))
+    ;; TODO: Remove the need for this special case
+    (when (memq msg-type '(:comm-open :comm-close :comm-msg))
+      (jupyter-widgets-sanitize-comm-msg msg))
+    (let ((req (jupyter-send
+                client channel msg-type (jupyter-message-content msg))))
+      (jupyter-add-callback req
+        t (lambda (msg)
+            (jupyter-widgets-queue-message client msg)
+            (jupyter-widgets-process-one-message client)))
+      ;; FIXME: Bottleneck here since `jupyter-request-id' is a synchronizing
+      ;; function.
+      (insert (concat "{\"id\":\"" (jupyter-request-id req) "\"}")))))
+
+(defun httpd/jupyter/widgets/recv (proc _path query &rest _args)
+  "Queue the widget client PROC asking to receive a message.
+QUERY should contain the key \"clientId\", which is the session
+ID of the `jupyter-widget-client' which will provide messages to
+PROC. If a message is already available, provide it to PROC."
+  (let* ((client-id (or (cadr (assoc "clientId" query))
+                        (error "No clientId specified")))
+         (client (jupyter-find-client-for-session client-id)))
+    (oset client widget-proc proc)
+    (jupyter-widgets-process-one-message client)))
+
+(defservlet* jupyter/widgets/state/:client-id "text/plain" ()
+  "Save the state of a `jupyter-widget-client' whose session ID is CLIENT-ID."
+  (let ((client (jupyter-find-client-for-session client-id)))
+    (oset client widget-state (cadr (assoc "Content" httpd-request)))
+    ;; The state is sent when the browser closes
+    (jupyter-set client 'jupyter-widgets-initialized nil)
+    (oset client widget-messages nil)))
+
+(cl-defmethod jupyter-handle-comm-open ((client jupyter-widget-client)
+                                        req
+                                        _id
+                                        _target-name
+                                        _target-module
+                                        _data)
+  (let ((msg (jupyter-request-last-message req)))
+    (when (member (jupyter-message-get msg :target_name)
+                  '("jupyter.widget"))
+      (unless (jupyter-get client 'jupyter-widgets-initialized)
+        (unless (get-process "httpd")
+          (httpd-start))
+        (jupyter-set client 'jupyter-widgets-initialized t)
+        (browse-url
+         (format "http://127.0.0.1:%d/jupyter/widgets/%s"
+                 httpd-port (jupyter-session-id (oref client session)))))
+      (jupyter-widgets-queue-message client msg)
+      (jupyter-widgets-process-one-message client)))
+  (cl-call-next-method))
+
+(cl-defmethod jupyter-handle-comm-close ((client jupyter-widget-client)
+                                         req
+                                         _id
+                                         _data)
+  (jupyter-widgets-queue-message client (jupyter-request-last-message req))
+  (jupyter-widgets-process-one-message client)
+  (cl-call-next-method))
+
+(cl-defmethod jupyter-handle-comm-msg ((client jupyter-widget-client)
+                                       req
+                                       _id
+                                       _data)
+  (jupyter-widgets-queue-message client (jupyter-request-last-message req))
+  (jupyter-widgets-process-one-message client)
+  (cl-call-next-method))
+
+(provide 'jupyter-widget-client)
+
+;;; jupyter-widget-client.el ends here

widget.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Jupyter Client</title>
+<script type="application/javascript" src="/skewer"></script>
+<script type="application/javascript" src="/jupyter"></script>
+<style type="text/css">
+* {
+     font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+</style>
+<script type="application/javascript">
+var kernel = new EmacsJupyter({username: "{username}", clientId: "{client-id}"}, "");
+document.addEventListener("DOMContentLoaded", function(event) {
+    var commManager = new CommManager(kernel);
+    var widgetManager = new WidgetManager(kernel, document.getElementById("widget"));
+    commManager.register_target(widgetManager.comm_target_name, function(comm, msg) {
+        widgetManager.handle_comm_open(comm, msg);
+    });
+    kernel.widgetManager = widgetManager;
+    kernel.commManager = commManager;
+    var state = {widget-state};
+    if(state) { kernel.widgetManager.set_state(state); }
+});
+</script>
+</head>
+<body>
+</body>
+</html>