Add jupyter-kernel.el and related

The new files added in this commit will eventually replace the manager
and kernel classes and favor struct types to represent kernels instead
of classes.  A kernel manager was a concept ripped from the
jupyter/jupyter_client reference implementation.

In Emacs the concept makes the client implementation more complicated
and is replaced by functions that manage the lifetime of a kernel:
`jupyter-launch`, `jupyter-shutdown`, and `jupyter-interrupt`.

* jupyter-kernel-manager.el
* jupyter-kernel-process-manager.el
* jupyter-server.el
* test/jupyter-server-test.el
* test/test-helper.el: Make `jupyter-kernel`, `jupyter-server-kernel`,
  and `jupyter-kernel-process` private classes.  The new files below
  use the public names, but as structs now.

* jupyter-kernel.el
* jupyter-kernel-process.el
* jupyter-server-kernel.el: New files.
This commit is contained in:
Nathaniel Nicandro 2020-04-23 13:07:08 -05:00
parent cc60e6abe2
commit 95f3e150b4
8 changed files with 806 additions and 24 deletions

View file

@ -78,7 +78,7 @@
;;; `jupyter-kernel'
(defclass jupyter-kernel (jupyter-kernel-lifetime)
(defclass jupyter--kernel (jupyter-kernel-lifetime)
((spec
:type jupyter-kernelspec
:initarg :spec
@ -102,10 +102,10 @@ implementation of `jupyter-kill-kernel'.
A convenience method, `jupyter-kernel-name', is provided to
access the name of the kernelspec.")
(cl-defmethod jupyter-kill-kernel ((_kernel jupyter-kernel))
(cl-defmethod jupyter-kill-kernel ((_kernel jupyter--kernel))
(ignore))
(cl-defmethod jupyter-kernel-name ((kernel jupyter-kernel))
(cl-defmethod jupyter-kernel-name ((kernel jupyter--kernel))
"Return the name of KERNEL."
(jupyter-kernelspec-name (oref kernel spec)))
@ -117,7 +117,7 @@ access the name of the kernelspec.")
jupyter-instance-tracker)
((tracking-symbol :initform 'jupyter--kernel-managers)
(kernel
:type jupyter-kernel
:type jupyter--kernel
:initarg :kernel
:documentation "The kernel that is being managed."))
:abstract t)

View file

@ -71,7 +71,7 @@ error. The error is raised before :timeout-form is evaluated."
;;; `jupyter-kernel-process'
(defclass jupyter-kernel-process (jupyter-kernel)
(defclass jupyter--kernel-process (jupyter--kernel)
((process
:type process
:documentation "The kernel process."))
@ -82,11 +82,11 @@ If the kernel was started on a remote host, ensure that local
tunnels are created when setting the session slot after the
kernel starts.")
(cl-defmethod jupyter-kernel-alive-p ((kernel jupyter-kernel-process))
(cl-defmethod jupyter-kernel-alive-p ((kernel jupyter--kernel-process))
(and (slot-boundp kernel 'process)
(process-live-p (oref kernel process))))
(cl-defmethod jupyter-start-kernel ((kernel jupyter-kernel-process) &rest args)
(cl-defmethod jupyter-start-kernel ((kernel jupyter--kernel-process) &rest args)
"Start a KERNEL process with ARGS."
(let ((name (jupyter-kernel-name kernel)))
(when jupyter--debug
@ -113,18 +113,18 @@ fatal signal."
(jupyter-weak-ref-resolve ref)))
(jupyter-kernel-died kernel)))))
(cl-defmethod jupyter-start-kernel :after ((kernel jupyter-kernel-process) &rest _args)
(cl-defmethod jupyter-start-kernel :after ((kernel jupyter--kernel-process) &rest _args)
(setf (process-sentinel (oref kernel process))
(jupyter--kernel-died-process-sentinel kernel)))
(cl-defmethod jupyter-kill-kernel ((kernel jupyter-kernel-process))
(cl-defmethod jupyter-kill-kernel ((kernel jupyter--kernel-process))
(with-slots (process) kernel
(delete-process process)
(when (buffer-live-p (process-buffer process))
(kill-buffer (process-buffer process))))
(cl-call-next-method))
(defclass jupyter-command-kernel (jupyter-kernel-process)
(defclass jupyter-command-kernel (jupyter--kernel-process)
()
:documentation "A Jupyter kernel process using the \"jupyter kernel\" command.")
@ -161,7 +161,7 @@ argument of the process."
:conn-info conn-info
:key (plist-get conn-info :key))))))
(defclass jupyter-spec-kernel (jupyter-kernel-process)
(defclass jupyter-spec-kernel (jupyter--kernel-process)
()
:documentation "A Jupyter kernel launched from a kernelspec.")
@ -313,7 +313,7 @@ subprocess."
(jupyter-kernel-name kernel)))
(jupyter-recv control-channel 'dont-wait))))
(_
(if (object-of-class-p kernel 'jupyter-kernel-process)
(if (object-of-class-p kernel 'jupyter--kernel-process)
(interrupt-process (oref kernel process) t)
(warn "Can't interrupt kernel"))))))))

335
jupyter-kernel-process.el Normal file
View file

@ -0,0 +1,335 @@
;;; jupyter-kernel-process.el --- Jupyter kernels as Emacs processes -*- lexical-binding: t -*-
;; Copyright (C) 2020 Nathaniel Nicandro
;; Author: Nathaniel Nicandro <nathanielnicandro@gmail.com>
;; Created: 25 Apr 2020
;; This program is free software; you can redistribute it and/or
;; modify it under the terms of the GNU General Public License as
;; published by the Free Software Foundation; either version 3, or (at
;; your option) any later version.
;; This program is distributed in the hope that it will be useful, but
;; WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;; General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING. If not, write to the
;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
;; Boston, MA 02111-1307, USA.
;;; Commentary:
;; Jupyter kernels as Emacs processes.
;;; Code:
(require 'jupyter-kernel)
(defgroup jupyter-kernel-process nil
"Jupyter kernels as Emacs processes"
:group 'jupyter)
(declare-function jupyter-channel-ioloop-set-session "jupyter-channel-ioloop")
;;; Kernel definition
(cl-defstruct (jupyter-kernel-process
(:include jupyter-kernel))
(process nil
:type (or null process)
:documentation "A kernel process."))
(cl-defmethod jupyter-alive-p ((kernel jupyter-kernel-process))
(pcase-let (((cl-struct jupyter-kernel-process process) kernel))
(and (process-live-p process)
(cl-call-next-method))))
(defun jupyter-kernel-process (&rest args)
"Return a representation of a kernel process.
Return a `jupyter-kernel-process' with ARGS being the slot values
used for initialization."
(let ((kernel (apply #'make-jupyter-kernel-process args))
(session (pcase (plist-get args :session)
((and (pred stringp) (pred file-exists-p) `,conn-file)
(let ((conn-info (jupyter-read-connection conn-file)))
(jupyter-session
:conn-info conn-info
:key (plist-get conn-info :key))))
((and (pred jupyter-session-p) `,session)
session)
(_
(prog1 (jupyter-session-with-random-ports)
;; This is here for stability when running
;; the tests. Sometimes the kernel ports are
;; not set up fast enough due to the hack
;; done in
;; `jupyter-session-with-random-ports'. The
;; effect seems to be messages that are sent
;; but never received by the kernel.
(sit-for 0.2)))))
(spec (pcase (plist-get args :spec)
((and (pred stringp) `,name)
(or (jupyter-guess-kernelspec name)
(error "No kernelspec matching name (%s)" name)))
(`,spec spec))))
(setf (jupyter-kernel-session kernel) session)
(setf (jupyter-kernel-spec kernel) spec)
kernel))
;;; Client connection
(cl-defstruct jupyter--proxy-channel endpoint alive-p)
(defun jupyter--make-channel-group (session)
(let ((endpoints (jupyter-session-endpoints session)))
(append
(list 'channel-group t
:hb (make-instance
'jupyter-hb-channel
:session session
:endpoint (plist-get endpoints :hb)))
(cl-loop
for channel in '(:control :shell :iopub :stdin)
collect channel
collect (make-jupyter--proxy-channel
:endpoint (plist-get endpoints channel)
:alive-p nil)))))
(defun jupyter--channel-alive-p (ioloop chgroup channel)
(if (eq channel :hb)
(let ((hb (plist-get chgroup channel)))
(and hb (jupyter-channel-alive-p hb)))
(and ioloop (jupyter-ioloop-alive-p ioloop)
(jupyter--proxy-channel-alive-p
(plist-get chgroup channel)))))
(defun jupyter--start-channel (ioloop chgroup channel)
(unless (jupyter--channel-alive-p ioloop chgroup channel)
(if (eq channel :hb) (jupyter-start-channel (plist-get chgroup channel))
(let ((endpoint (jupyter--proxy-channel-endpoint
(plist-get chgroup channel))))
(jupyter-send ioloop 'start-channel channel endpoint)
;; Verify that the channel starts
(jupyter-with-timeout
(nil jupyter-default-timeout
(error "Channel not started in ioloop subprocess (%s)" channel))
(jupyter--channel-alive-p ioloop chgroup channel))))))
(defun jupyter--stop-channel (ioloop chgroup channel)
(when (jupyter--channel-alive-p ioloop chgroup channel)
(if (eq channel :hb) (jupyter-stop-channel (plist-get chgroup channel))
(jupyter-send ioloop 'stop-channel channel)
;; Verify that the channel stops
(jupyter-with-timeout
(nil jupyter-default-timeout
(error "Channel not stopped in ioloop subprocess (%s)" channel))
(not (jupyter--channel-alive-p ioloop chgroup channel))))))
(cl-defmethod jupyter-connection ((kernel jupyter-kernel-process) (handler function))
(require 'jupyter-zmq-channel-ioloop)
(let* ((session (jupyter-kernel-session kernel))
(channels '(:hb :shell :iopub :stdin))
(chgroup (jupyter--make-channel-group session))
(ioloop (make-instance 'jupyter-zmq-channel-ioloop)))
(jupyter-channel-ioloop-set-session ioloop session)
;; session and ioloop are in the context of the connection and are
;; thus not accessible outside of it, therefore no other parts of
;; Emacs-Jupyter have to consider them.
(make-jupyter-connection
:hb (plist-get chgroup :hb)
:id (lambda ()
(format "session=%s" (truncate-string-to-width
(jupyter-session-id session)
9 nil nil "")))
:start (lambda (&optional channel)
(unless (jupyter-ioloop-alive-p ioloop)
(jupyter-ioloop-start
ioloop (lambda (event)
(pcase (car event)
;; These channel events are from
;; `jupyter-channel-ioloop'
('start-channel
(setf (jupyter--proxy-channel-alive-p
(plist-get chgroup (cadr event)))
t))
('stop-channel
(setf (jupyter--proxy-channel-alive-p
(plist-get chgroup (cadr event)))
nil))
(_
(funcall handler event))))))
(if channel (jupyter--start-channel ioloop chgroup channel)
(cl-loop
for channel in channels
do (jupyter--start-channel ioloop chgroup channel))))
:stop (lambda (&optional channel)
(if channel (jupyter--stop-channel ioloop chgroup channel)
(cl-loop
for channel in channels
do (jupyter--stop-channel ioloop chgroup channel))
(jupyter-ioloop-stop ioloop))
(jupyter-ioloop-stop ioloop))
:send (lambda (&rest event)
(apply #'jupyter-send ioloop event))
:alive-p (lambda (&optional channel)
(if channel (jupyter--channel-alive-p ioloop chgroup channel)
(cl-loop
for channel in channels
thereis (jupyter--channel-alive-p ioloop chgroup channel)))))))
;;; Kernel management
(defvar jupyter--kernel-processes '()
"The list of kernel processes launched.
Elements look like (PROCESS CONN-FILE) where PROCESS is a kernel
process and CONN-FILE the associated connection file.
This list is periodically cleaned up when a new process is
launched and when Emacs exits.")
(defun jupyter--gc-kernel-processes ()
(setq jupyter--kernel-processes
(cl-loop for (p conn-file) in jupyter--kernel-processes
if (process-live-p p) collect (list p conn-file)
else do (delete-process p)
(when (file-exists-p conn-file)
(delete-file conn-file))
and when (buffer-live-p (process-buffer p))
do (kill-buffer (process-buffer p)))))
(defun jupyter-delete-connection-files ()
"Delete all connection files created by Emacs."
;; Ensure Emacs can be killed on error
(ignore-errors
(cl-loop for (_ conn-file) in jupyter--kernel-processes
do (when (file-exists-p conn-file)
(delete-file conn-file)))))
(add-hook 'kill-emacs-hook #'jupyter-delete-connection-files)
(defun jupyter--start-kernel-process (name kernelspec conn-file)
(let* ((process-name (format "jupyter-kernel-%s" name))
(buffer-name (format " *jupyter-kernel[%s]*" name))
(process-environment
(append (jupyter-process-environment kernelspec)
process-environment))
(args (jupyter-kernel-argv kernelspec conn-file))
(atime (nth 4 (file-attributes conn-file)))
(process (apply #'start-file-process process-name
(generate-new-buffer buffer-name)
(car args) (cdr args))))
(set-process-query-on-exit-flag process jupyter--debug)
;; Wait until the connection file has been read before returning.
;; This is to give the kernel a chance to setup before sending it
;; messages.
;;
;; TODO: Replace with a check of the heartbeat channel.
(jupyter-with-timeout
((format "Starting %s kernel process..." name)
jupyter-long-timeout
(unless (process-live-p process)
(error "Kernel process exited:\n%s"
(with-current-buffer (process-buffer process)
(ansi-color-apply (buffer-string))))))
;; Windows systems may not have good time resolution when retrieving
;; the last access time of a file so we don't bother with checking that
;; the kernel has read the connection file and leave it to the
;; downstream initialization to ensure that we can communicate with a
;; kernel.
(or (memq system-type '(ms-dos windows-nt cygwin))
(let ((attribs (file-attributes conn-file)))
;; `file-attributes' can potentially return nil, in this case
;; just assume it has read the connection file so that we can
;; know for sure it is not connected if it fails to respond to
;; any messages we send it.
(or (null attribs)
(not (equal atime (nth 4 attribs)))))))
(jupyter--gc-kernel-processes)
(push (list process conn-file) jupyter--kernel-processes)
process))
(defun jupyter--kernel-died-process-sentinel (kernel)
"Return a sentinel function calling KERNEL's `jupyter-kernel-died' method.
The method will be called when the process exits or receives a
fatal signal."
(let ((ref (jupyter-weak-ref kernel)))
(lambda (process _)
(when-let (kernel (and (memq (process-status process) '(exit signal))
(jupyter-weak-ref-resolve ref)))
(jupyter-kernel-died kernel)))))
(cl-defmethod jupyter-launch :before ((kernel jupyter-kernel-process))
"Ensure KERNEL has a non-nil SESSION slot.
A `jupyter-session' with random port numbers for the channels and
a newly generated message signing key will be set as the value of
KERNEL's SESSION slot if it is nil."
(pcase-let (((cl-struct jupyter-kernel-process session) kernel))
(unless session
(setf (jupyter-kernel-session kernel) (jupyter-session-with-random-ports))
;; This is here for stability when running the tests. Sometimes
;; the kernel ports are not set up fast enough due to the hack
;; done in `jupyter-session-with-random-ports'. The effect
;; seems to be messages that are sent but never received by the
;; kernel.
(sit-for 0.2))))
(cl-defmethod jupyter-launch ((kernel jupyter-kernel-process))
"Start KERNEL's process.
Do nothing if KERNEL's process is already live.
The process arguments are constructed from KERNEL's SPEC. The
connection file passed as argument to the process is first
written to file, its contents are generated from KERNEL's SESSION
slot.
See also https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs"
(pcase-let (((cl-struct jupyter-kernel-process process spec session) kernel))
(unless (process-live-p process)
(setq process
(setf (jupyter-kernel-process-process kernel)
(jupyter--start-kernel-process
(jupyter-kernel-name kernel) spec
(jupyter-write-connection-file session))))
(setf (process-sentinel process)
;; TODO: Have the sentinel function do something like
;; notify clients. It should also handle auto-restarting
;; if that is wanted.
(jupyter--kernel-died-process-sentinel kernel))
(setf (jupyter-kernel-process-process kernel) process)))
(cl-call-next-method))
;; TODO: Add restart argument
(cl-defmethod jupyter-shutdown ((kernel jupyter-kernel-process))
"Shutdown KERNEL by killing its process unconditionally."
(pcase-let (((cl-struct jupyter-kernel-process process) kernel))
(when (process-live-p process)
;; The `process-sentinel' is ignored when shutting down because
;; killing the process when explicitly shutting it down is not
;; an unexpected exit.
(setf (process-sentinel process) #'ignore)
(kill-process process))
(cl-call-next-method)))
(cl-defmethod jupyter-interrupt ((kernel jupyter-kernel-process))
"Interrupt KERNEL's process.
The process can be interrupted when the interrupt mode of
KERNEL's SPEC is \"signal\" or not specified, otherwise the
KERNEL is interrupted by sending an :interrupt-request on
KERNEL's control channel.
See also https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs"
(pcase-let* (((cl-struct jupyter-kernel-process process spec) kernel)
((cl-struct jupyter-kernelspec plist) spec)
(imode (plist-get plist :interrupt_mode)))
(if (or (null imode) (string= imode "signal"))
(interrupt-process process t)
(cl-call-next-method))))
(provide 'jupyter-kernel-process)
;;; jupyter-kernel-process.el ends here

148
jupyter-kernel.el Normal file
View file

@ -0,0 +1,148 @@
;;; jupyter-kernel.el --- Kernels -*- lexical-binding: t -*-
;; Copyright (C) 2020 Nathaniel Nicandro
;; Author: Nathaniel Nicandro <nathanielnicandro@gmail.com>
;; Created: 21 Apr 2020
;; This program is free software; you can redistribute it and/or
;; modify it under the terms of the GNU General Public License as
;; published by the Free Software Foundation; either version 3, or (at
;; your option) any later version.
;; This program is distributed in the hope that it will be useful, but
;; WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;; General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING. If not, write to the
;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
;; Boston, MA 02111-1307, USA.
;;; Commentary:
;; Working with Jupyter kernels. This file contains the functions
;; used to control the lifetime of a kernel and how clients can
;; connect to launched kernels.
;;; Code:
(require 'jupyter-base)
(require 'jupyter-kernelspec)
(defgroup jupyter-kernel nil
"Kernels"
:group 'jupyter)
;;; Kernel definition
(cl-defstruct jupyter-kernel
"A Jupyter kernel."
(spec (make-jupyter-kernelspec)
:type jupyter-kernelspec
:documentation "The kernelspec of this kernel.")
;; TODO: Remove require cycle so that I can have
;; `jupyter-connection' be the type and `make-jupyter-connection'
;; the default value.
(connection nil
:documentation "Kernel communication.")
;; FIXME: Remove this slot, used by `jupyter-widget-client'.
(session nil :type jupyter-session)
(clients nil
:type (list-of jupyter-kernel-client)
:documentation "List of clients able to receive messages."))
(cl-defmethod jupyter-alive-p ((kernel jupyter-kernel))
"Return non-nil if KERNEL has been launched."
(and (jupyter-kernel-session kernel) t))
(defun jupyter-kernel (&rest args)
"Return a kernel constructed from ARGS.
ARGS are keyword arguments."
(if (plist-get args :conn-info)
(make-jupyter-kernel
:session (let ((conn-info
(if (stringp (plist-get args :conn-info))
(jupyter-read-connection
(plist-get args :conn-file))
(plist-get args :conn-info))))
(jupyter-session
:conn-info conn-info
:key (plist-get conn-info :key))))
(error "Implement")))
;;; Kernel management
(cl-defgeneric jupyter-launch ((kernel jupyter-kernel))
"Launch KERNEL."
(cl-assert (jupyter-alive-p kernel)))
(cl-defmethod jupyter-launch :before ((kernel jupyter-kernel))
"Notify that the kernel launched."
(message "Launching %s kernel..." (jupyter-kernel-name kernel)))
(cl-defmethod jupyter-launch :after ((kernel jupyter-kernel))
"Notify that the kernel launched."
(message "Launching %s kernel...done" (jupyter-kernel-name kernel)))
(cl-defgeneric jupyter-shutdown ((kernel jupyter-kernel))
"Shutdown KERNEL.
Once a kernel has been shutdown it has no more connected clients
and the process it represents no longer exists.
The default implementation of this method disconnects all
connected clients of KERNEL and sets KERNEL's session slot to
nil."
(cl-loop
for client in (jupyter-kernel-clients kernel)
do (jupyter-disconnect client kernel))
(setf (jupyter-kernel-session kernel) nil))
(cl-defmethod jupyter-shutdown :before ((kernel jupyter-kernel))
"Notify that the kernel launched."
(message "%s kernel shutdown..." (jupyter-kernel-name kernel)))
(cl-defmethod jupyter-shutdown :after ((kernel jupyter-kernel))
"Notify that the kernel launched."
(message "%s kernel shutdown...done" (jupyter-kernel-name kernel)))
(defun jupyter-restart (kernel)
"Shutdown then re-launch KERNEL."
(jupyter-shutdown kernel)
(jupyter-launch kernel))
(cl-defgeneric jupyter-interrupt ((kernel jupyter-kernel))
"Interrupt KERNEL.
The default implementation of this method sends an interrupt
request on KERNEL's control channel if its kernelspec has an
:interrupt_mode of \"message\"."
(pcase-let* (((cl-struct jupyter-kernel spec session) kernel)
((cl-struct jupyter-kernelspec plist) spec))
(when (string= (plist-get plist :interrupt_mode) "message")
(let ((channel
(make-instance
'jupyter-zmq-channel
:type :control
:session session
:endpoint (cl-destructuring-bind (&key transport ip control_port
&allow-other-keys)
(jupyter-session-conn-info session)
(format "%s://%s:%d" transport ip control_port)))))
;; TODO: `with-live-jupyter-channel'
(jupyter-start-channel channel)
(jupyter-send channel :interrupt-request
(jupyter-message-interrupt-request))
(jupyter-with-timeout
((format "Interrupting %s kernel"
(jupyter-kernel-name kernel))
jupyter-default-timeout
(message "No interrupt reply from kernel (%s)"
(jupyter-kernel-name kernel)))
(jupyter-recv channel 'dont-wait))
(jupyter-stop-channel channel)))))
(provide 'jupyter-kernel)
;;; jupyter-kernel.el ends here

299
jupyter-server-kernel.el Normal file
View file

@ -0,0 +1,299 @@
;;; jupyter-server-kernel.el --- Working with kernels behind a Jupyter server -*- lexical-binding: t -*-
;; Copyright (C) 2020 Nathaniel Nicandro
;; Author: Nathaniel Nicandro <nathanielnicandro@gmail.com>
;; Created: 23 Apr 2020
;; This program is free software; you can redistribute it and/or
;; modify it under the terms of the GNU General Public License as
;; published by the Free Software Foundation; either version 3, or (at
;; your option) any later version.
;; This program is distributed in the hope that it will be useful, but
;; WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;; General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs; see the file COPYING. If not, write to the
;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
;; Boston, MA 02111-1307, USA.
;;; Commentary:
;; Holds the definitions of `jupyter-server', what communicates to the
;; Jupyter server using the REST API, and `jupyter-kernel-server' a
;; representation of a kernel on a server.
;;; Code:
(require 'jupyter-kernel)
(require 'jupyter-rest-api)
(require 'jupyter-server-ioloop)
(defgroup jupyter-server-kernel nil
"Kernel behind a Jupyter server"
:group 'jupyter)
;;; `jupyter-server'
(defvar-local jupyter-current-server nil
"The `jupyter-server' associated with the current buffer.
Used in, e.g. a `jupyter-server-kernel-list-mode' buffer.")
(put 'jupyter-current-server 'permanent-local t)
(defvar jupyter--servers nil)
;; TODO: We should really rename `jupyter-server' to something like
;; `jupyter-server-client' since it isn't a representation of a server, but a
;; communication channel with one.
(defclass jupyter-server (jupyter-rest-client
eieio-instance-tracker)
((tracking-symbol :initform 'jupyter--servers)
(ioloop :type jupyter-ioloop)
(handlers :type list :initform nil)
(kernelspecs
:type json-plist
:initform nil
:documentation "Kernelspecs for the kernels available behind this gateway.
Access should be done through `jupyter-available-kernelspecs'.")))
(defun jupyter-servers ()
"Return a list of all `jupyter-server's."
jupyter--servers)
(defun jupyter-gc-servers ()
"Forget `jupyter-servers' that are no longer accessible at their hosts."
(dolist (server (jupyter-servers))
(unless (jupyter-api-server-exists-p server)
(when (jupyter-ioloop-alive-p (oref server ioloop))
(jupyter-ioloop-stop (oref server ioloop)))
(jupyter-api-delete-cookies (oref server url))
(delete-instance server))))
(defun jupyter-server--refresh-comm (server)
"Stop and then start SERVER communication.
Reconnect the previously connected kernels when starting."
(when (jupyter-ioloop-alive-p (oref server ioloop))
(let ((connected (cl-remove-if-not
(apply-partially #'jupyter-server-kernel-connected-p server)
(mapcar (lambda (kernel) (plist-get kernel :id))
(jupyter-api-get-kernel server)))))
(jupyter-ioloop-stop (oref server ioloop))
(jupyter-server--start-comm server)
(while connected
(jupyter-server--connect-channels server (pop connected))))))
(cl-defmethod jupyter-api-request :around ((server jupyter-server) _method &rest _plist)
(condition-case nil
(cl-call-next-method)
(jupyter-api-unauthenticated
(if (memq jupyter-api-authentication-method '(ask token password))
(oset server auth jupyter-api-authentication-method)
(error "Unauthenticated request, can't attempt re-authentication \
with default `jupyter-api-authentication-method'"))
(prog1 (cl-call-next-method)
(jupyter-server--refresh-comm server)))))
(cl-defmethod jupyter-server-kernel-connected-p ((comm jupyter-server) id)
"Return non-nil if COMM has a WebSocket connection to a kernel with ID."
(and (jupyter-ioloop-alive-p (oref comm ioloop))
(member id (process-get (oref (oref comm ioloop) process) :kernel-ids))))
(cl-defmethod jupyter-server-kernelspecs ((server jupyter-server) &optional refresh)
"Return the kernelspecs on SERVER.
By default the available kernelspecs are cached. To force an
update of the cached kernelspecs, give a non-nil value to
REFRESH.
The kernelspecs are returned in the same form as returned by
`jupyter-available-kernelspecs'."
(when (or refresh (null (oref server kernelspecs)))
(let ((specs (jupyter-api-get-kernelspec server)))
(unless specs
(error "Can't retrieve kernelspecs from server @ %s" (oref server url)))
(oset server kernelspecs specs)
(plist-put (oref server kernelspecs) :kernelspecs
(cl-loop
with specs = (plist-get specs :kernelspecs)
for (_ spec) on specs by #'cddr
for name = (plist-get spec :name)
collect (make-jupyter-kernelspec
:name name
:plist (plist-get spec :spec))))))
(plist-get (oref server kernelspecs) :kernelspecs))
(cl-defmethod jupyter-server-has-kernelspec-p ((server jupyter-server) name)
"Return non-nil if SERVER can launch kernels with kernelspec NAME."
(jupyter-guess-kernelspec name (jupyter-server-kernelspecs server)))
;;; Kernel definition
(cl-defstruct (jupyter-server-kernel
(:include jupyter-kernel))
(server jupyter-current-server
:documentation "The kernel server.")
(id nil
:type (or null string)
:documentation "The kernel ID."))
(cl-defmethod jupyter-alive-p ((kernel jupyter-server-kernel))
(pcase-let (((cl-struct jupyter-server-kernel server id) kernel))
(and id server
;; TODO: Cache this call
(condition-case err
(jupyter-api-get-kernel server id)
(file-error nil) ; Non-existent server
(jupyter-api-http-error
(unless (= (nth 1 err) 404) ; Not Found
(signal (car err) (cdr err)))))
(cl-call-next-method))))
(defun jupyter-server-kernel (&rest args)
"Return a representation of a kernel on a Jupyter server.
ARGS is a property list used to initialize the returned
`jupyter-server-kernel'. The following keys of ARGS are handled
specially:
- If :spec is present and it is the name of a kernelspec, then
the SPEC of the returned kernel will be the one associated
with that name on the server."
(cl-assert (jupyter-server-p (plist-get args :server)))
(when (stringp (plist-get args :spec))
(let ((server (plist-get args :server))
(name (plist-get args :spec)))
(plist-put args :spec
(or (jupyter-guess-kernelspec
name (jupyter-server-kernelspecs server))
(error "No kernelspec matching %s @ %s" name
(oref server url))))))
(apply #'make-jupyter-server-kernel args))
;;;; Kernel management
(cl-defmethod jupyter-launch ((kernel jupyter-server-kernel))
"Launch KERNEL based on its kernelspec.
When KERNEL does not have an ID yet, launch KERNEL on SERVER
using its SPEC."
(pcase-let
(((cl-struct jupyter-server-kernel server id spec session) kernel))
(unless session
(and id (setq id (or (jupyter-server-kernel-id-from-name server id) id)))
(if id
;; When KERNEL already has an ID before it has a session,
;; assume we are connecting to an already launched kernel. In
;; this case, make sure the KERNEL's SPEC is the same as the
;; one being connected to.
;;
;; Note, this also has the side effect of raising an error
;; when the ID does not match one on the server.
(unless spec
(let ((model (jupyter-api-get-kernel server id)))
(setf (jupyter-kernel-spec kernel)
(jupyter-guess-kernelspec
(plist-get model :name)
(jupyter-server-kernelspecs server)))))
(let ((plist (jupyter-api-start-kernel
server (jupyter-kernelspec-name spec))))
(setf (jupyter-server-kernel-id kernel) (plist-get plist :id))))
;; TODO: Replace with the real session object
(setf (jupyter-kernel-session kernel) (jupyter-session))))
(cl-call-next-method))
(cl-defmethod jupyter-shutdown ((kernel jupyter-server-kernel))
(pcase-let
(((cl-struct jupyter-server-kernel server id session) kernel))
(cl-call-next-method)
(when session
(jupyter-api-shutdown-kernel server id))))
(cl-defmethod jupyter-interrupt ((kernel jupyter-server-kernel))
(pcase-let (((cl-struct jupyter-server-kernel server id) kernel))
(jupyter-api-interrupt-kernel server id)))
(cl-defstruct jupyter-server--event-handler id fn)
(defun jupyter-server--start-comm (server)
(unless (and (slot-boundp server 'ioloop)
(jupyter-ioloop-alive-p (oref server ioloop)))
;; Write the cookies to file so that they can be read
;; by the subprocess.
(url-cookie-write-file)
(let ((ioloop (jupyter-server-ioloop
:url (oref server url)
:ws-url (oref server ws-url)
:ws-headers (jupyter-api-auth-headers server))))
(oset server ioloop ioloop)
(jupyter-ioloop-start
ioloop
(lambda (event)
(let ((event-type (car event))
(event-kid (cadr event)))
(pcase event-type
('connect-channels
(cl-callf append (process-get (oref ioloop process) :kernel-ids)
(list event-kid)))
('disconnect-channels
(cl-callf2 remove event-kid
(process-get (oref ioloop process) :kernel-ids)))
(_
(setq event (cons event-type (cddr event)))
(cl-loop
for handler in (oref server handlers)
when (string= event-kid
(jupyter-server--event-handler-id handler))
do (funcall
(jupyter-server--event-handler-fn handler)
event))))))))))
(defun jupyter-server--connect-channels (server id)
(jupyter-send (oref server ioloop) 'connect-channels id)
(unless (jupyter-server-kernel-connected-p server id)
(jupyter-with-timeout
(nil jupyter-default-timeout
(error "Timeout when connecting websocket to kernel id %s" id))
(jupyter-server-kernel-connected-p server id))))
(defun jupyter-server--disconnect-channels (server id)
;; from the comm-remove-handler of a server
(jupyter-send (oref server ioloop) 'disconnect-channels id)
(unless (jupyter-ioloop-wait-until (oref server ioloop)
'disconnect-channels #'identity)
(error "Timeout when disconnecting websocket for kernel id %s" id)))
(cl-defmethod jupyter-connection ((kernel jupyter-server-kernel) (handler function))
(pcase-let* (((cl-struct jupyter-server-kernel server id) kernel)
(-handler (make-jupyter-server--event-handler
:id id :fn handler)))
(jupyter-server--start-comm server)
(make-jupyter-connection
:id (lambda ()
(or (jupyter-server-kernel-name server id)
(format "kid=%s" (truncate-string-to-width id 9 nil nil ""))))
:start (lambda (&optional channel)
(if channel (error "Can't start individual channels")
(jupyter-server--connect-channels server id)
(cl-callf2 cl-adjoin -handler (oref server handlers))))
:stop (lambda (&optional channel)
(if channel (error "Can't stop individual channels")
(jupyter-server--disconnect-channels server id)
(cl-callf2 delq -handler (oref server handlers))))
:send (lambda (&rest event)
(apply #'jupyter-send (oref server ioloop)
(car event) id (cdr event)))
:alive-p (lambda (&optional _channel)
(and (jupyter-server-kernel-connected-p server id)
(memq -handler (oref server handlers)))))))
(provide 'jupyter-server-kernel)
;;; jupyter-server-kernel.el ends here

View file

@ -127,7 +127,7 @@ Access should be done through `jupyter-available-kernelspecs'.")))
;;; `jupyter-server-kernel'
;; TODO: Add the server as a slot
(defclass jupyter-server-kernel (jupyter-kernel)
(defclass jupyter--server-kernel (jupyter--kernel)
((server
:type jupyter-server
:initarg :server
@ -137,7 +137,7 @@ Access should be done through `jupyter-available-kernelspecs'.")))
:initarg :id
:documentation "The kernel ID.")))
(cl-defmethod jupyter-kernel-alive-p ((kernel jupyter-server-kernel))
(cl-defmethod jupyter-kernel-alive-p ((kernel jupyter--server-kernel))
(and (slot-boundp kernel 'id)
(slot-boundp kernel 'server)
;; TODO: Cache this call
@ -148,18 +148,18 @@ Access should be done through `jupyter-available-kernelspecs'.")))
(unless (= (nth 1 err) 404) ; Not Found
(signal (car err) (cdr err)))))))
(cl-defmethod jupyter-start-kernel ((kernel jupyter-server-kernel) &rest _ignore)
(cl-defmethod jupyter-start-kernel ((kernel jupyter--server-kernel) &rest _ignore)
(with-slots (server spec) kernel
(cl-destructuring-bind (&key id &allow-other-keys)
(jupyter-api-start-kernel server (jupyter-kernelspec-name spec))
(oset kernel id id))))
(cl-defmethod jupyter-kill-kernel ((_kernel jupyter-server-kernel))
(cl-defmethod jupyter-kill-kernel ((_kernel jupyter--server-kernel))
;; The notebook server already takes care of forcing shutdown of a kernel.
(ignore))
(defclass jupyter-server-kernel-comm (jupyter-comm-layer)
((kernel :type jupyter-server-kernel :initarg :kernel)))
((kernel :type jupyter--server-kernel :initarg :kernel)))
(cl-defmethod jupyter-comm-id ((comm jupyter-server-kernel-comm))
(let* ((kernel (oref comm kernel))
@ -638,7 +638,7 @@ the kernel whose class is CLIENT-CLASS. Note that the clients
see jupyter-make-client."
(or client-class (setq client-class 'jupyter-kernel-client))
(let* ((specs (jupyter-server-kernelspecs server))
(kernel (jupyter-server-kernel
(kernel (jupyter--server-kernel
:server server
:spec (jupyter-guess-kernelspec kernel-name specs)))
(manager (jupyter-server-kernel-manager :kernel kernel)))
@ -707,7 +707,7 @@ the same meaning as in `jupyter-connect-repl'."
(or (jupyter-server-find-manager server kernel-id)
(let ((model (jupyter-api-get-kernel server kernel-id)))
(jupyter-server-kernel-manager
:kernel (jupyter-server-kernel
:kernel (jupyter--server-kernel
:id kernel-id
:server server
:spec (car (jupyter-find-kernelspecs

View file

@ -329,10 +329,10 @@
(when (process-live-p (car jupyter-test-notebook))
(delete-process (car jupyter-test-notebook))))))
(ert-deftest jupyter-server-kernel ()
(ert-deftest jupyter--server-kernel ()
:tags '(kernel server)
(jupyter-test-with-notebook server
(let ((kernel (jupyter-server-kernel
(let ((kernel (jupyter--server-kernel
:server server
:spec (jupyter-guess-kernelspec
"python" (jupyter-server-kernelspecs server)))))
@ -352,7 +352,7 @@
(ert-deftest jupyter-server-kernel-manager ()
:tags '(server)
(jupyter-test-with-notebook server
(let* ((kernel (jupyter-server-kernel
(let* ((kernel (jupyter--server-kernel
:server server
:spec (jupyter-guess-kernelspec
"python" (jupyter-server-kernelspecs server))))
@ -538,7 +538,7 @@
(server (jupyter-server :url "http://localhost:8882"))
(client (jupyter-kernel-client)))
(oset client kcomm (jupyter-server-kernel-comm
:kernel (jupyter-server-kernel
:kernel (jupyter--server-kernel
:id "id1"
:server server)))
(jupyter-server-name-client-kernel client "foo")

View file

@ -356,7 +356,7 @@ For `url-retrieve', the callback will be called with a nil status."
(defmacro jupyter-test-with-server-kernel (server name kernel &rest body)
(declare (indent 3))
(let ((id (make-symbol "id")))
`(let ((,kernel (jupyter-server-kernel
`(let ((,kernel (jupyter--server-kernel
:server server
:spec (jupyter-guess-kernelspec
,name (jupyter-server-kernelspecs ,server)))))