;;; ob-jupyter.el --- Jupyter integration with org-mode -*- lexical-binding: t -*- ;; Copyright (C) 2018 Nathaniel Nicandro ;; Author: Nathaniel Nicandro ;; Created: 21 Jan 2018 ;; Version: 0.0.1 ;; This program is free software; you can redistribute it and/or ;; modify it under the terms of the GNU General Public License as ;; published by the Free Software Foundation; either version 2, or (at ;; your option) any later version. ;; This program is distributed in the hope that it will be useful, but ;; WITHOUT ANY WARRANTY; without even the implied warranty of ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ;; General Public License for more details. ;; You should have received a copy of the GNU General Public License ;; along with GNU Emacs; see the file COPYING. If not, write to the ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, ;; Boston, MA 02111-1307, USA. ;;; Commentary: ;; TODO: Properly replace the source block results when multiple results are ;; appended. Currently what happens is that only the first result is removed ;; when re-execution the src block due to the behavior of ;; `org-babel-result-end'. The solution is to keep calling ;; `org-babel-result-end' moving point to the end of the results until ;; `org-babel-result-end' returns the same position twice. We may need to ;; advise this function to implement this behavior for jupyter blocks. ;; ;; An alternative is to collect all results in both async and sync cases before ;; insertion. Then if there are multiple types of data, for example images and ;; text, we can insert them all into a single RESULTS drawer. ;;; Code: (defgroup ob-jupyter nil "Jupyter integration with org-mdoe" :group 'org-babel) (require 'jupyter) (require 'ob) (declare-function org-element-at-point "org-element") (declare-function org-at-drawer-p "org") (declare-function org-element-property "org-element" (property element)) (declare-function org-element-type "org-element" (element)) (declare-function org-element-context "org-element" (&optional element)) (declare-function org-babel-variable-assignments:python "ob-python" (params)) (declare-function org-babel-python-table-or-string "ob-python" (results)) (declare-function org-babel-expand-body:generic "ob-core" (body params &optional var-lines)) (defcustom org-babel-jupyter-resource-directory "./.ob-jupyter/" "Directory used to store automatically generated image files. See `org-babel-jupyter-file-name'." :group 'ob-jupyter) (defvar org-babel-jupyter-session-clients (make-hash-table :test #'equal) "A hash table mapping session names to `jupyter-repl-client's.") (defvar org-babel-header-args:jupyter '((kernel . :any) (async . ((yes no)))) "Available header arguments for Jupter src-blocks.") (defvar org-babel-default-header-args:jupyter '((:kernel . "python") (:async . "no")) "Default header arguments for Jupyter src-blocks.") (defvar org-babel-jupyter-language-regex "^[ \t]*#\\+begin_src[ \t]+jupyter-\\([^ \f\t\n\r\v]+\\)[ \t]*" "Regular expression used to extract a source block's language name.") (defun org-babel-variable-assignments:jupyter (params &optional lang) "Assign variables in PARAMS according to the Jupyter kernel language. LANG is the kernel language of the source block. If LANG is nil, get the kernel language from the current source block. The variables are assigned by looking for the function `org-babel-variable-assignments:LANG'. If this function does not exist or if LANG cannot be determined, assign variables using `org-babel-variable-assignments:python'." (let* ((lang (or lang (save-excursion ;; TODO: This is not the most general case since we have ;; to consider inline calls. (when (and (org-in-src-block-p) (re-search-backward org-babel-jupyter-language-regex nil t)) (match-string 1))))) (fun (when lang (intern (concat "org-babel-variable-assignments:" lang))))) (if (functionp fun) (funcall fun params) (org-babel-variable-assignments:python params)))) (defun org-babel-expand-body:jupyter (body params &optional var-lines lang) "Expand BODY according to PARAMS. BODY is the code to expand, PARAMS should be the header arguments of the src block with BODY as its code, and VAR-LINES should be the list of strings containing the variables to evaluate before executing body. LANG is the kernel language of the source block. This function is similar to `org-babel-variable-assignments:jupyter' in that it attempts to find the kernel language of the source block if LANG is not provided. BODY is expanded by calling the function `org-babel-expand-body:LANG'. If this function doesn't exist or if LANG cannot be determined, fall back to `org-babel-expand-body:generic'." (let* ((lang (or lang (save-excursion (when (re-search-backward org-babel-jupyter-language-regex nil t) (match-string 1))))) (fun (when lang (intern (concat "org-babel-expand-body:" lang))))) (if (functionp fun) (funcall fun body params var-lines) (org-babel-expand-body:generic body params var-lines)))) (defun org-babel-edit-prep:jupyter (info) "Prepare the edit buffer according to INFO." (let* ((params (nth 2 info)) (session (alist-get :session params))) (jupyter-repl-associate-buffer (org-babel-jupyter-initiate-session session params)))) (defun org-babel-prep-session:jupyter (session params &optional no-execute) "Prepare a Jupyter SESSION according to PARAMS. If optional argument NO-EXECUTE is non-nil, do not execute any of the header variables in PARAMS." (let ((buffer (org-babel-jupyter-initiate-session session params)) (var-lines (org-babel-variable-assignments:jupyter params))) (with-current-buffer buffer (goto-char (point-max)) (when var-lines (jupyter-repl-replace-cell-code (mapconcat #'identity var-lines "\n")) ;; For `org-babel-load-session:jupyter', ensure that the loaded code ;; starts on a new line. (when no-execute (insert "\n"))) (unless no-execute (jupyter-send-execute-request jupyter-repl-current-client)) (current-buffer)))) (defun org-babel-load-session:jupyter (session body params) "In a Jupyter SESSION, load BODY according to PARAMS." (save-window-excursion (let ((buffer (org-babel-prep-session:jupyter session params 'noexecute))) (with-current-buffer buffer (insert (org-babel-chomp body)) (current-buffer))))) (defun org-babel-jupyter-initiate-session-by-key (session params) "Return the `jupyter-repl-client' buffer for SESSION. If SESSION does not have a client already, one is created based on SESSION and PARAMS. If SESSION ends with \".json\" then SESSION is interpreted as a kernel connection file and a new kernel connected to SESSION is created. Otherwise a kernel is run based on the `:kernel' parameter in PARAMS which should be either a valid kernel name or a prefix of one. The first kernel that is returned by `jupyter-find-kernelspecs' will be used." (let* ((kernel (alist-get :kernel params)) (key (concat session "-" kernel)) (client (or (gethash key org-babel-jupyter-session-clients) (let ((client (if (string-suffix-p ".json" session) (connect-jupyter-repl session) (run-jupyter-repl kernel)))) (jupyter-set client 'jupyter-include-other-output nil) (with-jupyter-repl-buffer client (let ((name (buffer-name))) (when (string-match "^\\*\\(.+\\)\\*" name) (rename-buffer (concat "*" (match-string 1 name) "-" session "*") 'unique))) (add-hook 'kill-buffer-hook (lambda () (remhash key org-babel-jupyter-session-clients)) nil t)) (puthash key client org-babel-jupyter-session-clients))))) (oref client buffer))) (defun org-babel-jupyter-initiate-session (&optional session params) "Initialize a Jupyter SESSION according to PARAMS." (if (equal session "none") (error "Need a session to run") (org-babel-jupyter-initiate-session-by-key session params))) (defun org-babel-jupyter-scratch-buffer () "Display a scratch buffer connected to the current block's session." (interactive) (let (buffer) (org-babel-do-in-edit-buffer (setq buffer (jupyter-repl-scratch-buffer))) (if buffer (pop-to-buffer buffer) (user-error "No source block at point")))) (defun org-babel-jupyter-file-name (data ext) "Return a file name based on DATA and EXT. `org-babel-jupyter-default-directory' is used as the directory name, the `sha1' hash of DATA is used as the base name, and EXT is used as the extension." (let ((dir (prog1 org-babel-jupyter-resource-directory (unless (file-directory-p org-babel-jupyter-resource-directory) (make-directory org-babel-jupyter-resource-directory)))) (ext (if (= (aref ext 0) ?.) ext (concat "." ext)))) (concat (file-name-as-directory dir) (sha1 data) ext))) (defun org-babel-jupyter--image-result (data file &optional overwrite base64-encoded) "Possibly write image DATA to FILE. If OVERWRITE is non-nil, overwrite FILE if it already exists. Otherwise if FILE already exists, DATA is not written to FILE. If BASE64-ENCODED is non-nil, the DATA is assumed to be encoded with the base64 encoding and is first decoded before writing to FILE. Return the cons cell (\"file\" . FILE), see `org-babel-jupyter-prepare-result'." (cons "file" (prog1 file (when (or overwrite (not (file-exists-p file))) (let ((buffer-file-coding-system (if base64-encoded 'binary buffer-file-coding-system)) (require-final-newline nil)) (with-temp-file file (insert data) (when base64-encoded (base64-decode-region (point-min) (point-max))))))))) (defun org-babel-jupyter-prepare-result (data metadata params) "Return the rendered DATA. DATA is converted into a representation suitable for display in an `org-mode' buffer depending on DATA is a plist, (:mimetype1 value1 ...), containing the different representations of a result returned by a kernel. Preparing a result which is used to render a result which can be passed to `org-babel-insert-result'. METADATA is the metadata plist used to render DATA with, as returned by the Jupyter kernel. This plist typically contains information such as the size of an image to be rendered. The metadata plist is currently unused. PARAMS is the source block parameter list as passed to `org-babel-execute:jupyter'. Currently this is used to extract the file name of an image file when DATA can be rendered as an image. If no file name is given, one is generated based on the image data and mimetype, see `org-babel-jupyter-file-name'. PARAMS is also used to intelligently choose the rendering parameter used for result insertion. This function returns a cons cell (RENDER-PARAM . RESULT) where RENDER-PARAM is either a result parameter, i.e. one of the result parameters of `org-babel-insert-result', or a key value pair which should be appended to the PARAMS list when rendering RESULT. For example, if DATA only contains the mimetype `:text/markdown', the RESULT-PARAM will be (:wrap . \"SRC markdown\") and RESULT will be the markdown text which should be wrapped in an \"EXPORT markdown\" block. See `org-babel-insert-result'." (let ((mimetypes (cl-loop for elem in data if (keywordp elem) collect elem)) (result-params (alist-get :result-params params)) itype) (cond ((memq :text/org mimetypes) (cons (unless (member "raw" result-params) "org") (plist-get data :text/org))) ((memq :text/html mimetypes) (let ((html (plist-get data :text/html))) (save-match-data ;; Allow handling of non-string data but with an html mimetype at a ;; higher level (if (and (stringp html) (string-match "^