;;; ob-jupyter.el --- Jupyter integration with org-mode -*- lexical-binding: t -*- ;; Copyright (C) 2018 Nathaniel Nicandro ;; Author: Nathaniel Nicandro ;; Created: 21 Jan 2018 ;; Version: 0.0.1 ;; This program is free software; you can redistribute it and/or ;; modify it under the terms of the GNU General Public License as ;; published by the Free Software Foundation; either version 2, or (at ;; your option) any later version. ;; This program is distributed in the hope that it will be useful, but ;; WITHOUT ANY WARRANTY; without even the implied warranty of ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ;; General Public License for more details. ;; You should have received a copy of the GNU General Public License ;; along with GNU Emacs; see the file COPYING. If not, write to the ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, ;; Boston, MA 02111-1307, USA. ;;; Commentary: ;; ;;; Code: (defgroup ob-jupyter nil "Jupyter integration with org-mdoe" :group 'org-babel) (require 'jupyter) (require 'ob) (declare-function cddar "cl" (x)) (declare-function org-element-at-point "org-element") (declare-function org-element-property "org-element" (property element)) (declare-function org-element-type "org-element" (element)) (declare-function org-element-context "org-element" (&optional element)) (declare-function org-babel-variable-assignments:python "ob-python" (params)) (declare-function org-babel-expand-body:generic "ob-core" (body params &optional var-lines)) (defcustom org-babel-jupyter-resource-directory "./.ob-jupyter/" "Directory used to store automatically generated image files. See `org-babel-jupyter-file-name'." :group 'ob-jupyter) (defvar org-babel-jupyter-session-clients (make-hash-table :test #'equal) "A hash table mapping session names to `jupyter-repl-client's.") (defvar org-babel-header-args:jupyter '((kernel . :any) (async . ((yes no)))) "Available header arguments for Jupter src-blocks.") (defvar org-babel-default-header-args:jupyter '((:kernel . "python") (:async . "no")) "Default header arguments for Jupyter src-blocks.") (defvar org-babel-jupyter-language-regex "^[ \t]*#\\+begin_src[ \t]+jupyter-\\([^ \f\t\n\r\v]+\\)[ \t]*" "Regular expression used to extract a source block's language name.") ;; TODO: Handle the case when the kernel changes ;; ;; TODO: How to cache results properly? To handle the case when the kernel ;; argument changes, we are checking for lang every time. Previously we were ;; adding a text property at the beginning of the block, what we can do instead ;; is add one as the last character of the kernel argument. ;; ;; ;; FIXME: Parsing with `org-element-at-point' seems to depend on how far away a ;; source block is from the subtree header line. This is not good if we want ;; this to be fast, it seems the default value of `org-element-use-cache' is to ;; blame. In the meantime, do not write large subtrees or large src blocks. ;; Shouldn't have to anyway. (defun org-babel-jupyter-src-block-lang (&optional context) "Get the kernel language of a Jupyter src-block. CONTEXT should be a src-block org element and and defaults to the `org-element-context'." (or context (setq context (org-element-context))) (let ((type (org-element-type context))) (when (and (memq type '(src-block inline-src-block)) (equal (org-element-property :language context) "jupyter")) (let* ((kernel (or (alist-get :kernel (ignore-errors (org-babel-parse-header-arguments (org-element-property :parameters context)))) (alist-get :kernel (ignore-errors (apply #'append (mapcar #'org-babel-parse-header-arguments (org-element-property :header context))))) (alist-get :kernel (ignore-errors (org-with-point-at (org-element-property :begin context) ;; FIXME: This looks expensive, but it seems like ;; having the kernel language be a file local variable ;; would be common (org-babel-params-from-properties "jupyter")))))) ;; Possibly cache based on the kernel name. Since ;; `jupyter-find-kernelspecs' matches prefixes, it could ;; possibly match single letters which is not good. (spec (cddar (jupyter-find-kernelspecs kernel)))) (plist-get spec :language))))) ;; All calls of `org-src--get-lang-mode' in `org-src' currently have `point' at ;; the src-block <2018-01-23 Tue>. ;; (defun org-babel-jupyter--get-lang-mode (orig-fun lang &rest args) ;; "Identical to `org-src--get-lang-mode' but handle jupyter blocks specially. ;; This is an advice function for `org-src--get-lang-mode' which ;; should be passed as the ORIG-FUN argument. Whenever LANG is ;; \"jupyter\" pass the result of `org-babel-jupyter-src-block-lang' ;; to ORIG-FUN as its first argument instead of \"jupyter\". ARGS ;; are any additional arguments to pass to ORIG-FUN. This should be ;; nil." ;; (apply orig-fun ;; (if (equal lang "jupyter") ;; (org-babel-jupyter-src-block-lang) ;; lang) ;; args)) ;; (advice-add 'org-src--get-lang-mode :around #'org-babel-jupyter--get-lang-mode) (defun org-babel-variable-assignments:jupyter (params &optional lang) "Assign variables in PARAMS according to the Jupyter kernel language. Use `org-babel-jupyter-src-block-lang' to get the kernel language of the src-block ELEMENT and call the variable assignment function for the language. ELEMENT defaults to the `org-element-at-point'. So if LANG is the kernel language, then call org-babel-variable-assignments:LANG If the above function doesn't exist or if no kernel langauge can be found, fall back to `org-babel-variable-assignments:python'." (let* ((lang (or lang (save-excursion (when (re-search-backward org-babel-jupyter-language-regex nil t) (match-string 1))))) (fun (when lang (intern (concat "org-babel-variable-assignments:" lang))))) (if (functionp fun) (funcall fun params) (org-babel-variable-assignments:python params)))) (defun org-babel-expand-body:jupyter (body params &optional var-lines lang) "Expand BODY according to PARAMS. BODY is the code to expand, PARAMS should be the header arguments of the src block with BODY as its code, and VAR-LINES should be the list of strings containing the variables to evaluate before executing body. This function is similar to `org-babel-variable-assignments:jupyter' in that it finds the kernel language of the src-block ELEMENT, defaulting to the `org-element-at-point', to find get the kernel language of BODY. So if LANG is the kernel language, call the function org-babel-expand-body:LANG to expand BODY. If the above function doesn't exist or if no kernel langauge can be found fall back to `org-babel-expand-body:generic'." (let* ((lang (or lang (save-excursion (when (re-search-backward org-babel-jupyter-language-regex nil t) (match-string 1))))) (fun (when lang (intern (concat "org-babel-expand-body:" lang))))) (if (functionp fun) (funcall fun body params var-lines) (org-babel-expand-body:generic body params var-lines)))) (defun org-babel-edit-prep:jupyter (info) "Prepare the edit buffer according to INFO." (let* ((params (nth 2 info)) (session (alist-get :session params))) (jupyter-repl-associate-buffer (org-babel-jupyter-initiate-session session params)))) (defun org-babel-prep-session:jupyter (session params &optional no-execute) "Prepare a Jupyter SESSION according to PARAMS. If optional argument NO-EXECUTE is non-nil, do not execute any of the header variables in PARAMS." (let ((buffer (org-babel-jupyter-initiate-session session params)) (var-lines (org-babel-variable-assignments:jupyter params))) (with-current-buffer buffer (goto-char (point-max)) (when var-lines (jupyter-repl-replace-cell-code (mapconcat #'identity var-lines "\n")) ;; For `org-babel-load-session:jupyter' (when no-execute (insert "\n"))) (unless no-execute (jupyter-execute-request jupyter-repl-current-client)) (current-buffer)))) (defun org-babel-load-session:jupyter (session body params) "In a Jupyter SESSION, load BODY according to PARAMS." (save-window-excursion (let ((buffer (org-babel-prep-session:jupyter session params 'noexecute))) (with-current-buffer buffer (insert (org-babel-chomp body)) (current-buffer))))) (defun org-babel-jupyter-initiate-session-by-key (session params) "Return the `jupyter-repl-client' for SESSION. If SESSION does not have a client already, one is created based on SESSION and PARAMS. If SESSION ends with \".json\" then SESSION is interpreted as a kernel connection file and a new kernel connected to SESSION is created. Otherwise a kernel is run based on the `:kernel' parameter in PARAMS which should be either a valid kernel name or a prefix of one. The first kernel that is returned by `jupyter-find-kernelspecs' will be used." (let* ((kernel (alist-get :kernel params)) (key (concat session "-" kernel))) (oref (or (gethash key org-babel-jupyter-session-clients) (let ((client (if (string-suffix-p ".json" session) (connect-jupyter-repl session) (run-jupyter-repl kernel)))) (jupyter-set client 'jupyter-include-other-output nil) (with-jupyter-repl-buffer client (let ((name (buffer-name))) (when (string-match "^\\*\\(.+\\)\\*" name) (rename-buffer (concat "*" (match-string 1 name) "-" session "*") 'unique))) (add-hook 'kill-buffer-hook (lambda () (remhash key org-babel-jupyter-session-clients)) nil t)) (puthash key client org-babel-jupyter-session-clients))) buffer))) (defun org-babel-jupyter-initiate-session (&optional session params) "Initialize a Jupyter SESSION according to PARAMS." (if (equal session "none") (error "Need a session to run") (org-babel-jupyter-initiate-session-by-key session params))) (defun org-babel-jupyter-file-name (data ext) "Return a file name based on DATA. DATA is usually the contents of an image to create a file name for. The generated absolute file name is based on the following: 1. The value of `org-babel-jupyter-resource-directory' 2. The `sha1' hash of DATA 3. .EXT Where EXT should be the file extension to give the file (excluding the dot)." (let ((dir (prog1 org-babel-jupyter-resource-directory (unless (file-directory-p org-babel-jupyter-resource-directory) (make-directory org-babel-jupyter-resource-directory))))) (concat (file-name-as-directory dir) (sha1 data) "." ext))) (defun org-babel-jupyter--image-result (data file &optional overwrite base64-encoded) "Possibly write DATA to FILE. If OVERWRITE is non-nil, overwrite FILE if it already exists. Otherwise if FILE already exists, DATA is not written to FILE. If BASE64-ENCODED is non-nil, the DATA is assumed to be encoded with the base64 encoding and is first decoded before writing to FILE. Return the cons cell (\"file\" . FILE), see `org-babel-jupyter-prepare-result'." (cons "file" (prog1 file (when (or overwrite (not (file-exists-p file))) (let ((buffer-file-coding-system (if base64-encoded 'binary buffer-file-coding-system)) (require-final-newline nil)) (with-temp-file file (insert data) (when base64-encoded (base64-decode-region (point-min) (point-max))))))))) (defun org-babel-jupyter-prepare-result (data metadata params) "Return the rendered DATA. DATA is a plist, (:mimetype1 value1 ...), which is used to render a result which can be passed to `org-babel-insert-result'. METADATA is the metadata plist used to render DATA with, as returned by the Jupyter kernel. This plist typically contains information such as the size of an image to be rendered. The metadata plist is currently unused. PARAMS is the source block parameter list as passed to `org-babel-execute:jupyter'. Currently this is only used to extract the file name of an image file when DATA can be rendered as an image type (either `:image/png' or `:image/svg+xml') when a file name is passed to the code block. If no file name is given one is generated based on DATA and the mimetype, see `org-babel-jupyter-file-name'. This function returns a cons cell (RESULT-PARAM . RESULT) where RESULT-PARAM is either a result parameter, i.e. one of the result paramters of `org-babel-insert-result', or a key value pair which should be appended to the PARAMS list when to render RESULT. For example, if DATA only contains the mimetype `:text/markdown', the RESULT-PARAM will be (:wrap . \"EXPORT markdown\") and RESULT will be the markdown text which should be wrapped in an \"EXPORT markdown\" block. See `org-babel-insert-result'." (let ((mimetypes (seq-filter #'keywordp data)) (result-params (alist-get :result-params params))) (cond ((memq :text/org mimetypes) (cons (unless (member "raw" result-params) "org") (plist-get data :text/org))) ((memq :text/html mimetypes) (let ((html (plist-get data :text/html))) (save-match-data ;; FIXME: This regex does not match all possibilities of the DATA URL ;; scheme. (if (string-match "^