From f0984eab55c0b04751d51894286bfd98556bb807 Mon Sep 17 00:00:00 2001 From: dickmao Date: Sat, 1 Dec 2018 18:54:58 -0500 Subject: [PATCH] jupyterhub basic (PAM only) `ein:login` or `ein:notebooklist-login` is the preferred way to access jupyterhub, although `ein:jupyterhub-connect` is still autoloaded. --- .travis.yml | 6 +- Makefile | 1 - features/jupyterhub.feature | 7 + features/notebooklist.feature | 32 +-- features/step-definitions/ein-steps.el | 49 +++-- features/support/env.el | 6 +- lisp/ein-contents-api.el | 10 +- lisp/ein-core.el | 5 +- lisp/ein-dev.el | 5 - lisp/ein-jupyter.el | 128 +++++------ lisp/ein-jupyterhub.el | 284 ++++++++++++++----------- lisp/ein-kernel.el | 34 +-- lisp/ein-notebook.el | 8 +- lisp/ein-notebooklist.el | 128 ++++++----- lisp/ein-pager.el | 1 - lisp/ein-pkg.el | 1 - lisp/ein-process.el | 33 +-- lisp/ein-query.el | 75 +------ lisp/ein-smartrep.el | 3 +- lisp/ein-utils.el | 3 +- lisp/ein-websocket.el | 29 +-- test/ein-testing.el | 9 +- test/test-ein-utils.el | 1 + 23 files changed, 400 insertions(+), 458 deletions(-) create mode 100644 features/jupyterhub.feature diff --git a/.travis.yml b/.travis.yml index bf2aa91..43fd3f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ addons: packages: - gnutls-bin - sharutils + - nodejs cache: directories: @@ -25,7 +26,7 @@ matrix: env: EVM_EMACS=emacs-26.1-travis IPYTHON=6.2.1 - os: linux python: 3.6 - env: EVM_EMACS=emacs-26.1-travis IPYTHON=7.0.1 + env: EVM_EMACS=emacs-26.1-travis IPYTHON=7.2.0 - os: osx language: generic env: EVM_EMACS=emacs-25.2 IPYTHON=5.8.0 TOXENV=py27 @@ -37,6 +38,7 @@ install: - sh tools/install-virtualenv.sh - if [ "x$TRAVIS_OS_NAME" = "xosx" ]; then eval "$(pyenv init -)" ; pyenv activate $TOXENV ; fi - pip install jupyter ipython\<=$IPYTHON + - if [[ "x$TRAVIS_PYTHON_VERSION" == x3* ]]; then pip install jupyterhub ; npm install -g configurable-http-proxy ; pip install jupyterhub-dummyauthenticator ; fi - jupyter kernelspec list - ipython --version @@ -48,4 +50,4 @@ before_script: script: - make test-install - - make test || ( ( zip -q - log/{testein,testfunc,ecukes}.* 2>/dev/null | uuencode log.zip ) && ( printf "To diagnose, travis logs -i | dos2unix | sed '/^begin 644/,/^end/!d' | uudecode" ) && false ) + - make test || ( ( zip -q - log/{testein,testfunc,ecukes}.* 2>/dev/null | uuencode log.zip ) && ( printf "To diagnose, travis logs -i | dos2unix | sed '/^begin 644/,/^end/!d' | uudecode" ) && false) diff --git a/Makefile b/Makefile index 62998ec..4fee8a1 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,6 @@ test: quick test-int test-int: cask exec ert-runner -L ./lisp -L ./test -l test/testfunc.el test/test-func.el cask exec ecukes - cask exec ecukes --tags @timestamp .PHONY: test-unit test-unit: diff --git a/features/jupyterhub.feature b/features/jupyterhub.feature new file mode 100644 index 0000000..651074b --- /dev/null +++ b/features/jupyterhub.feature @@ -0,0 +1,7 @@ +@jupyterhub +Scenario: basic login + Given I start and login to jupyterhub configured "c.JupyterHub.answer_yes=True\nc.JupyterHub.authenticator_class='dummyauthenticator.DummyAuthenticator'\nc.JupyterHub.cookie_secret_file = '/var/tmp/jupyterhub_cookie_secret'" + And I switch to log expr "ein:log-all-buffer-name" + Then I should not see "[warn]" + And I should not see "[error]" + Given I start and login to the server configured "\n" diff --git a/features/notebooklist.feature b/features/notebooklist.feature index 0f75abd..6cb0fe9 100644 --- a/features/notebooklist.feature +++ b/features/notebooklist.feature @@ -35,6 +35,15 @@ Scenario: Stop after closing notebook And I go to beginning of line And I click without going top on "Open" +@content +Scenario: Read a massive directory + Given I create a directory "/var/tmp/fg7Cv8" with depth 5 and width 10 + And I get into notebook mode "/var/tmp/fg7Cv8" "8/4/3/bar.ipynb" + And I open notebook "bar.ipynb" + And I open file "foo.txt" + And notebooklist-list-paths does not contain "5/5/5/foo.txt" + And notebooklist-list-paths contains "foo.txt" + @login Scenario: No token server Given I start the server configured "c.NotebookApp.token = u''\n" @@ -42,14 +51,6 @@ Scenario: No token server Then I should not see "[warn]" And I should not see "[error]" -@login -Scenario: With token server - Given I start the server configured "\n" - And I login if necessary - And I switch to log expr "ein:log-all-buffer-name" - Then I should not see "[warn]" - And I should not see "[error]" - @login Scenario: With token server, get from server buffer Given I start the server configured "\n" @@ -84,11 +85,10 @@ Scenario: Logging into nowhere Given I login erroneously to adfljdsf.org:8432 Then I should see message "ein: [error] Login to https://adfljdsf.org:8432 failed" -@content -Scenario: Read a massive directory - Given I create a directory "/var/tmp/fg7Cv8" with depth 5 and width 10 - And I get into notebook mode "/var/tmp/fg7Cv8" "8/4/3/bar.ipynb" - And I open notebook "bar.ipynb" - And I open file "foo.txt" - And notebooklist-list-paths does not contain "5/5/5/foo.txt" - And notebooklist-list-paths contains "foo.txt" +@login +Scenario: With token server + Given I start the server configured "\n" + And I login if necessary + And I switch to log expr "ein:log-all-buffer-name" + Then I should not see "[warn]" + And I should not see "[error]" diff --git a/features/step-definitions/ein-steps.el b/features/step-definitions/ein-steps.el index d32046c..1a61ea9 100644 --- a/features/step-definitions/ein-steps.el +++ b/features/step-definitions/ein-steps.el @@ -6,7 +6,7 @@ (When "^header \\(does not \\)?says? \"\\(.+\\)\"$" (lambda (negate says) (let ((equal-p (string= (substitute-command-keys says) (substitute-command-keys (slot-value (slot-value ein:%notification% 'kernel) 'message))))) - (cl-assert (if negate (not equal-p) equal-p))))) + (if negate (should-not equal-p) (should equal-p))))) (When "^I kill kernel$" (lambda () @@ -18,7 +18,7 @@ (cl-letf (((symbol-function 'y-or-n-p) (lambda (&rest ignore) t))) (ein:kernel-reconnect-session (ein:$notebook-kernel ein:%notebook%) (lambda (kernel session-p) - (assert (not session-p))))))) + (should-not session-p)))))) (When "I restart kernel$" (lambda () @@ -55,7 +55,8 @@ (When "^I am in notebooklist buffer$" (lambda () - (switch-to-buffer (ein:notebooklist-get-buffer (car (ein:jupyter-server-conn-info)))))) + (switch-to-buffer (ein:notebooklist-get-buffer (car (ein:jupyter-server-conn-info)))) + (ein:testing-wait-until (lambda () (eq major-mode 'ein:notebooklist-mode))))) (When "^I wait \\([.0-9]+\\) seconds?$" (lambda (seconds) @@ -82,6 +83,7 @@ (When "^I stop the server$" (lambda () + (cancel-function-timers #'ein:notebooklist-reload) (cl-letf (((symbol-function 'y-or-n-p) #'ignore)) (ein:jupyter-server-stop t)) (loop repeat 10 @@ -89,9 +91,21 @@ until (null (get-buffer-process buffer)) do (sleep-for 1) finally do (ein:aif (get-buffer-process buffer) (delete-process it))) + (clrhash ein:notebooklist-map) (When "I clear log expr \"ein:log-all-buffer-name\"") (When "I clear log expr \"ein:jupyter-server-buffer-name\""))) +(When "^I start and login to jupyterhub configured \"\\(.*\\)\"$" + (lambda (config) + (When "I stop the server") + (cl-letf (((symbol-function 'ein:notebooklist-ask-user-pw-pair) + (lambda (&rest args) (list (intern (user-login-name)) "")))) + (with-temp-file ".ecukes-temp-config.py" (insert (s-replace "\\n" "\n" config))) + (let ((ein:jupyter-server-args + '("--debug" "--no-db" "--config=.ecukes-temp-config.py"))) + (ein:jupyter-server-start (executable-find "jupyterhub") nil)) + (ein:testing-wait-until (lambda () (ein:notebooklist-list)) nil 20000 1000)))) + (When "^I start \\(and login to \\)?the server configured \"\\(.*\\)\"$" (lambda (login config) (When "I stop the server") @@ -158,7 +172,7 @@ until (search stop (buffer-string)) do (And (format "I click on \"%s\"" go)) do (sleep-for 0 1000) - finally do (if (not (search stop (buffer-string))) (assert nil))))) + finally do (should (search stop (buffer-string)))))) (When "^I click\\( without going top\\)? on \"\\(.+\\)\"$" (lambda (stay word) @@ -167,7 +181,7 @@ (goto-char (point-min))) (let ((search (re-search-forward (format "\\[%s\\]" word) nil t)) (msg "Cannot go to link '%s' in buffer: %s")) - (cl-assert search nil msg word (buffer-string)) + (should search) (backward-char) (When "I press \"RET\"") (sit-for 0.8) @@ -233,7 +247,10 @@ (when (f-exists? dir) (f-delete dir t)) (f-mkdir dir) - (ein:testing-make-directory-level dir 1 (string-to-number width) (string-to-number depth)))) + (ein:testing-make-directory-level dir 1 + (string-to-number width) + (string-to-number depth)) + (call-process-shell-command (format "find %s -print | xargs du" dir)))) (When "^\"\\(.+\\)\" should \\(not \\)?include \"\\(.+\\)\"$" (lambda (variable negate value) @@ -257,8 +274,7 @@ (When "I stop the server") (When (format "I find file \"%s\"" (concat (file-name-as-directory notebook-dir) file-path))) (When "I press \"C-c C-z\"") - (ein:testing-wait-until (lambda () (ein:notebooklist-list)) nil 20000 1000) - )) + (ein:testing-wait-until (lambda () (ein:notebooklist-list)) nil 20000 1000))) (When "^I find file \"\\(.+\\)\"$" (lambda (file-name) @@ -267,23 +283,18 @@ (ein:ipynb-mode)) )) -(When "notebooklist-list-paths does not contain \"\\(.+\\)\"$" - (lambda (file-name) +(When "notebooklist-list-paths\\( does not\\)? contains? \"\\(.+\\)\"$" + (lambda (negate file-name) (Given "I am in notebooklist buffer") - (let ((nbpath (ein:url (car (ein:jupyter-server-conn-info)) file-name))) - (assert (not (member nbpath (ein:notebooklist-list-paths))))))) - -(When "notebooklist-list-paths contains \"\\(.+\\)\"$" - (lambda (file-name) - (Given "I am in notebooklist buffer") - (let ((nbpath (ein:url (car (ein:jupyter-server-conn-info)) file-name))) - (assert (member nbpath (ein:notebooklist-list-paths)))))) + (let* ((nbpath (ein:url (car (ein:jupyter-server-conn-info)) file-name)) + (contains-p (member nbpath (ein:notebooklist-list-paths)))) + (should (if negate (not contains-p) contains-p))))) (When "I open \\(notebook\\|file\\) \"\\(.+\\)\"$" (lambda (content-type file-name) (Given "I am in notebooklist buffer") (And "I clear log expr \"ein:log-all-buffer-name\"") - (let ((nbpath (ein:url (car (ein:jupyter-server-conn-info)) file-name))) + (lexical-let ((nbpath (ein:url (car (ein:jupyter-server-conn-info)) file-name))) (cl-letf (((symbol-function 'ein:notebooklist-ask-path) (lambda (&rest args) nbpath))) (When (format "I press \"C-c C-%s\"" (if (string= content-type "file") "f" "o"))) diff --git a/features/support/env.el b/features/support/env.el index 6b8ca40..3ff92d1 100644 --- a/features/support/env.el +++ b/features/support/env.el @@ -19,6 +19,9 @@ (require 'ein-timestamp) (!cons "timestamp" ecukes-exclude-tags)) +(if (null (executable-find "jupyterhub")) + (!cons "jupyterhub" ecukes-exclude-tags)) + (defvar ein:testing-jupyter-server-root (f-parent (f-dirname load-file-name))) (defun ein:testing-after-scenario () @@ -63,8 +66,7 @@ (ein:testing-after-scenario)) (Teardown - (cl-letf (((symbol-function 'y-or-n-p) #'ignore)) - (ein:jupyter-server-stop t))) + (Given "I stop the server")) (Fail (if noninteractive diff --git a/lisp/ein-contents-api.el b/lisp/ein-contents-api.el index c6571b0..bbe905a 100644 --- a/lisp/ein-contents-api.el +++ b/lisp/ein-contents-api.el @@ -204,7 +204,7 @@ global setting. For global setting and more information, see content)) (defun ein:content-query-hierarchy* (url-or-port path callback sessions depth content) - "Returns list (tree) of content objects" + "Returns list (tree) of content objects. CALLBACK accepts tree." (lexical-let* ((url-or-port url-or-port) (path path) (callback callback) @@ -222,8 +222,8 @@ global setting. For global setting and more information, see with c0 if (not (string= "directory" (plist-get item :type))) do (setf c0 (ein:new-content url-or-port path item)) - (setf (ein:$content-session-p c0) - (gethash (ein:$content-path c0) sessions)) + (setf (ein:$content-session-p c0) + (gethash (ein:$content-path c0) sessions)) and collect c0 end))) (deferred:$ @@ -246,8 +246,8 @@ global setting. For global setting and more information, see (deferred:nextc it (lambda (tree) (let ((result (append others tree))) - (if (string= path "") - (setf (gethash url-or-port *ein:content-hierarchy*) (-flatten result))) + (when (string= path "") + (setf (gethash url-or-port *ein:content-hierarchy*) (-flatten result))) (funcall callback result))))))) (defun ein:content-query-hierarchy (url-or-port callback) diff --git a/lisp/ein-core.el b/lisp/ein-core.el index b741042..28f34e4 100644 --- a/lisp/ein-core.el +++ b/lisp/ein-core.el @@ -37,7 +37,6 @@ (require 'ein) ; get autoloaded functions into namespace (require 'ein-utils) - (defgroup ein nil "IPython notebook client in Emacs" :group 'applications @@ -220,13 +219,11 @@ the source is in git repository) or elpa version." (ein:log 'warn "No recorded notebook version for %s" url-or-port) 5)) -;; TODO: Use symbols instead of numbers for notebook version ('jupyter and 'legacy)? (defun ein:query-notebook-version (url-or-port callback) "Send for notebook version of URL-OR-PORT with CALLBACK arity 0 (just a semaphore)" (ein:query-singleton-ajax (list 'query-notebook-version url-or-port) - (ein:jupyterhub-correct-query-url-maybe - (ein:url url-or-port "api")) + (ein:url url-or-port "api") :parser #'ein:json-read :sync ein:force-sync :complete (apply-partially #'ein:query-notebook-version--complete url-or-port callback))) diff --git a/lisp/ein-dev.el b/lisp/ein-dev.el index eff2f0c..32c3a14 100644 --- a/lisp/ein-dev.el +++ b/lisp/ein-dev.el @@ -31,11 +31,6 @@ (require 'ein-notebook) (require 'ein-subpackages) -(defcustom ein:dev-prefer-deferred nil - "Deferred chains have unpredictable and often delayed timings. For some user interactions, it may be preferable to act synchronously." - :group 'ein - :type 'boolean) - ;;;###autoload (defun ein:dev-insert-mode-map (map-string) "Insert mode-map into rst document. For README.rst." diff --git a/lisp/ein-jupyter.el b/lisp/ein-jupyter.el index e66126b..6af8f34 100644 --- a/lisp/ein-jupyter.el +++ b/lisp/ein-jupyter.el @@ -61,20 +61,16 @@ the notebook directory, you can set it here for future calls to "Return the emacs process object of our session" (get-buffer-process (get-buffer ein:jupyter-server-buffer-name))) -(defun ein:jupyter-server--cmd (path dir) - (append (list path - "notebook" - (format "--notebook-dir=%s" (convert-standard-filename dir))) - ein:jupyter-server-args)) - (defun ein:jupyter-server--run (buf cmd dir &optional args) - (let ((proc (apply #'start-process + (let* ((vargs (append (if dir + `("notebook" ,(format "--notebook-dir=%s" + (convert-standard-filename dir)))) + (or args ein:jupyter-server-args))) + (proc (apply #'start-process *ein:jupyter-server-process-name* buf cmd - "notebook" - (format "--notebook-dir=%s" (convert-standard-filename dir)) - (or args ein:jupyter-server-args)))) + vargs))) (set-process-query-on-exit-flag proc nil) proc)) @@ -89,14 +85,14 @@ the notebook directory, you can set it here for future calls to (goto-char (point-max)) (re-search-backward (format "Process %s" *ein:jupyter-server-process-name*) nil "") ;; important if we start-stop-start - (if (and (re-search-forward "otebook [iI]s [rR]unning" nil t) - (re-search-forward "\\(https?://[^:]+:[0-9]+\\)\\(?:/\\?token=\\([[:alnum:]]+\\)\\)?" nil t)) - (let ((raw-url (match-string 1)) - (token (or (match-string 2) ""))) - (setq result (list (ein:url raw-url) token))))))) + (when (re-search-forward "\\([[:alnum:]]+\\) is\\( now\\)? running" nil t) + (let ((hub-p (search "jupyterhub" (downcase (match-string 1))))) + (when (re-search-forward "\\(https?://[^:]*:[0-9]+\\)\\(?:/\\?token=\\([[:alnum:]]+\\)\\)?" nil t) + (let ((raw-url (match-string 1)) + (token (or (match-string 2) (and (not hub-p) "")))) + (setq result (list (ein:url raw-url) token))))))))) result)) -;;;###autoload (defun ein:jupyter-server-login-and-open (&optional callback) "Log in and open a notebooklist buffer for a running jupyter notebook server. @@ -111,17 +107,22 @@ via a call to `ein:notebooklist-open'." (ein:notebooklist-login url-or-port callback)))) (defsubst ein:set-process-sentinel (proc url-or-port) - "Adjust notebooklist corresponding to URL-OR-PORT when the PROC gets signalled. Would use `add-function' if it didn't produce gv-ref warnings." + "URL-OR-PORT might get redirected from (ein:jupyter-server-conn-info). +This is currently only the case for jupyterhub. +Once login handshake provides the new URL-OR-PORT, we set various state as pertains +our singleton jupyter server process here." + + ;; Would have used `add-function' if it didn't produce gv-ref warnings. (set-process-sentinel proc - (apply-partially (lambda (url-or-port* sentinel process event) - (ein:aif sentinel (funcall it process event)) - (funcall #'ein:notebooklist-proc--sentinel url-or-port* process event)) + (apply-partially (lambda (url-or-port* sentinel proc* event) + (ein:aif sentinel (funcall it proc* event)) + (funcall #'ein:notebooklist-sentinel url-or-port* proc* event)) url-or-port (process-sentinel proc)))) ;;;###autoload (defun ein:jupyter-server-start (server-cmd-path notebook-directory &optional no-login-p login-callback) - "Start SERVER-CMD_PATH with `--notebook-dir' NOTEBOOK-DIRECTORY. Login after connection established unless NO-LOGIN-P is set. LOGIN-CALLBACK taking single argument, the buffer created by ein:notebooklist-open--finish. + "Start SERVER-CMD_PATH with `--notebook-dir' NOTEBOOK-DIRECTORY. Login after connection established unless NO-LOGIN-P is set. LOGIN-CALLBACK takes two arguments, the buffer created by ein:notebooklist-open--finish, and the url-or-port argument of ein:notebooklist-open*. This command opens an asynchronous process running the jupyter notebook server and then tries to detect the url and password to @@ -150,7 +151,8 @@ the log of the running jupyter server." (read-directory-name "Notebook directory: " (or *ein:last-jupyter-directory* ein:jupyter-default-notebook-directory)))) - (list server-cmd-path notebook-directory nil #'pop-to-buffer))) + (list server-cmd-path notebook-directory nil (lambda (buffer url-or-port) + (pop-to-buffer buffer))))) (assert (and (file-exists-p server-cmd-path) (file-executable-p server-cmd-path)) t "Command %s is not valid!" server-cmd-path) @@ -160,44 +162,26 @@ the log of the running jupyter server." (error "Please first M-x ein:jupyter-server-stop")) (add-hook 'kill-emacs-hook #'(lambda () (ignore-errors (ein:jupyter-server-stop t)))) - (lexical-let* (done-p - (no-login-p no-login-p) - (login-callback login-callback) - (proc (ein:jupyter-server--run ein:jupyter-server-buffer-name - *ein:last-jupyter-command* - *ein:last-jupyter-directory*)) - (buf (process-buffer proc))) + (let ((proc (ein:jupyter-server--run ein:jupyter-server-buffer-name + *ein:last-jupyter-command* + *ein:last-jupyter-directory*))) (when (eql system-type 'windows-nt) (accept-process-output proc (/ ein:jupyter-server-run-timeout 1000))) - (if ein:dev-prefer-deferred - (deferred:$ - (deferred:timeout - ein:jupyter-server-run-timeout 'timeout - (deferred:lambda () - (ein:aif (car (ein:jupyter-server-conn-info)) - (progn (ein:set-process-sentinel proc it) no-login-p) - (deferred:nextc (deferred:wait (/ ein:jupyter-server-run-timeout 5)) self)))) - (deferred:nextc it - (lambda (no-login-p) - (if (eq no-login-p 'timeout) - (progn - (setf done-p 'error) - (ein:log 'warn "Jupyter server failed to start, cancelling operation.") - (ein:jupyter-server-stop t)) - (setf done-p t) - (unless no-login-p - (ein:jupyter-server-login-and-open login-callback)))))) - (loop repeat 30 - until (car (ein:jupyter-server-conn-info buf)) - do (sleep-for 0 500) - finally do - (ein:aif (car (ein:jupyter-server-conn-info buf)) - (progn (ein:set-process-sentinel proc it) (setf done-p t)) - (setf done-p "error") - (ein:log 'warn "Jupyter server failed to start, cancelling operation") - (ein:jupyter-server-stop t))) - (if (and (not no-login-p) (ein:jupyter-server-process)) - (ein:jupyter-server-login-and-open login-callback))))) + (loop repeat 30 + until (car (ein:jupyter-server-conn-info ein:jupyter-server-buffer-name)) + do (sleep-for 0 500) + finally do + (unless (car (ein:jupyter-server-conn-info ein:jupyter-server-buffer-name)) + (ein:log 'warn "Jupyter server failed to start, cancelling operation") + (ein:jupyter-server-stop t))) + (when (and (not no-login-p) (ein:jupyter-server-process)) + (unless login-callback + (setq login-callback #'ignore)) + (add-function :after login-callback + (apply-partially (lambda (proc* buffer url-or-port) + (ein:set-process-sentinel proc* url-or-port)) + proc)) + (ein:jupyter-server-login-and-open login-callback)))) ;;;###autoload (defalias 'ein:run 'ein:jupyter-server-start) @@ -227,12 +211,18 @@ there is no running server then no action will be taken. (remhash name check-hash)) (list (ein:$notebook-notebook-name nb) check-for-saved))))) (loop for x upfrom 0 by 1 - until (or (= (hash-table-count check-for-saved) 0) - (> x 1000000)) - do (sit-for 0.1))) + until (or (zerop (hash-table-count check-for-saved)) + (> x 20)) + do (sleep-for 0 500))) (mapc #'ein:notebook-close (ein:notebook-opened-notebooks)) + (loop repeat 10 + do (ein:query-gc-running-process-table) + when (zerop (hash-table-count ein:query-running-process-table)) + return t + do (sleep-for 0 500)) + ;; Both (quit-process) and (delete-process) leaked child kernels, so signal (ein:aif (ein:jupyter-server-process) (progn @@ -246,20 +236,4 @@ there is no running server then no action will be taken. (with-current-buffer ein:jupyter-server-buffer-name (write-region (point-min) (point-max) log))))) -(defun ein:jupyter-server-list--cmd (&optional args) - (append (list "notebook" - "list") - args)) - -(defun ein:jupyter-query-running-notebooks () - (with-temp-buffer - (let ((res (apply #'call-process (or *ein:last-jupyter-command* - ein:jupyter-default-server-command) - nil - t - nil - (ein:jupyter-server-list--cmd))) - (contents (rest (s-lines (buffer-string))))) - contents))) - (provide 'ein-jupyter) diff --git a/lisp/ein-jupyterhub.el b/lisp/ein-jupyterhub.el index 5e53c96..90b4a36 100644 --- a/lisp/ein-jupyterhub.el +++ b/lisp/ein-jupyterhub.el @@ -21,6 +21,7 @@ ;; along with ein-jupyter.el. If not, see . ;;; Commentary: +;; ;;; ;;; An interface to the Jupyterhub login and management API as described in ;;; http://jupyterhub.readthedocs.io/en/latest/api/index.html @@ -30,133 +31,178 @@ ;;; Code: (require 'ein-query) -(require 'ein-contents-api) +(require 'ein-websocket) +(require 'ein-notebooklist) -(defun ein:jupyterhub-api-url (url-or-port command &rest args) - (if args - (apply #'ein:url url-or-port "hub/api" command args) - (ein:url url-or-port "hub/api" command))) +(defvar *ein:jupyterhub-connections* (make-hash-table :test #'equal)) -(defun ein:jh-ask-url-or-port () - (let* ((url-or-port-list (mapcar (lambda (x) (format "%s" x)) - ein:url-or-port)) - (default (format "%s" (ein:default-url-or-port))) - (url-or-port - (completing-read (format "URL or port number (default %s): " default) - url-or-port-list - nil nil nil nil - default))) - (if (string-match "^[0-9]+$" url-or-port) - (string-to-number url-or-port) - url-or-port))) +(defstruct ein:$jh-conn + "Data representing a connection to a jupyterhub server." + url-or-port + version + user + token) -(defun ein:jupyterhub--do-connect (url-or-port user password) - (deferred:$ - (ein:query-deferred - (ein:jupyterhub-api-url url-or-port "/") - :type "GET" - :parser #'ein:json-read) - (deferred:nextc it - (lambda (response) - (when (and response (request-response-data response)) - (let ((conn (make-ein:$jh-conn :url (or (ein:get-response-redirect response) - url-or-port) - :version (plist-get (request-response-data response) :version)))) - conn)))) - (deferred:nextc it - (lambda (conn) - (ein:jupyterhub-login conn user password))) - (deferred:nextc it - (lambda (conn) - (unless conn - (error "Connection to Jupyterhub server at %s failed! Maybe you used the wrong URL?" url-or-port)) - (ein:jupyterhub-token-request conn))) - (deferred:nextc it - (lambda (conn) - (setf (gethash (ein:$jh-conn-url conn) *ein:jupyterhub-servers*) conn) - (ein:jupyterhub-start-server conn user))))) +(defstruct ein:$jh-user + "A jupyterhub user, per https://jupyterhub.readthedocs.io/en/latest/_static/rest-api/index.html#/definitions/User" + name + admin + groups + server + pending + last-activity) -(defun ein:jupyterhub-login (conn username password) - (deferred:$ - (ein:query-deferred - (ein:url (ein:$jh-conn-url conn) "hub/login") - :type "POST" - :parser #'ein:json-read - :data (format "username=%s&password=%s" username password) ;; (json-encode`((username . ,username) - ;; (password . , password))) - ) - (deferred:nextc it - (lambda (response) - (ein:log 'info "Login for user %s with response %s." username (request-response-status-code response)) - conn)))) +(defsubst ein:jupyterhub-user-path (url-or-port &rest paths) + "Goes from URL-OR-PORT/PATHS to URL-OR-PORT/user/someone/PATHS" + (let ((user-base (ein:aif (gethash url-or-port *ein:jupyterhub-connections*) + (ein:$jh-user-server (ein:$jh-conn-user it))))) + (apply #'ein:url url-or-port user-base paths))) -(defun ein:jupyterhub-token-request (conn) - (deferred:$ - (ein:query-deferred - (ein:jupyterhub-api-url (ein:$jh-conn-url conn) - "authorizations/token") - :type "POST" - :timeout ein:content-query-timeout - :parser #'ein:json-read) - (deferred:nextc it - (lambda (response) - (ein:log 'info "response-data: %s, %s" - (request-response-data response) - (cadr (request-response-data response))) ;; FIXME: Why doesn't plist-get work? - (unless (eql (request-response-status-code response) 403) - (setf (ein:$jh-conn-token conn) (cadr (request-response-data response)))) - conn)))) +(defsubst ein:jupyterhub-api-path (url-or-port &rest paths) + (apply #'ein:url url-or-port "hub/api" paths)) -(defun ein:jupyterhub-get-user (conn username) - (deferred:$ - (ein:query-deferred - (ein:jupyterhub-api-url (ein:$jh-conn-url conn) - "users" - username) - :type "GET" - :parser #'ein:json-read) - (deferred:nextc it - (lambda (response) - (let* ((data (request-response-data response)) - (user (make-ein:$jh-user :name (plist-get data :name) - :admin (plist-get data :admin) - :groups (plist-get data :groups) - :server (plist-get data :server) - :pending (plist-get data :pending) - :last-activity (plist-get data :last_activity)))) - (ein:log 'info "Jupyterhub: Found user: %s" user) - user))))) +(defun ein:jupyterhub--store-cookies (conn) + "Websockets use the url-cookie API" + (let* ((url-or-port (ein:$jh-conn-url-or-port conn)) + (parsed-url (url-generic-parse-url url-or-port)) + (host-port (if (url-port-if-non-default parsed-url) + (format "%s:%s" (url-host parsed-url) (url-port parsed-url)) + (url-host parsed-url))) + (securep (string= (url-type parsed-url) "https")) + (cookies (append + (request-cookie-alist (url-host parsed-url) "/hub/" securep) + (ein:aand (ein:$jh-conn-user conn) (ein:$jh-user-server it) + (request-cookie-alist (url-host parsed-url) it securep))))) + (dolist (c cookies) + (ein:websocket-store-cookie c host-port + (car (url-path-and-query parsed-url)) securep)))) -(defun ein:jupyterhub-start-server (conn username) - (deferred:$ - (ein:query-deferred - (ein:jupyterhub-api-url (ein:$jh-conn-url conn) - "users" - username - "server") - :type "POST" - :parser #'ein:json-read) - (deferred:nextc it - (lambda (response) - (ein:log 'info "Jupyterhub: Response status: %s" (request-response-status-code response)) - (case (request-response-status-code response) - ((201 400) - (ein:log 'info "Jupyterhub: Finding user: %s" username) - (ein:jupyterhub-get-user conn username))))) - (deferred:nextc it - (lambda (user) - (ein:log 'info "Jupyterhub: Found user? (%s)" user) - (when (ein:$jh-user-p user) - (setf (ein:$jh-conn-user conn) user) - (ein:log 'info "Jupyterhub: Opening notebook at %s: " (ein:$jh-conn-url conn)) - (ein:notebooklist-open* (ein:$jh-conn-url conn) nil nil #'pop-to-buffer)))))) +(defun* ein:jupyterhub--login-complete (dobj conn &key response &allow-other-keys) + (deferred:callback-post dobj (list conn response))) + +(defmacro ein:jupyterhub--add-header (header) + `(setq my-settings + (plist-put my-settings :headers + (append (plist-get my-settings :headers) (list ,header))))) + +(defmacro ein:jupyterhub-query (conn-key url cb cbargs &rest settings) + `(let ((my-settings (list ,@settings))) + (ein:and-let* ((conn (gethash ,conn-key *ein:jupyterhub-connections*))) + (ein:jupyterhub--add-header + (cons "Referer" (ein:url (ein:$jh-conn-url-or-port conn) "hub/login"))) + (ein:aif (ein:$jh-conn-token conn) + (ein:jupyterhub--add-header + (cons "Authorization" (format "token %s" it))))) + (apply #'ein:query-singleton-ajax + ,url ,url + :error + (lambda (&rest args) + (ein:log 'error "ein:jupyterhub-query--error (%s) %s (%s)" ,url + (request-response-status-code (plist-get args :response)) + (plist-get args :symbol-status))) + :complete + (lambda (&rest args) + (ein:log 'debug "ein:jupyterhub-query--complete (%s) %s (%s)" ,url + (request-response-status-code (plist-get args :response)) + (plist-get args :symbol-status))) + :success + (lambda (&rest args) + (apply ,cb (request-response-data (plist-get args :response)) ,cbargs)) + my-settings))) + +(defun ein:jupyterhub--receive-version (data url-or-port callback username password) + (let ((conn (make-ein:$jh-conn + :url-or-port url-or-port + :version (plist-get data :version)))) + (setf (gethash url-or-port *ein:jupyterhub-connections*) conn) + (ein:jupyterhub--query-login callback username password conn))) + +(defun ein:jupyterhub--receive-user (data callback username password conn iteration) + (let ((user (make-ein:$jh-user :name (plist-get data :name) + :admin (plist-get data :admin) + :groups (plist-get data :groups) + :server (plist-get data :server) + :pending (plist-get data :pending) + :last-activity (plist-get data :last_activity)))) + (setf (ein:$jh-conn-user conn) user) + (ein:jupyterhub--store-cookies conn) + (if (not (ein:$jh-user-server user)) + (if (<= iteration 0) + (ein:jupyterhub--query-token callback username password conn) + (ein:display-warning "jupyterhub cannot start single-user server" :error)) + (ein:notebooklist-open* + (ein:jupyterhub-user-path (ein:$jh-conn-url-or-port conn)) + nil nil callback)))) + +(defun ein:jupyterhub--receive-login (_data callback username password conn) + (ein:jupyterhub--store-cookies conn) + (ein:jupyterhub--query-user callback username password conn 0)) + +(defun ein:jupyterhub--receive-token (data callback username password conn) + (setf (ein:$jh-conn-token conn) (plist-get data :token)) + (ein:jupyterhub--query-server callback username password conn)) + +(defun ein:jupyterhub--receive-server (_data callback username password conn) + (ein:jupyterhub--query-user callback username password conn 1)) + +(defun ein:jupyterhub--query-token (callback username password conn) + (ein:jupyterhub-query + (ein:$jh-conn-url-or-port conn) + (ein:jupyterhub-api-path (ein:$jh-conn-url-or-port conn) + "authorizations/token") + #'ein:jupyterhub--receive-token + `(,callback ,username ,password ,conn) + :type "POST" + :data (json-encode `((:username . ,username) + (:password . ,password))) + :parser #'ein:json-read)) + +(defsubst ein:jupyterhub--query-server (callback username password conn) + (ein:jupyterhub-query + (ein:$jh-conn-url-or-port conn) + (ein:jupyterhub-api-path (ein:$jh-conn-url-or-port conn) + "users" username "server") + #'ein:jupyterhub--receive-server + `(,callback ,username ,password ,conn) + :type "POST" + :parser #'ein:json-read)) + +(defsubst ein:jupyterhub--query-user (callback username password conn iteration) + (ein:jupyterhub-query + (ein:$jh-conn-url-or-port conn) + (ein:jupyterhub-api-path (ein:$jh-conn-url-or-port conn) "users" username) + #'ein:jupyterhub--receive-user + `(,callback ,username ,password ,conn ,iteration) + :type "GET" + :parser #'ein:json-read)) + +(defsubst ein:jupyterhub--query-login (callback username password conn) + (ein:jupyterhub-query + (ein:$jh-conn-url-or-port conn) + (ein:url (ein:$jh-conn-url-or-port conn) "hub/login") + #'ein:jupyterhub--receive-login + `(,callback ,username ,password ,conn) + ;; :type "POST" ;; no type here else redirect will use POST + :parser #'ignore + :data `(("username" . ,username) + ("password" . ,password)))) + +(defsubst ein:jupyterhub--query-version (url-or-port callback username password) + (ein:jupyterhub-query + url-or-port + (ein:jupyterhub-api-path url-or-port) + #'ein:jupyterhub--receive-version + `(,url-or-port ,callback ,username ,password) + :type "GET" + :parser #'ein:json-read)) ;;;###autoload -(defun ein:jupyterhub-connect (url user password) - "Log on to a jupyterhub server using PAM authentication. Requires jupyterhub version 0.8 or greater." - (interactive (list (ein:jh-ask-url-or-port) - (read-string "User: ") - (read-passwd "Password: "))) - (ein:jupyterhub--do-connect url user password)) +(defun ein:jupyterhub-connect (url-or-port username password callback) + "Log on to a jupyterhub server using PAM authentication. Requires jupyterhub version 0.8 or greater. CALLBACK takes two arguments, the resulting buffer and the singleuser url-or-port" + (interactive (let ((url-or-port (ein:notebooklist-ask-url-or-port)) + (pam-plist (ein:notebooklist-ask-user-pw-pair "User" "Password"))) + (loop for (user pw) on pam-plist by (function cddr) + return (list url-or-port (symbol-name user) pw (lambda (buffer _url-or-port) (pop-to-buffer buffer)))))) + (ein:jupyterhub--query-version url-or-port callback username password)) (provide 'ein-jupyterhub) diff --git a/lisp/ein-kernel.el b/lisp/ein-kernel.el index ef553d4..f8c36e1 100644 --- a/lisp/ein-kernel.el +++ b/lisp/ein-kernel.el @@ -240,31 +240,17 @@ CALLBACK with arity 0 (e.g., execute cell now that we're reconnected)" callback*)))) callback))) -(defun ein:kernel--ws-url (url-or-port &optional securep) - "Use `ein:$kernel-url-or-port' if BASE_URL is an empty string. -See: https://github.com/ipython/ipython/pull/3307" - (let* ((base-url url-or-port) - (url-or-port (ein:jupyterhub-correct-query-url-maybe url-or-port)) - (protocol (if (or securep - (and (stringp url-or-port) - (string-match "^https://" url-or-port))) - "wss" - "ws"))) - (if (integerp url-or-port) - (format "%s://127.0.0.1:%s" protocol url-or-port) - (let* ((url (if (string-match "^https?://" url-or-port) - url-or-port - (format "http://%s" url-or-port))) - (parsed-url (url-generic-parse-url url))) - (if (ein:jupyterhub-url-p base-url) - (ein:trim-right (format "%s://%s:%s%s" - protocol - (url-host parsed-url) - (url-port parsed-url) - (url-filename parsed-url)) - "/") - (format "%s://%s:%s" protocol (url-host parsed-url) (url-port parsed-url))))))) +(defun ein:kernel--ws-url (url-or-port) + "Assuming URL-OR-PORT already normalized by `ein:url' +See https://github.com/ipython/ipython/pull/3307" + (let* ((parsed-url (url-generic-parse-url url-or-port)) + (protocol (if (string= (url-type parsed-url) "https") "wss" "ws"))) + (format "%s://%s:%s%s" + protocol + (url-host parsed-url) + (url-port parsed-url) + (url-filename parsed-url)))) (defun ein:kernel-send-cookie (channel host) ;; cookie can be an empty string for IPython server with no password, diff --git a/lisp/ein-notebook.el b/lisp/ein-notebook.el index 5882792..be10665 100644 --- a/lisp/ein-notebook.el +++ b/lisp/ein-notebook.el @@ -243,9 +243,7 @@ the jupyter server dies and restarted on a different port. If you have enabled token or password security on server running at the new url/port, then please be aware that this new url-port combo must match exactly these url/port you used format -`ein:notebooklist-login'. For example, as far as Emacs and -jupyter are concerned, 'localhost:8888' and '127.0.0.1:8888' are -*not* the same URL." +`ein:notebooklist-login'." (interactive (list (ein:notebooklist-ask-url-or-port) (ein:get-notebook-or-error))) @@ -303,7 +301,7 @@ will be updated with kernel's cwd." (defun ein:notebook-open--decorate-callback (notebook existing pending-clear callback) "In addition to CALLBACK, also clear the pending semaphore, pop-to-buffer the new notebook, and save to disk the kernelspec metadata." (apply-partially (lambda (notebook* created callback* pending-clear*) - (funcall pending-clear*) + (funcall pending-clear* nil) (with-current-buffer (ein:notebook-buffer notebook*) (ein:worksheet-focus-cell)) (pop-to-buffer (ein:notebook-buffer notebook*)) @@ -338,7 +336,7 @@ notebook buffer. Let's warn for now to see who is doing this. (ein:notebooklist-parse-nbpath (ein:notebooklist-ask-path "notebook"))) (let* ((pending-key (cons url-or-port path)) (pending-p (gethash pending-key *ein:notebook--pending-query*)) - (pending-clear (apply-partially (lambda (pending-key*) + (pending-clear (apply-partially (lambda (pending-key* _contents) (remhash pending-key* *ein:notebook--pending-query*)) pending-key)) diff --git a/lisp/ein-notebooklist.el b/lisp/ein-notebooklist.el index 7e51402..5c604b3 100644 --- a/lisp/ein-notebooklist.el +++ b/lisp/ein-notebooklist.el @@ -40,6 +40,8 @@ (require 'dash) (require 'ido) +(autoload 'ein:jupyterhub-connect "ein-jupyterhub") + (defcustom ein:notebooklist-login-timeout (truncate (* 6.3 1000)) "Timeout in milliseconds for logging into server" :group 'ein @@ -154,7 +156,7 @@ This function adds NBLIST to `ein:notebooklist-map'." ((>= version 3) "api/contents")))) (ein:url url-or-port base-path path))) -(defun ein:notebooklist-proc--sentinel (url-or-port process event) +(defun ein:notebooklist-sentinel (url-or-port process event) "Remove URL-OR-PORT from ein:notebooklist-map when PROCESS dies" (when (not (string= "open" (substring event 0 4))) (ein:log 'info "Process %s %s %s" @@ -164,6 +166,7 @@ This function adds NBLIST to `ein:notebooklist-map'." (ein:notebooklist-list-remove url-or-port))) (defun ein:notebooklist-get-buffer (url-or-port) + (assert url-or-port) (get-buffer-create (format ein:notebooklist-buffer-name-template url-or-port))) @@ -234,14 +237,17 @@ port the instance is running on." (defun ein:notebooklist-open* (url-or-port &optional path resync callback errback) "The main entry to server at URL-OR-PORT. Users should not directly call this, but instead `ein:notebooklist-login'. -PATH is specifying directory from file navigation. PATH is empty on login. RESYNC is requery server attributes such as ipython version and kernelspecs. CALLBACK takes one argument, the resulting buffer. ERRBACK takes one argument, the resulting buffer. +PATH is specifying directory from file navigation. PATH is empty on login. RESYNC is requery server attributes such as ipython version and kernelspecs. CALLBACK takes two arguments, the resulting buffer and URL-OR-PORT. ERRBACK takes one argument, the resulting buffer. + +TODO: going to maintain jupyterhub hooks here " (unless path (setq path "")) (setq url-or-port (ein:url url-or-port)) ;; should work towards not needing this (ein:subpackages-load) (lexical-let* ((url-or-port url-or-port) (path path) - (success (apply-partially #'ein:notebooklist-open--finish url-or-port callback)) + (success (apply-partially #'ein:notebooklist-open--finish + url-or-port callback)) (failure errback)) (if (or resync (not (ein:notebooklist-list-get url-or-port))) (deferred:$ @@ -315,7 +321,7 @@ automatically be called during calls to `ein:notebooklist-open`." (setq ein:notebooklist--keepalive-timer nil)) (defun ein:notebooklist-open--finish (url-or-port callback content) - "Called via `ein:notebooklist-open'." + "Called via `ein:notebooklist-open*'." (let ((path (ein:$content-path content)) (nb-version (ein:$content-notebook-version content)) (data (ein:$content-raw-content content))) @@ -336,8 +342,8 @@ automatically be called during calls to `ein:notebooklist-open`." (when ein:enable-keepalive (ein:notebooklist-enable-keepalive url-or-port)) (when callback - (funcall callback (current-buffer))) - (current-buffer))))) + (funcall callback (current-buffer) url-or-port))) + (current-buffer)))) (defun* ein:notebooklist-open-error (url-or-port path &key error-thrown @@ -435,7 +441,8 @@ TODO - New and open should be separate, and we should flag an exception if we tr (ein:log 'error "Failed to open new notebook (error: %S). \ You may find the new one in the notebook list." error) - (ein:notebooklist-open* url-or-port nil nil #'pop-to-buffer)) + (ein:notebooklist-open* url-or-port nil nil (lambda (buffer url-or-port) + (pop-to-buffer buffer)))) ;;;###autoload (defun ein:notebooklist-new-notebook-with-name (name kernelspec url-or-port &optional path) @@ -584,7 +591,9 @@ You may find the new one in the notebook list." error) (widget-create 'link :notify (lambda (&rest ignore) - (ein:notebooklist-open* url-or-port path nil #'pop-to-buffer)) + (ein:notebooklist-open* url-or-port path nil + (lambda (buffer url-or-port) + (pop-to-buffer buffer)))) name))) (widget-insert " |\n\n")) @@ -685,7 +694,11 @@ You may find the new one in the notebook list." error) (lambda (&rest ignore) ;; each directory creates a whole new notebooklist (ein:notebooklist-open* url-or-port - (concat (file-name-as-directory (ein:$notebooklist-path ein:%notebooklist%)) name) nil #'pop-to-buffer))) + (concat (file-name-as-directory + (ein:$notebooklist-path ein:%notebooklist%)) + name) + nil + (lambda (buffer url-or-port) (pop-to-buffer buffer))))) "Dir") (widget-insert " : " name) (widget-insert "\n")) @@ -771,16 +784,21 @@ Notebook list data is passed via the buffer local variable for url-or-port = (ein:$notebooklist-url-or-port nblist) collect (loop for content in (ein:content-need-hierarchy url-or-port) - when (or (null content-type) (string= (ein:$content-type content) content-type)) + when (or (null content-type) + (string= (ein:$content-type content) content-type)) collect (ein:url url-or-port (ein:$content-path content)))))) -(defsubst ein:notebooklist-parse-nbpath (nbpath) +(defun ein:notebooklist-parse-nbpath (nbpath) "Return `(,url-or-port ,path) from URL-OR-PORT/PATH" - (let* ((parsed (url-generic-parse-url nbpath)) - (path (url-filename parsed))) - (list (substring nbpath 0 (- (length nbpath) (length path))) - (substring path 1)))) + (loop for url-or-port in (ein:hash-keys ein:notebooklist-map) + if (search url-or-port nbpath :end2 (length url-or-port)) + return (list (substring nbpath 0 (length url-or-port)) + (substring nbpath (1+ (length url-or-port)))) + end + finally (ein:display-warning + (format "%s not among: %s" nbpath (ein:hash-keys ein:notebooklist-map)) + :error))) (defsubst ein:notebooklist-ask-path (&optional content-type) (ido-completing-read (format "Open %s: " content-type) @@ -807,40 +825,35 @@ in order to make this code work. See also: `ein:connect-to-default-notebook', `ein:connect-default-notebook'." - (ein:notebooklist-open* url-or-port nil)) + (ein:notebooklist-open* url-or-port)) ;;; Login (defun ein:notebooklist-login--iteration (url-or-port callback errback token iteration response-status) - "Called from `ein:notebooklist-login'." (ein:log 'debug "Login attempt #%d in response to %s from %s." iteration response-status url-or-port) (unless callback (setq callback #'ignore)) (unless errback (setq errback #'ignore)) - (lexical-let (done-p) - (add-function :after callback (lambda (&rest ignore) (setq done-p t))) - (add-function :after errback (lambda (&rest ignore) (setq done-p t))) - (ein:query-singleton-ajax - (list 'notebooklist-login--iteration url-or-port) - (ein:url url-or-port "login") - ;; do not use :type "POST" here (see git history) - :timeout ein:notebooklist-login-timeout - :data (if token (concat "password=" (url-hexify-string token))) - :parser #'ein:notebooklist-login--parser - :complete (apply-partially #'ein:notebooklist-login--complete url-or-port) - :error (apply-partially #'ein:notebooklist-login--error url-or-port token callback errback iteration) - :success (apply-partially #'ein:notebooklist-login--success url-or-port callback errback token iteration)) - (unless noninteractive - (with-local-quit - (loop until done-p - do (sleep-for 0 450)))))) + (ein:query-singleton-ajax + (list 'notebooklist-login--iteration url-or-port) + (ein:url url-or-port "login") + ;; do not use :type "POST" here (see git history) + :timeout ein:notebooklist-login-timeout + :data (if token (concat "password=" (url-hexify-string token))) + :parser #'ein:notebooklist-login--parser + :complete (apply-partially #'ein:notebooklist-login--complete url-or-port) + :error (apply-partially #'ein:notebooklist-login--error url-or-port token + callback errback iteration) + :success (apply-partially #'ein:notebooklist-login--success url-or-port callback + errback token iteration))) ;;;###autoload (defun ein:notebooklist-open (url-or-port callback) "This is now an alias for ein:notebooklist-login" - (interactive `(,(ein:notebooklist-ask-url-or-port) ,#'pop-to-buffer)) + (interactive `(,(ein:notebooklist-ask-url-or-port) + ,(lambda (buffer url-or-port) (pop-to-buffer buffer)))) (ein:notebooklist-login url-or-port callback)) (make-obsolete 'ein:notebooklist-open 'ein:notebooklist-login "0.14.2") @@ -848,20 +861,21 @@ See also: ;;;###autoload (defalias 'ein:login 'ein:notebooklist-login) -(defun ein:notebooklist-ask-one-cookie () - "If we need more than one cookie, we first need to ask for how many. Returns list of name and content." - (plist-put nil (intern (read-no-blanks-input "Cookie name: ")) - (read-no-blanks-input "Cookie content: "))) +(defun ein:notebooklist-ask-user-pw-pair (user-prompt pw-prompt) + "Currently used for cookie and jupyterhub additional inputs. If we need more than one cookie, we first need to ask for how many. Returns list of name and content." + (plist-put nil (intern (read-no-blanks-input (format "%s: " user-prompt))) + (read-no-blanks-input (format "%s: " pw-prompt)))) ;;;###autoload (defun ein:notebooklist-login (url-or-port callback &optional cookie-plist) "Deal with security before main entry of ein:notebooklist-open*. -CALLBACK takes one argument, the buffer created by ein:notebooklist-open--success." +CALLBACK takes two arguments, the buffer created by ein:notebooklist-open--success +and the url-or-port argument of ein:notebooklist-open*." (interactive `(,(ein:notebooklist-ask-url-or-port) - ,#'pop-to-buffer - ,(if current-prefix-arg (ein:notebooklist-ask-one-cookie)))) - (unless callback (setq callback (lambda (buffer)))) + ,(lambda (buffer url-or-port) (pop-to-buffer buffer)) + ,(if current-prefix-arg (ein:notebooklist-ask-user-pw-pair "Cookie name" "Cookie content")))) + (unless callback (setq callback (lambda (buffer url-or-port)))) (when cookie-plist (let* ((parsed-url (url-generic-parse-url (file-name-as-directory url-or-port))) @@ -901,12 +915,18 @@ CALLBACK takes one argument, the buffer created by ein:notebooklist-open--succes &allow-other-keys &aux (response-status (request-response-status-code response))) - (if (plist-get data :bad-page) - (if (>= iteration 0) - (ein:notebooklist-login--error-1 url-or-port errback) - (setq token (read-passwd (format "Password for %s: " url-or-port))) - (ein:notebooklist-login--iteration url-or-port callback errback token (1+ iteration) response-status)) - (ein:notebooklist-login--success-1 url-or-port callback errback))) + (cond ((plist-get data :bad-page) + (if (>= iteration 0) + (ein:notebooklist-login--error-1 url-or-port errback) + (setq token (read-passwd (format "Password for %s: " url-or-port))) + (ein:notebooklist-login--iteration url-or-port callback errback token (1+ iteration) response-status))) + ((request-response-header response "x-jupyterhub-version") + (let ((pam-plist (ein:notebooklist-ask-user-pw-pair "User" "Password"))) + (destructuring-bind (user pw) + (loop for (user pw) on pam-plist by (function cddr) + return (list (symbol-name user) pw)) + (ein:jupyterhub-connect url-or-port user pw callback)))) + (t (ein:notebooklist-login--success-1 url-or-port callback errback)))) (defun* ein:notebooklist-login--error (url-or-port token callback errback iteration &key @@ -944,7 +964,7 @@ on all the notebooks opened from the current notebooklist." (open-nb (ein:notebook-opened-notebooks #'(lambda (nb) (equal (ein:$notebook-url-or-port nb) (ein:$notebooklist-url-or-port current-nblist)))))) - (ein:notebooklist-open* new-url-or-port nil) + (ein:notebooklist-open* new-url-or-port) (loop for x upfrom 0 by 1 until (or (get-buffer (format ein:notebooklist-buffer-name-template new-url-or-port)) (= x 100)) @@ -952,7 +972,8 @@ on all the notebooks opened from the current notebooklist." (dolist (nb open-nb) (ein:notebook-update-url-or-port new-url-or-port nb)) (kill-buffer (ein:notebooklist-get-buffer old-url)) - (ein:notebooklist-open* new-url-or-port nil nil #'pop-to-buffer))) + (ein:notebooklist-open* new-url-or-port nil nil (lambda (buffer url-or-port) + (pop-to-buffer buffer))))) (defun ein:notebooklist-change-url-port--deferred (new-url-or-port) (lexical-let* ((current-nblist ein:%notebooklist%) @@ -965,7 +986,7 @@ on all the notebooks opened from the current notebooklist." (deferred:$ (deferred:next (lambda () - (ein:notebooklist-open* new-url-or-port nil) + (ein:notebooklist-open* new-url-or-port) (loop until (get-buffer (format ein:notebooklist-buffer-name-template new-url-or-port)) do (sit-for 0.1)))) (deferred:nextc it @@ -975,7 +996,8 @@ on all the notebooks opened from the current notebooklist." (deferred:nextc it (lambda () (kill-buffer (ein:notebooklist-get-buffer old-url)) - (ein:notebooklist-open* new-url-or-port nil nil #'pop-to-buffer)))))) + (ein:notebooklist-open* new-url-or-port nil nil (lambda (buffer url-or-port) + (pop-to-buffer buffer)))))))) ;;; Generic getter diff --git a/lisp/ein-pager.el b/lisp/ein-pager.el index fb0398e..5c8daab 100644 --- a/lisp/ein-pager.el +++ b/lisp/ein-pager.el @@ -30,7 +30,6 @@ (require 'ein-core) (require 'ein-events) (require 'view) -(require 'ess-help nil t) ;; FIXME: Make a class with `:get-notebook-name' slot like `ein:worksheet' diff --git a/lisp/ein-pkg.el b/lisp/ein-pkg.el index 2431d75..32f9a71 100644 --- a/lisp/ein-pkg.el +++ b/lisp/ein-pkg.el @@ -5,7 +5,6 @@ (auto-complete "1.4.0") (request "0.3") (deferred "0.5") - (request-deferred "0.2.0") (cl-generic "0.3") (dash "2.13.0") (s "1.11.0") diff --git a/lisp/ein-process.el b/lisp/ein-process.el index a3b9ab4..d2dd472 100644 --- a/lisp/ein-process.el +++ b/lisp/ein-process.el @@ -136,22 +136,6 @@ :dir (directory-file-name notebook_dir)) ein:%processes%)))) -(defun ein:process-ps-refresh-processes () - "Can delete this. It pokes around unix ps when it's far better to use `jupyter notebook list'" - (loop for pid in (list-system-processes) - for attrs = (process-attributes pid) - for args = (alist-get 'args attrs) - with seen = (mapcar #'ein:$process-pid (ein:hash-vals ein:%processes%)) - if (and (null (member pid seen)) - (string-match ein:process-jupyter-regexp (alist-get 'comm attrs))) - do (ein:and-let* ((dir (ein:process-divine-dir pid args)) - (port (ein:process-divine-port pid args)) - (ip (ein:process-divine-ip pid args))) - (puthash dir (make-ein:$process :pid pid - :url (ein:url (format "http://%s:%s" ip port)) :dir dir) - ein:%processes%)) - end)) - (defun ein:process-dir-match (filename) "Return ein:process whose directory is prefix of FILENAME." (loop for dir in (ein:hash-keys ein:%processes%) @@ -173,23 +157,22 @@ (if proc (let* ((url-or-port (ein:process-url-or-port proc)) (path (ein:process-path proc filename)) - (callback1 (apply-partially (lambda (url-or-port* path* callback* buffer) + (callback2 (apply-partially (lambda (path* callback* buffer url-or-port) (ein:notebook-open - url-or-port* path* nil callback*)) - url-or-port path callback))) + url-or-port path* nil callback*)) + path callback))) (if (ein:notebooklist-list-get url-or-port) (ein:notebook-open url-or-port path nil callback) - (ein:notebooklist-login url-or-port callback1))) + (ein:notebooklist-login url-or-port callback2))) (let* ((nbdir (read-directory-name "Notebook directory: " (ein:process-suitable-notebook-dir filename))) (path (subseq filename (length (file-name-as-directory nbdir)))) - (callback1 (apply-partially (lambda (path* callback* buffer) + (callback2 (apply-partially (lambda (path* callback* buffer url-or-port) (pop-to-buffer buffer) - (ein:notebook-open - (car (ein:jupyter-server-conn-info)) - path* nil callback*)) + (ein:notebook-open url-or-port + path* nil callback*)) path callback))) - (ein:jupyter-server-start (executable-find ein:jupyter-default-server-command) nbdir nil callback1))))) + (ein:jupyter-server-start (executable-find ein:jupyter-default-server-command) nbdir nil callback2))))) (defun ein:process-open-notebook (&optional filename buffer-callback) "When FILENAME is unspecified the variable `buffer-file-name' diff --git a/lisp/ein-query.el b/lisp/ein-query.el index 26d76b6..d4245ef 100644 --- a/lisp/ein-query.el +++ b/lisp/ein-query.el @@ -27,7 +27,6 @@ (eval-when-compile (require 'cl)) (require 'request) -(require 'request-deferred) (require 'url) (require 'ein-core) @@ -71,79 +70,18 @@ aborts). Instead you will see Race! in debug messages. :group 'ein) -;;; Jupyterhub -(defvar *ein:jupyterhub-servers* (make-hash-table :test #'equal)) - -(defstruct ein:$jh-conn - "Data representing a connection to a jupyterhub server." - url - version - user - token) - -(defstruct ein:$jh-user - "A jupyterhub user, per https://jupyterhub.readthedocs.io/en/latest/_static/rest-api/index.html#/definitions/User." - name - admin - groups - server - pending - last-activity) - - - -(defun ein:get-jh-conn (url) - (gethash url *ein:jupyterhub-servers*)) - -(defun ein:reset-jh-servers () - (setq *ein:jupyterhub-servers* (make-hash-table :test #'equal))) - -(defun ein:jupyterhub-url-p (url) - "Does URL reference a jupyterhub server? If so then return the -connection structure representing the server." - (let ((parsed (url-generic-parse-url url))) - (or (gethash (format "http://%s:%s" (url-host parsed) (url-port parsed)) - *ein:jupyterhub-servers*) - (gethash (format "https://%s:%s" (url-host parsed) (url-port parsed)) - *ein:jupyterhub-servers*)))) - -(defun ein:jupyterhub-correct-query-url-maybe (url-or-port) - (let* ((parsed-url (url-generic-parse-url url-or-port)) - (hostport (format "http://%s:%s" (url-host parsed-url) (url-port parsed-url))) - (command (url-filename parsed-url))) - (ein:aif (ein:jupyterhub-url-p hostport) - (let ((user-server-path (ein:$jh-user-server (ein:$jh-conn-user it)))) - (ein:url hostport - user-server-path - command)) - url-or-port))) - ;;; Functions (defvar ein:query-running-process-table (make-hash-table :test 'equal)) (defun ein:query-prepare-header (url settings &optional securep) - "Ensure that REST calls to the jupyter server have the correct -_xsrf argument." + "Ensure that REST calls to the jupyter server have the correct _xsrf argument." (let* ((parsed-url (url-generic-parse-url url)) (cookies (request-cookie-alist (url-host parsed-url) "/" securep))) (ein:aif (assoc-string "_xsrf" cookies) (setq settings (plist-put settings :headers (append (plist-get settings :headers) (list (cons "X-XSRFTOKEN" (cdr it))))))) - (ein:aif (ein:jupyterhub-url-p (format "http://%s:%s" (url-host parsed-url) (url-port parsed-url))) - (progn - (unless (string-equal (ein:$jh-conn-url it) - (ein:url (ein:$jh-conn-url it) "hub/login")) - (setq settings (plist-put settings :headers (append (plist-get settings :headers) - (list (cons "Referer" - (ein:url (ein:$jh-conn-url it) - "hub/login"))))))) - (when (ein:$jh-conn-token it) - (setq settings (plist-put settings :headers (append (plist-get settings :headers) - (list (cons "Authorization" - (format "token %s" - (ein:$jh-conn-token it)))))))))) settings)) (defcustom ein:max-simultaneous-queries 100 @@ -192,20 +130,11 @@ KEY, then call `request' with URL and SETTINGS. KEY is compared by ;; This seems to result in clobbered cookie jars ;;(request-abort it) ; This will run callbacks (ein:log 'debug "Race! %s %s" key (request-response-data it)))) - (let ((response (apply #'request (url-encode-url (ein:jupyterhub-correct-query-url-maybe url)) + (let ((response (apply #'request (url-encode-url url) (ein:query-prepare-header url settings)))) (puthash key response ein:query-running-process-table) response))) -(defun* ein:query-deferred (url &rest settings - &key - (timeout ein:query-timeout) - &allow-other-keys) - "Appears to be used by ein-jupyterhub only" - (ein:query-enforce-curl) - (apply #'request-deferred (url-encode-url url) - (ein:query-prepare-header url settings))) - (defun ein:query-gc-running-process-table () "Garbage collect dead processes in `ein:query-running-process-table'." (maphash diff --git a/lisp/ein-smartrep.el b/lisp/ein-smartrep.el index 75ddf39..4c3fb94 100644 --- a/lisp/ein-smartrep.el +++ b/lisp/ein-smartrep.el @@ -25,10 +25,9 @@ ;;; Code: -(require 'smartrep nil t) (require 'ein-notebook) -(autoload 'smartrep-define-key "smartrep") +(declare-function smartrep-define-key "smartrep") (defcustom ein:smartrep-notebook-mode-alist '(("C-t" . ein:worksheet-toggle-cell-type) diff --git a/lisp/ein-utils.el b/lisp/ein-utils.el index b7abc9e..0fb7ae0 100644 --- a/lisp/ein-utils.el +++ b/lisp/ein-utils.el @@ -207,7 +207,8 @@ at point, i.e. any word before then \"(\", if it is present." (when (null (url-host parsed-url)) (setq url-or-port (concat "https://" url-or-port)) (setq parsed-url (url-generic-parse-url url-or-port))) - (when (string= (url-host parsed-url) "localhost") + (when (or (string= (url-host parsed-url) "localhost") + (string= (url-host parsed-url) "")) (setf (url-host parsed-url) ein:url-localhost)) (directory-file-name (concat (file-name-as-directory (url-recreate-url parsed-url)) (apply #'ein:glom-paths paths)))))) diff --git a/lisp/ein-websocket.el b/lisp/ein-websocket.el index 19bda9f..bb4a03f 100644 --- a/lisp/ein-websocket.el +++ b/lisp/ein-websocket.el @@ -61,30 +61,22 @@ name value)))) -;;(advice-add 'request--netscape-cookie-parse :around #'fix-request-netscape-cookie-parse) +(defsubst ein:websocket-store-cookie (c host-port url-filename securep) + (url-cookie-store (car c) (cdr c) nil host-port url-filename securep)) -;; Websocket gets its cookies using the url-cookie API, so we need to copy over -;; any cookies that are made and stored during the contents API calls via -;; emacs-request. +;;(advice-add 'request--netscape-cookie-parse :around #'fix-request-netscape-cookie-parse) (defun ein:websocket--prepare-cookies (url) - (let* ((jh-conn (ein:jupyterhub-url-p url)) - (parsed-url (url-generic-parse-url url)) + "Websocket gets its cookies using the url-cookie API, so we need to copy over + any cookies that are made and stored during the contents API calls via + emacs-request." + (let* ((parsed-url (url-generic-parse-url url)) (host-port (if (url-port-if-non-default parsed-url) (format "%s:%s" (url-host parsed-url) (url-port parsed-url)) (url-host parsed-url))) (securep (string-match "^wss://" url)) - (http-only-cookies (request-cookie-alist (concat "#HttpOnly_" (url-host (url-generic-parse-url url))) "/" securep)) ;; Current version of Jupyter store cookies as HttpOnly) - (cookies (request-cookie-alist (url-host (url-generic-parse-url url)) "/" securep)) - (hub-cookies (request-cookie-alist (url-host (url-generic-parse-url url)) "/hub/" securep)) - (user-cookies (and jh-conn - (request-cookie-alist - (url-host (url-generic-parse-url url)) - (ein:$jh-user-server (ein:$jh-conn-user jh-conn)) - securep)))) - (when (or cookies http-only-cookies hub-cookies user-cookies) - (ein:log 'debug "EIN:WEBSOCKET--PREPARE-COOKIES Storing cookies in prep for opening websocket (%s)" cookies) - (dolist (c (append cookies http-only-cookies hub-cookies user-cookies)) - (url-cookie-store (car c) (cdr c) nil host-port (car (url-path-and-query parsed-url)) securep))))) + (cookies (request-cookie-alist (url-host parsed-url) "/" securep))) + (dolist (c cookies) + (ein:websocket-store-cookie c host-port (car (url-path-and-query parsed-url)) securep)))) (defun ein:websocket (url kernel on-message on-close on-open) (ein:websocket--prepare-cookies url) @@ -99,7 +91,6 @@ (setf (websocket-client-data ws) websocket) websocket)) - (defun ein:websocket-open-p (websocket) (eql (websocket-ready-state (ein:$websocket-ws websocket)) 'open)) diff --git a/test/ein-testing.el b/test/ein-testing.el index 4e59c36..a8fa7a2 100644 --- a/test/ein-testing.el +++ b/test/ein-testing.el @@ -64,15 +64,16 @@ if I call this between links in a deferred chain. Adding a flush-queue." nil ms interval t)) (defun ein:testing-make-directory-level (parent current-depth width depth) - (f-touch (concat (file-name-as-directory parent) "foo.txt")) - (f-touch (concat (file-name-as-directory parent) "bar.ipynb")) - (f-write-text "{ + (let ((write-region-inhibit-sync nil)) + (f-touch (concat (file-name-as-directory parent) "foo.txt")) + (f-touch (concat (file-name-as-directory parent) "bar.ipynb")) + (f-write-text "{ \"cells\": [], \"metadata\": {}, \"nbformat\": 4, \"nbformat_minor\": 2 } -" 'utf-8 (concat (file-name-as-directory parent) "bar.ipynb")) +" 'utf-8 (concat (file-name-as-directory parent) "bar.ipynb"))) (if (< current-depth depth) (loop for w from 1 to width for dir = (concat (file-name-as-directory parent) (number-to-string w)) diff --git a/test/test-ein-utils.el b/test/test-ein-utils.el index 3d22aa7..aef289a 100644 --- a/test/test-ein-utils.el +++ b/test/test-ein-utils.el @@ -7,6 +7,7 @@ (ert-deftest ein-url-simple () (should (null (ein:url nil))) (should (equal (ein:url 8888) "http://127.0.0.1:8888")) + (should (equal (ein:url "http://:8000") "http://127.0.0.1:8000")) (should (equal (ein:url "http://localhost") "http://127.0.0.1")) (should (equal (ein:url "https://localhost:8888") "https://127.0.0.1:8888")) (should (equal (ein:url "http://localhost:8000" "" "" "" "Untitled.ipynb") "http://127.0.0.1:8000/Untitled.ipynb"))