Fix jupyter-tunnel-connection

* .travis.yml: Add setup for testing SSH components.

* jupyter-base.el (jupyter-tunnel-connection): Document behavior more
  completely.  Refactor for readability. Ensure that the :ip field of the
  returned property list is set to 127.0.0.1 since it represents the connection
  info of the remote system but with local ports and should not have a remote
  IP address.

* test/jupyter-test.el (jupyter-tunnel-connection): New test.

* test/test-helper.el (jupyter-test-with-kernel): New macro.

.travis.yml

@@ -2,6 +2,10 @@
 sudo: required
 dist: trusty
 language: nix
+addons:
+  apt:
+    packages:
+    - openssh-server
 matrix:
   # Report build failure/success before allowed failures complete
   fast_finish: true
@@ -37,6 +41,10 @@ before_script:
 script:
   - export PATH=$HOME/.cask/bin:$PATH
   - cd $TRAVIS_BUILD_DIR
+  # Ensure ssh does not prompt for anything during tests
+  - ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
+  - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+  - ssh -o StrictHostKeyChecking=no localhost exit
   - make dev
   - make compile
   - make test

jupyter-base.el

@@ -491,20 +491,30 @@ following fields:
    ;; seconds
    "sleep 60"))
 
-(defun jupyter-tunnel-connection (conn-file &optional server)
+(defun jupyter-tunnel-connection (conn-file &optional remote-host)
-  "Forward local ports to the remote ports in CONN-FILE.
+  "Forward local ports to the remote ports specified in CONN-FILE.
-CONN-FILE is the path to a Jupyter connection file, SERVER is the
+CONN-FILE is the path to a Jupyter connection file, the traffic
-host that the kernel connection in CONN-FILE is located.  Return a
+on the local ports will be directed to the remote ports listening
-copy of the connection plist in CONN-FILE, but with the ports
+at the ip specified in CONN-FILE on the remote network accessed
-replaced by the local ports used for the forwarding.
+via REMOTE-HOST, an SSH host.
 
-If CONN-FILE is a `tramp' file name, the SERVER argument will be
+Return a copy of the connection info. in CONN-FILE as a property
-ignored and the host will be extracted from the information
+list with the port fields (:iopub_port, :shell_port, ...)
-contained in the file name.
+replaced by the local ports used for the tunneling and the :ip
+field equal to \"127.0.0.1\".
 
-Note only SSH tunnels are currently supported."
+If CONN-FILE is a remote file, the REMOTE-HOST argument is
-  (catch 'no-tunnels
+ignored.  Instead, the remote host information is obtained from
-    (let ((conn-info (jupyter-read-plist conn-file)))
+information contained in the remote file name.
+
+If CONN-FILE is a remote file in a Docker container, i.e. it has
+a file name prefixed with /docker:, do not tunnel any ports.
+Assume the ports in CONN-FILE can also be used on localhost for
+connecting to those same ports in the container.  That is, assume
+the container was launched using a --publish argument like
+\"--publish 1234:1234\".  In this case, only the :ip field is
+replaced and set to \"127.0.0.1\"."
+  (catch 'no-tunnels-when-docker-conn-file
     (when (and (file-remote-p conn-file)
                (functionp 'tramp-dissect-file-name))
       (pcase-let (((cl-struct tramp-file-name method user host)
@@ -513,26 +523,31 @@ Note only SSH tunnels are currently supported."
           ;; TODO: Document this in the README along with the fact that
           ;; connection files can use /ssh: TRAMP files.
           ("docker"
-             ;; Assume docker is using the -p argument to publish its exposed
+           (throw 'no-tunnels-when-docker-conn-file
-             ;; ports to the localhost.  The ports used in the container should
+                  (let ((conn-info (jupyter-read-plist conn-file)))
-             ;; be the same ports accessible on the local host.  For example, if
+                    (plist-put conn-info :ip "127.0.0.1")
-             ;; the shell port is on 1234 in the container, the published port
+                    conn-info)))
-             ;; flag should be "-p 1234:1234".
-             (throw 'no-tunnels conn-info))
           (_
-             (setq server (if user (concat user "@" host)
+           (setq remote-host (if user (concat user "@" host)
                                host))))))
-      (let* ((keys '(:hb_port :shell_port :control_port
+    (unless (executable-find "ssh")
+      (error "SSH not found on `exec-path'"))
+    (let* ((port-keys (list :hb_port :shell_port :control_port
                             :stdin_port :iopub_port))
-             (lports (jupyter-available-local-ports (length keys))))
+           (local-ports (jupyter-available-local-ports (length port-keys)))
+           (conn-info (jupyter-read-plist conn-file)))
       (cl-loop
-         with remoteip = (plist-get conn-info :ip)
+       with remote-ip = (plist-get conn-info :ip)
-         for (key maybe-rport) on conn-info by #'cddr
+       for (key value) on conn-info by #'cddr
-         collect key and if (memq key keys)
+       collect key
-         collect (let ((lport (pop lports)))
+       if (eq key :ip) collect "127.0.0.1"
-                   (prog1 lport
+       else if (memq key port-keys)
-                     (jupyter-make-ssh-tunnel lport maybe-rport server remoteip)))
+       collect (let ((remote-port value)
-         else collect maybe-rport)))))
+                     (local-port (pop local-ports)))
+                 (prog1 local-port
+                   (jupyter-make-ssh-tunnel
+                    local-port remote-port remote-host remote-ip)))
+       else collect value))))
 
 ;;; Helper functions
 

test/jupyter-test.el

@@ -772,6 +772,44 @@
         (delete-file file)))
     (should-not (memq fun kill-emacs-hook))))
 
+;; TODO: Docker test
+(ert-deftest jupyter-tunnel-connection ()
+  :tags '(client ssh)
+  (let ((kernel (jupyter-command-kernel
+                 :spec (jupyter-guess-kernelspec "python"))))
+    (jupyter-start-kernel kernel "--ip=0.0.0.0")
+    (unwind-protect
+        (with-current-buffer (process-buffer (oref kernel process))
+          (goto-char (point-min))
+          (re-search-forward "Connection file: \\(.+\\)\n")
+          ;; Ensure this is required since TRAMP will fail without it
+          ;; on Travis.
+          (require 'files-x)
+          (let* ((method (cdr (assoc "ssh" tramp-methods)))
+                 (conn-file (match-string 1))
+                 (conn-info (jupyter-read-plist conn-file))
+                 (ssh-conn-file (concat "/ssh:localhost:" conn-file))
+                 (ssh-conn-info (jupyter-tunnel-connection ssh-conn-file)))
+            (should (equal (plist-get conn-info :ip) "0.0.0.0"))
+            (should (equal (plist-get ssh-conn-info :ip) "127.0.0.1"))
+            (cl-loop for port in '(:hb_port
+                                   :control_port
+                                   :shell_port
+                                   :iopub_port
+                                   :stdin_port)
+                     do (should-not
+                         (equal (plist-get conn-info port)
+                                (plist-get ssh-conn-info port))))
+            ;; Can we talk to the kernel
+            (let* ((manager (jupyter-kernel-process-manager
+                             :kernel kernel))
+                   (jupyter-current-client
+                    (jupyter-make-client manager 'jupyter-kernel-client)))
+              (jupyter-start-channels jupyter-current-client)
+              (should (equal "2" (jupyter-eval "1 + 1")))
+              (jupyter-stop-channels jupyter-current-client))))
+      (jupyter-kill-kernel kernel))))
+
 (ert-deftest jupyter-client-channels ()
   :tags '(client channels)
   (ert-info ("Starting/stopping channels")

test/test-helper.el

@@ -234,6 +234,17 @@ If the `current-buffer' is not a REPL, this is identical to
          (accept-process-output nil 1)
          ,@body))))
 
+(defmacro jupyter-test-with-kernel (kernel-name kernel &rest body)
+  "Start a new kernel with name KERNEL-NAME, bind it to KERNEL, evaluate BODY.
+KERNEL will be a `jupyter-command-kernel'."
+  (declare (indent 2))
+  `(let ((,kernel (jupyter-command-kernel
+                   :spec (jupyter-guess-kernelspec ,kernel-name))))
+     (jupyter-start-kernel ,kernel)
+     (unwind-protect
+         (progn ,@body)
+       (jupyter-kill-kernel ,kernel))))
+
 (defmacro jupyter-test-with-kernel-client (kernel client &rest body)
   "Start a new KERNEL client, bind it to CLIENT, evaluate BODY.
 This only starts a single global client unless the variable