commit 4ffe836dd702cd6da8ff2a87c3d8118134191a17 Author: Rehno Lindeque Date: Mon Mar 7 12:15:31 2016 +0000 Initial commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..978cb3e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.ipynb filter=nbstripout diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f07c613 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Secrets +.env +.args + +# Python +*.pyc +.venv + +# Jupyter +notebooks/.ipynb_checkpoints + +# Vim +*.swp + diff --git a/README.md b/README.md new file mode 100644 index 0000000..061be01 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# Jupyter environment for Nix +A nix environment for Jupyter along with deployment options. This is a work in progress, but contributions are welcome. + +Supported kernels + +* IPython +* IHaskell - WORK IN PROGRESS + +Deployment options: + +* Nix shell environment - WORK IN PROGRESS +* NixOps AWS (EC2 spot instance) - WORK IN PROGRESS +* NixOps VirtualBox - WORK IN PROGRESS + diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..af288a6 --- /dev/null +++ b/default.nix @@ -0,0 +1,9 @@ +{ mkDerivation, base, stdenv, bin-package-db }: +mkDerivation { + pname = "jupyter-env"; + version = "0.1.0.0"; + src = ./.; + libraryHaskellDepends = [ base ]; + description = "Example environment for running a Jupyter environment in Nix that includes ipython and ihaskell (contributions welcome)"; + license = stdenv.lib.licenses.mit; +} diff --git a/ec2-spot.nix b/ec2-spot.nix new file mode 100644 index 0000000..b6791c1 --- /dev/null +++ b/ec2-spot.nix @@ -0,0 +1,52 @@ +{ + ... +}: + +let region = "us-east-1"; +in + { + resources = + { + ec2KeyPairs.ec2-spot-jupyter = { inherit region; }; + }; + jupyter-ec2-spot = + { resources, config, pkgs, nodes, lib, ... }: + { + imports = + [ + # TODO + ]; + + services.journald = { + rateLimitBurst = 0; + rateLimitInterval = "0"; + }; + + networking.firewall = + { + enable = true; + allowedTCPPorts = [ 22 80 443 ]; + }; + + deployment = + { + targetEnv = "ec2"; + ec2 = + { + region = "us-east-1"; + instanceType = "m4.xlarge"; + spotInstancePrice = 5; + securityGroupIds = + [ + "sg-xxxxxxxx" # "allow-ssh-by-ip" + "sg-xxxxxxxx" # "allow-http-by-ip" + "sg-xxxxxxxx" # "allow-outbound" + ]; + subnetId = "subnet-xxxxxxxx"; + keyPair = resources.ec2KeyPairs.ec2-spot-jupyter; + elasticIPv4 = "xx.xx.xx.xxx"; + associatePublicIpAddress = true; + }; + }; + }; + } diff --git a/env.nix b/env.nix new file mode 100644 index 0000000..dc9b297 --- /dev/null +++ b/env.nix @@ -0,0 +1,179 @@ +{ + nixpkgs ? import {} +, haskellCompiler ? "ghc7103" +, pythonCompiler ? "python34" +}: + +let + inherit (nixpkgs) stdenv pkgs; + + pythonOverrides = + { + overrides = self: super: + { + nbstripout = super.buildPythonPackage rec { + version = "0.2.4a"; + name = "nbstripout-${version}"; + + /* Note that 0.2.4 is currently broken with python3 */ + /* src = pkgs.fetchurl { */ + /* url = "https://pypi.python.org/packages/source/n/nbstripout/${name}.tar.gz"; */ + /* sha256 = "1gphp7dl8cw5wmylk90vc2jbq4lgp680w3ybv9k0qq6ra2balcyk"; */ + /* }; */ + src = pkgs.fetchgit + { + url = "git://github.com/kynan/nbstripout.git"; + sha256 = "12spiwh9wncbrvqhcfnv74zy6qnsxyjl6ma0zknkaxk8ga0bss1z"; + rev = "fe1f767053462254a13c09bed8c5e24ec216a728"; + }; + + # TODO: what should build inputs look like? + buildInputs = with self; [ /*pytest*/ /*nbformat*/ /*jupyter*/ ]; + propagatedBuildInputs = with self; [ ]; + doCheck = false; + + meta = + { + description = "strip output from Jupyter and IPython notebooks"; + homepage = "https://github.com/kynan/nbstripout"; + license = pkgs.licenses.mit; + maintainers = [ ]; + }; + }; + }; + }; + pythonPackages = pkgs.${pythonCompiler + "Packages"} // pythonOverrides.overrides pythonPackages pkgs.${pythonCompiler + "Packages"}; + + # It appears to be necessary to use python.buildEnv instead of pkgs.buildEnv in order to maintain the correct PYTHONPATH + ipython-env = pythonPackages.python.buildEnv.override + { + extraLibs = + with pythonPackages; + [ + jupyter + + # Python packages (uncomment as needed) + /* scipy */ + /* toolz */ + /* numpy */ + /* matplotlib */ + /* networkx */ + /* pandas */ + /* seaborn */ + + # Utilities (notebook formatting, cleaning for git etc) + nbstripout # use this with .gitattributes for clean git commits + ]; + }; + + filterDist = src: + let f = name: type: !(type == "directory" && baseNameOf (toString name) == "dist"); + in builtins.filterSource f src; + + haskellOverrides = + { + overrides = self: super: + let + callLocalPackage = path: self.callPackage (filterDist path) {}; + in + { + # Overides for bleeding-edge ghc 8.0.x (Unfortunately this does not work yet) + /* monads-tf = pkgs.haskell.lib.doJailbreak super.monads-tf; */ + /* ihaskell = */ + /* self.callPackage */ + /* ({ mkDerivation, aeson, base, base64-bytestring # , bin-package-db */ + /* , bytestring, cereal, cmdargs, containers, directory, filepath, ghc */ + /* , ghc-parser, ghc-paths, haskeline, haskell-src-exts, hlint, hspec */ + /* , http-client, http-client-tls, HUnit, sa-kernel, mtl, parsec */ + /* , process, random, setenv, shelly, split, stm, strict, system-argv0 */ + /* , text, transformers, unix, unordered-containers, utf8-string, uuid */ + /* , vector */ + /* }: */ + /* mkDerivation { */ + /* pname = "ihaskell"; */ + /* version = "0.8.3.0"; */ + /* sha256 = "c486e0b6342fa6261c671ad6a891f5763f7979bc225781329fe9f913a3833107"; */ + /* revision = "1"; */ + /* editedCabalFile = "4079263fe3b633e589775753fe7e3bbab21c800fd7d54c2aa6761478c5019654"; */ + /* isLibrary = true; */ + /* isExecutable = true; */ + /* libraryHaskellDepends = [ */ + /* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */ + /* cmdargs containers directory filepath ghc ghc-parser ghc-paths */ + /* haskeline haskell-src-exts hlint http-client http-client-tls */ + /* sa-kernel mtl parsec process random shelly split stm strict */ + /* system-argv0 text transformers unix unordered-containers */ + /* utf8-string uuid vector */ + /* ]; */ + /* ds = [ */ + /* aeson base /*bin-package-db* / bytestring containers directory ghc */ + /* sa-kernel process strict text transformers unix */ + /* ]; */ + /* testHaskellDepends = [ */ + /* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */ + /* cmdargs containers directory filepath ghc ghc-parser ghc-paths */ + /* haskeline haskell-src-exts hlint hspec http-client http-client-tls */ + /* HUnit sad-kernel mtl parsec process random setenv shelly split */ + /* stm strict system-argv0 text transformers unix unordered-containers */ + /* utf8-string uuid vector */ + /* ]; */ + /* doCheck = false; */ + /* homepage = "http://github.com/gibiansky/IHaskell"; */ + /* description = "A Haskell backend kernel for the IPython project"; */ + /* license = stdenv.lib.licenses.mit; */ + /* }) {}; */ + }; + }; + haskellPackages = pkgs.haskell.packages.${haskellCompiler}.override haskellOverrides; + ihaskellWithPackages = packages: haskellPackages.ghcWithPackages + ( + self: + with self; + [ + ihaskell + + # IHaskell packages (uncomment as needed) + /* ihaskell-blaze */ + /* ihaskell-diagrams */ + /* ihaskell-display */ + + # Haskell packages (uncomment as needed) + /* opaleye */ + /* cassava */ + ] ++ packages self + ); + ihaskell-env = + ( + pkgs.buildEnv + { + name = "ihaskell-env"; + paths = + [ + ( + ihaskellWithPackages + ( + self: + with self; + [ + # Add your own haskell packages here... + ] + ) + ) + ]; + } + ); + +in + + { + inherit pythonPackages haskellPackages; + jupyter-env = pkgs.buildEnv + { + name = "jupyter-env"; + paths = + [ + ipython-env + ihaskell-env + ]; + }; + } diff --git a/notebooks/test.ipynb b/notebooks/test.ipynb new file mode 100644 index 0000000..3d7cd46 --- /dev/null +++ b/notebooks/test.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "{-# LANGUAGE NoImplicitPrelude #-}\n", + "import Prelude hiding (sum)\n", + "\n", + "import Diagrams.Prelude\n", + "import Opaleye\n", + "\n", + "testTable :: Table (Column PGText)\n", + "testTable = Table \"test\" \n", + " ( p2 \n", + " ( required \"name\"\n", + " )\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "\"Test\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Haskell", + "language": "haskell", + "name": "haskell" + }, + "language_info": { + "codemirror_mode": "ihaskell", + "file_extension": ".hs", + "name": "haskell", + "version": "7.10.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..2eada52 --- /dev/null +++ b/shell.nix @@ -0,0 +1,151 @@ +{ + nixpkgs ? + let + inherit (import {}) fetchgit; + in + import + ( fetchgit + { + url = "git://github.com/NixOS/nixpkgs.git"; + sha256 = "1zfk41ddixdycq4w6sxbn9qzgmahfp98niilzlzp6yfg88xpp5v1"; + # Using the latest unstable nixpkgs for now to get access to recent updates + # TODO: should probably peg to a stable version or unpeg in future + rev = "2737365741406a2726e755c76390d8d2081119d7"; + } + ) {} +}: + +let + inherit (nixpkgs) pkgs; +in + pkgs.stdenv.mkDerivation + { + name = "jupyter-env"; + src = ./.; + buildInputs = + ( + with import ./env.nix { inherit nixpkgs; }; + with pkgs; + [ + jupyter-env + + # Utilities (uncomment as needed) + /* cabal2nix */ + /* nixops */ + /* pythonPackages.pgcli */ + /* pythonPackages.mycli */ + /* pythonPackages.virtualenv */ + /* pythonPackages.pip */ + + ] + ); + shellHook = + let + nc="\\e[0m"; # No Color + white="\\e[1;37m"; + black="\\e[0;30m"; + blue="\\e[0;34m"; + light_blue="\\e[1;34m"; + green="\\e[0;32m"; + light_green="\\e[1;32m"; + cyan="\\e[0;36m"; + light_cyan="\\e[1;36m"; + red="\\e[0;31m"; + light_red="\\e[1;31m"; + purple="\\e[0;35m"; + light_purple="\\e[1;35m"; + brown="\\e[0;33m"; + yellow="\\e[1;33m"; + grey="\\e[0;30m"; + light_grey="\\e[0;37m"; + in + # localstate.nixops keeps state (IP addresses etc) local, but we're not checking this into git at present + # See https://blog.wearewizards.io/how-to-use-nixops-in-a-team + '' + echoline() { echo "------------------------------------------------------------------------------------------------------------------------"; } + fail() { + echo "" + echoline + cleanup + exit 1 + } + assert_exist() { + if [ ! -e "$1" ]; then + printf "$2\n" + fail + fi + } + assert_nonempty() { + if [ -z "$1" ]; then + printf "$2\n" + fail + fi + } + cleanup() { + unset -f echoline + unset -f fail + unset -f assert_exist + unset -f assert_nonempty + unset -f cleanup + } + + echoline + echo "Jupyter shell environment" + echo "" + + # if [ ! -e secrets.nix ]; then + # printf "{ ... }:\n\n{\n}" > secrets.nix + # fi + + # assert_exist ".env" "Missing ${white}.env${nc} file detected. Please create one with your EC2 credentials before getting started." + # ( + # export $(xargs < .env) + # assert_nonempty "$EC2_ACCESS_KEY" "Environment variable ${white}\$EC2_ACCESS_KEY${nc} is empty. Please add it to your ${white}.env${nc} file." + # assert_nonempty "$EC2_SECRET_KEY" "Environment variable ${white}\$EC2_SECRET_KEY${nc} is empty. Please add it to your ${white}.env${nc} file." + # ) || exit + + echo "Usage:" + printf " ${white}nixops${nc} is aliased to use your ${white}.env${nc} file in order to deploy using EC2 keys.\n" + printf " ${white}jupyter-remote${nc} is an alias for ${white}nixops -d jupyter-remote${nc}.\n" + printf " ${white}notebook${nc} is an alias for ${white}jupyter-notebook${nc}.\n" + printf " ${white}livenotebook${nc} is an alias for ${white}jupyter-notebook${nc} using your ${white}.env${nc} file in order to get a db connection.\n" + printf " ${white}liveipython${nc} is an alias for ${white}ipython${nc} using your ${white}.env${nc} file in order to get a db connection.\n" + printf " ${white}liveihaskell${nc} is an alias for ${white}ihaskell${nc} using your ${white}.env${nc} file in order to get a db connection.\n" + echo "" + echo "E.g." + printf " ${white}$ jupyter-remote create ./ec2-spot.nix ${light_grey}# create your deployment${nc}\n" + printf " ${white}$ jupyter-remote deploy ${light_grey}# deploy a spot instance to amazon EC2${nc}\n" + printf " ${white}$ jupyter-remote destroy ${light_grey}# destroy your running instance${nc}\n" + printf " ${white}$ notebook ${light_grey}# run jupyter notebook without a db connection${nc}\n" + printf " ${white}$ livenotebook ${light_grey}# run jupyter notebook with a db connection${nc}\n" + printf " ${white}$ liveipython ${light_grey}# run ipython console with a db connection${nc}\n" + printf " ${white}$ liveihaskell ${light_grey}# run ihaskell console with a db connection${nc}\n" + echo "" + + export NIXOPS_STATE=.localstate.nixops + alias withenv='env $(xargs < .env)' + # alias nixops='withenv \nixops' # The slash is used to avoid recursion + alias jupyter-remote='withenv NIXOPS_DEPLOYMENT=jupyter-remote nixops' + alias notebook='jupyter notebook' + alias livenotebook='withenv jupyter notebook' + alias liveipython='withenv ipython console' + alias liveihaskell='withenv ipython console --kernel haskell' + + echo "Install git filters..." + git config filter.nbstripout.clean 'nbstripout' + git config filter.nbstripout.smudge cat + git config filter.nbstripout.required true + # TODO: lhs would be very easy to manage in git, but unfortunately ihaskell convert doesn't handle stdout/stdin + # git config filter.lhsconvert.clean 'ihaskell convert' + # git config filter.lhsconvert.smudge cat + # git config filter.lhsconvert.required true + + # echo "Run in a local environment so that we can use pip as needed..." + # virtualenv --python=python3.4 .venv + # source .venv/bin/activate + + echoline + cleanup + ''; + + } diff --git a/sources.nix b/sources.nix new file mode 100644 index 0000000..b05dc82 --- /dev/null +++ b/sources.nix @@ -0,0 +1,4 @@ +{ + # Download your own sources from github + # TODO +} diff --git a/vm.nix b/vm.nix new file mode 100644 index 0000000..39fbf52 --- /dev/null +++ b/vm.nix @@ -0,0 +1,18 @@ +{ + ... +}: + +{ + jupyter-vm = { resources, ... }: + { + deployment = + { + targetEnv = "virtualbox"; + virtualbox = + { + memorySize = 1024; + headless = true; + }; + }; + }; +}