hiro/nix-jupyter-env
1
0
Fork 0
mirror of https://github.com/vale981/nix-jupyter-env synced 2025-03-04 17:31:41 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Initial commit

Browse source
This commit is contained in:
Rehno Lindeque 2016-03-07 12:15:31 +00:00
commit 4ffe836dd7
10 changed files with 503 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitattributes vendored Normal file
View file

@ -0,0 +1 @@
*.ipynb filter=nbstripout

14
.gitignore vendored Normal file
View file

@ -0,0 +1,14 @@
# Secrets
.env
.args
# Python
*.pyc
.venv
# Jupyter
notebooks/.ipynb_checkpoints
# Vim
*.swp

14
README.md Normal file
View file

@ -0,0 +1,14 @@
# Jupyter environment for Nix
A nix environment for Jupyter along with deployment options. This is a work in progress, but contributions are welcome.
Supported kernels
* IPython
* IHaskell - WORK IN PROGRESS
Deployment options:
* Nix shell environment - WORK IN PROGRESS
* NixOps AWS (EC2 spot instance) - WORK IN PROGRESS
* NixOps VirtualBox - WORK IN PROGRESS

9
default.nix Normal file
View file

@ -0,0 +1,9 @@
{ mkDerivation, base, stdenv, bin-package-db }:
mkDerivation {
pname = "jupyter-env";
version = "0.1.0.0";
src = ./.;
libraryHaskellDepends = [ base ];
description = "Example environment for running a Jupyter environment in Nix that includes ipython and ihaskell (contributions welcome)";
license = stdenv.lib.licenses.mit;
}

52
ec2-spot.nix Normal file
View file

@ -0,0 +1,52 @@
{
...
}:
let region = "us-east-1";
in
{
resources =
{
ec2KeyPairs.ec2-spot-jupyter = { inherit region; };
};
jupyter-ec2-spot =
{ resources, config, pkgs, nodes, lib, ... }:
{
imports =
[
# TODO
];
services.journald = {
rateLimitBurst = 0;
rateLimitInterval = "0";
};
networking.firewall =
{
enable = true;
allowedTCPPorts = [ 22 80 443 ];
};
deployment =
{
targetEnv = "ec2";
ec2 =
{
region = "us-east-1";
instanceType = "m4.xlarge";
spotInstancePrice = 5;
securityGroupIds =
[
"sg-xxxxxxxx" # "allow-ssh-by-ip"
"sg-xxxxxxxx" # "allow-http-by-ip"
"sg-xxxxxxxx" # "allow-outbound"
];
subnetId = "subnet-xxxxxxxx";
keyPair = resources.ec2KeyPairs.ec2-spot-jupyter;
elasticIPv4 = "xx.xx.xx.xxx";
associatePublicIpAddress = true;
};
};
};
}

179
env.nix Normal file
View file

@ -0,0 +1,179 @@
{
nixpkgs ? import <nixpkgs> {}
, haskellCompiler ? "ghc7103"
, pythonCompiler ? "python34"
}:
let
inherit (nixpkgs) stdenv pkgs;
pythonOverrides =
{
overrides = self: super:
{
nbstripout = super.buildPythonPackage rec {
version = "0.2.4a";
name = "nbstripout-${version}";
/* Note that 0.2.4 is currently broken with python3 */
/* src = pkgs.fetchurl { */
/* url = "https://pypi.python.org/packages/source/n/nbstripout/${name}.tar.gz"; */
/* sha256 = "1gphp7dl8cw5wmylk90vc2jbq4lgp680w3ybv9k0qq6ra2balcyk"; */
/* }; */
src = pkgs.fetchgit
{
url = "git://github.com/kynan/nbstripout.git";
sha256 = "12spiwh9wncbrvqhcfnv74zy6qnsxyjl6ma0zknkaxk8ga0bss1z";
rev = "fe1f767053462254a13c09bed8c5e24ec216a728";
};
# TODO: what should build inputs look like?
buildInputs = with self; [ /*pytest*/ /*nbformat*/ /*jupyter*/ ];
propagatedBuildInputs = with self; [ ];
doCheck = false;
meta =
{
description = "strip output from Jupyter and IPython notebooks";
homepage = "https://github.com/kynan/nbstripout";
license = pkgs.licenses.mit;
maintainers = [ ];
};
};
};
};
pythonPackages = pkgs.${pythonCompiler + "Packages"} // pythonOverrides.overrides pythonPackages pkgs.${pythonCompiler + "Packages"};
# It appears to be necessary to use python.buildEnv instead of pkgs.buildEnv in order to maintain the correct PYTHONPATH
ipython-env = pythonPackages.python.buildEnv.override
{
extraLibs =
with pythonPackages;
[
jupyter
# Python packages (uncomment as needed)
/* scipy */
/* toolz */
/* numpy */
/* matplotlib */
/* networkx */
/* pandas */
/* seaborn */
# Utilities (notebook formatting, cleaning for git etc)
nbstripout # use this with .gitattributes for clean git commits
];
};
filterDist = src:
let f = name: type: !(type == "directory" && baseNameOf (toString name) == "dist");
in builtins.filterSource f src;
haskellOverrides =
{
overrides = self: super:
let
callLocalPackage = path: self.callPackage (filterDist path) {};
in
{
# Overides for bleeding-edge ghc 8.0.x (Unfortunately this does not work yet)
/* monads-tf = pkgs.haskell.lib.doJailbreak super.monads-tf; */
/* ihaskell = */
/* self.callPackage */
/* ({ mkDerivation, aeson, base, base64-bytestring # , bin-package-db */
/* , bytestring, cereal, cmdargs, containers, directory, filepath, ghc */
/* , ghc-parser, ghc-paths, haskeline, haskell-src-exts, hlint, hspec */
/* , http-client, http-client-tls, HUnit, sa-kernel, mtl, parsec */
/* , process, random, setenv, shelly, split, stm, strict, system-argv0 */
/* , text, transformers, unix, unordered-containers, utf8-string, uuid */
/* , vector */
/* }: */
/* mkDerivation { */
/* pname = "ihaskell"; */
/* version = "0.8.3.0"; */
/* sha256 = "c486e0b6342fa6261c671ad6a891f5763f7979bc225781329fe9f913a3833107"; */
/* revision = "1"; */
/* editedCabalFile = "4079263fe3b633e589775753fe7e3bbab21c800fd7d54c2aa6761478c5019654"; */
/* isLibrary = true; */
/* isExecutable = true; */
/* libraryHaskellDepends = [ */
/* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */
/* cmdargs containers directory filepath ghc ghc-parser ghc-paths */
/* haskeline haskell-src-exts hlint http-client http-client-tls */
/* sa-kernel mtl parsec process random shelly split stm strict */
/* system-argv0 text transformers unix unordered-containers */
/* utf8-string uuid vector */
/* ]; */
/* ds = [ */
/* aeson base /*bin-package-db* / bytestring containers directory ghc */
/* sa-kernel process strict text transformers unix */
/* ]; */
/* testHaskellDepends = [ */
/* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */
/* cmdargs containers directory filepath ghc ghc-parser ghc-paths */
/* haskeline haskell-src-exts hlint hspec http-client http-client-tls */
/* HUnit sad-kernel mtl parsec process random setenv shelly split */
/* stm strict system-argv0 text transformers unix unordered-containers */
/* utf8-string uuid vector */
/* ]; */
/* doCheck = false; */
/* homepage = "http://github.com/gibiansky/IHaskell"; */
/* description = "A Haskell backend kernel for the IPython project"; */
/* license = stdenv.lib.licenses.mit; */
/* }) {}; */
};
};
haskellPackages = pkgs.haskell.packages.${haskellCompiler}.override haskellOverrides;
ihaskellWithPackages = packages: haskellPackages.ghcWithPackages
(
self:
with self;
[
ihaskell
# IHaskell packages (uncomment as needed)
/* ihaskell-blaze */
/* ihaskell-diagrams */
/* ihaskell-display */
# Haskell packages (uncomment as needed)
/* opaleye */
/* cassava */
] ++ packages self
);
ihaskell-env =
(
pkgs.buildEnv
{
name = "ihaskell-env";
paths =
[
(
ihaskellWithPackages
(
self:
with self;
[
# Add your own haskell packages here...
]
)
)
];
}
);
in
{
inherit pythonPackages haskellPackages;
jupyter-env = pkgs.buildEnv
{
name = "jupyter-env";
paths =
[
ipython-env
ihaskell-env
];
};
}

61
notebooks/test.ipynb Normal file
View file

@ -0,0 +1,61 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"{-# LANGUAGE NoImplicitPrelude #-}\n",
"import Prelude hiding (sum)\n",
"\n",
"import Diagrams.Prelude\n",
"import Opaleye\n",
"\n",
"testTable :: Table (Column PGText)\n",
"testTable = Table \"test\" \n",
" ( p2 \n",
" ( required \"name\"\n",
" )\n",
" )\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"\"Test\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Haskell",
"language": "haskell",
"name": "haskell"
},
"language_info": {
"codemirror_mode": "ihaskell",
"file_extension": ".hs",
"name": "haskell",
"version": "7.10.3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

151
shell.nix Normal file
View file

@ -0,0 +1,151 @@
{
nixpkgs ?
let
inherit (import <nixpkgs> {}) fetchgit;
in
import
( fetchgit
{
url = "git://github.com/NixOS/nixpkgs.git";
sha256 = "1zfk41ddixdycq4w6sxbn9qzgmahfp98niilzlzp6yfg88xpp5v1";
# Using the latest unstable nixpkgs for now to get access to recent updates
# TODO: should probably peg to a stable version or unpeg in future
rev = "2737365741406a2726e755c76390d8d2081119d7";
}
) {}
}:
let
inherit (nixpkgs) pkgs;
in
pkgs.stdenv.mkDerivation
{
name = "jupyter-env";
src = ./.;
buildInputs =
(
with import ./env.nix { inherit nixpkgs; };
with pkgs;
[
jupyter-env
# Utilities (uncomment as needed)
/* cabal2nix */
/* nixops */
/* pythonPackages.pgcli */
/* pythonPackages.mycli */
/* pythonPackages.virtualenv */
/* pythonPackages.pip */
]
);
shellHook =
let
nc="\\e[0m"; # No Color
white="\\e[1;37m";
black="\\e[0;30m";
blue="\\e[0;34m";
light_blue="\\e[1;34m";
green="\\e[0;32m";
light_green="\\e[1;32m";
cyan="\\e[0;36m";
light_cyan="\\e[1;36m";
red="\\e[0;31m";
light_red="\\e[1;31m";
purple="\\e[0;35m";
light_purple="\\e[1;35m";
brown="\\e[0;33m";
yellow="\\e[1;33m";
grey="\\e[0;30m";
light_grey="\\e[0;37m";
in
# localstate.nixops keeps state (IP addresses etc) local, but we're not checking this into git at present
# See https://blog.wearewizards.io/how-to-use-nixops-in-a-team
''
echoline() { echo "------------------------------------------------------------------------------------------------------------------------"; }
fail() {
echo ""
echoline
cleanup
exit 1
}
assert_exist() {
if [ ! -e "$1" ]; then
printf "$2\n"
fail
fi
}
assert_nonempty() {
if [ -z "$1" ]; then
printf "$2\n"
fail
fi
}
cleanup() {
unset -f echoline
unset -f fail
unset -f assert_exist
unset -f assert_nonempty
unset -f cleanup
}
echoline
echo "Jupyter shell environment"
echo ""
# if [ ! -e secrets.nix ]; then
# printf "{ ... }:\n\n{\n}" > secrets.nix
# fi
# assert_exist ".env" "Missing ${white}.env${nc} file detected. Please create one with your EC2 credentials before getting started."
# (
# export $(xargs < .env)
# assert_nonempty "$EC2_ACCESS_KEY" "Environment variable ${white}\$EC2_ACCESS_KEY${nc} is empty. Please add it to your ${white}.env${nc} file."
# assert_nonempty "$EC2_SECRET_KEY" "Environment variable ${white}\$EC2_SECRET_KEY${nc} is empty. Please add it to your ${white}.env${nc} file."
# ) || exit
echo "Usage:"
printf " ${white}nixops${nc} is aliased to use your ${white}.env${nc} file in order to deploy using EC2 keys.\n"
printf " ${white}jupyter-remote${nc} is an alias for ${white}nixops <command> -d jupyter-remote${nc}.\n"
printf " ${white}notebook${nc} is an alias for ${white}jupyter-notebook${nc}.\n"
printf " ${white}livenotebook${nc} is an alias for ${white}jupyter-notebook${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
printf " ${white}liveipython${nc} is an alias for ${white}ipython${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
printf " ${white}liveihaskell${nc} is an alias for ${white}ihaskell${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
echo ""
echo "E.g."
printf " ${white}$ jupyter-remote create ./ec2-spot.nix ${light_grey}# create your deployment${nc}\n"
printf " ${white}$ jupyter-remote deploy ${light_grey}# deploy a spot instance to amazon EC2${nc}\n"
printf " ${white}$ jupyter-remote destroy ${light_grey}# destroy your running instance${nc}\n"
printf " ${white}$ notebook ${light_grey}# run jupyter notebook without a db connection${nc}\n"
printf " ${white}$ livenotebook ${light_grey}# run jupyter notebook with a db connection${nc}\n"
printf " ${white}$ liveipython ${light_grey}# run ipython console with a db connection${nc}\n"
printf " ${white}$ liveihaskell ${light_grey}# run ihaskell console with a db connection${nc}\n"
echo ""
export NIXOPS_STATE=.localstate.nixops
alias withenv='env $(xargs < .env)'
# alias nixops='withenv \nixops' # The slash is used to avoid recursion
alias jupyter-remote='withenv NIXOPS_DEPLOYMENT=jupyter-remote nixops'
alias notebook='jupyter notebook'
alias livenotebook='withenv jupyter notebook'
alias liveipython='withenv ipython console'
alias liveihaskell='withenv ipython console --kernel haskell'
echo "Install git filters..."
git config filter.nbstripout.clean 'nbstripout'
git config filter.nbstripout.smudge cat
git config filter.nbstripout.required true
# TODO: lhs would be very easy to manage in git, but unfortunately ihaskell convert doesn't handle stdout/stdin
# git config filter.lhsconvert.clean 'ihaskell convert'
# git config filter.lhsconvert.smudge cat
# git config filter.lhsconvert.required true
# echo "Run in a local environment so that we can use pip as needed..."
# virtualenv --python=python3.4 .venv
# source .venv/bin/activate
echoline
cleanup
'';
}

4
sources.nix Normal file
View file

@ -0,0 +1,4 @@
{
# Download your own sources from github
# TODO
}

18
vm.nix Normal file
View file

@ -0,0 +1,18 @@
{
...
}:
{
jupyter-vm = { resources, ... }:
{
deployment =
{
targetEnv = "virtualbox";
virtualbox =
{
memorySize = 1024;
headless = true;
};
};
};
}