mirror of
https://github.com/vale981/nix-jupyter-env
synced 2025-03-04 09:21:41 -05:00
Initial commit
This commit is contained in:
Rehno Lindeque
commit
4ffe836dd7
10 changed files with 503 additions and 0 deletions
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
*.ipynb filter=nbstripout
|
||||
14
.gitignore
vendored
Normal file
14
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# Secrets
|
||||
.env
|
||||
.args
|
||||
|
||||
# Python
|
||||
*.pyc
|
||||
.venv
|
||||
|
||||
# Jupyter
|
||||
notebooks/.ipynb_checkpoints
|
||||
|
||||
# Vim
|
||||
*.swp
|
||||
|
||||
14
README.md
Normal file
14
README.md
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# Jupyter environment for Nix
|
||||
A nix environment for Jupyter along with deployment options. This is a work in progress, but contributions are welcome.
|
||||
|
||||
Supported kernels
|
||||
|
||||
* IPython
|
||||
* IHaskell - WORK IN PROGRESS
|
||||
|
||||
Deployment options:
|
||||
|
||||
* Nix shell environment - WORK IN PROGRESS
|
||||
* NixOps AWS (EC2 spot instance) - WORK IN PROGRESS
|
||||
* NixOps VirtualBox - WORK IN PROGRESS
|
||||
|
||||
9
default.nix
Normal file
9
default.nix
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{ mkDerivation, base, stdenv, bin-package-db }:
|
||||
mkDerivation {
|
||||
pname = "jupyter-env";
|
||||
version = "0.1.0.0";
|
||||
src = ./.;
|
||||
libraryHaskellDepends = [ base ];
|
||||
description = "Example environment for running a Jupyter environment in Nix that includes ipython and ihaskell (contributions welcome)";
|
||||
license = stdenv.lib.licenses.mit;
|
||||
}
|
||||
52
ec2-spot.nix
Normal file
52
ec2-spot.nix
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
{
|
||||
...
|
||||
}:
|
||||
|
||||
let region = "us-east-1";
|
||||
in
|
||||
{
|
||||
resources =
|
||||
{
|
||||
ec2KeyPairs.ec2-spot-jupyter = { inherit region; };
|
||||
};
|
||||
jupyter-ec2-spot =
|
||||
{ resources, config, pkgs, nodes, lib, ... }:
|
||||
{
|
||||
imports =
|
||||
[
|
||||
# TODO
|
||||
];
|
||||
|
||||
services.journald = {
|
||||
rateLimitBurst = 0;
|
||||
rateLimitInterval = "0";
|
||||
};
|
||||
|
||||
networking.firewall =
|
||||
{
|
||||
enable = true;
|
||||
allowedTCPPorts = [ 22 80 443 ];
|
||||
};
|
||||
|
||||
deployment =
|
||||
{
|
||||
targetEnv = "ec2";
|
||||
ec2 =
|
||||
{
|
||||
region = "us-east-1";
|
||||
instanceType = "m4.xlarge";
|
||||
spotInstancePrice = 5;
|
||||
securityGroupIds =
|
||||
[
|
||||
"sg-xxxxxxxx" # "allow-ssh-by-ip"
|
||||
"sg-xxxxxxxx" # "allow-http-by-ip"
|
||||
"sg-xxxxxxxx" # "allow-outbound"
|
||||
];
|
||||
subnetId = "subnet-xxxxxxxx";
|
||||
keyPair = resources.ec2KeyPairs.ec2-spot-jupyter;
|
||||
elasticIPv4 = "xx.xx.xx.xxx";
|
||||
associatePublicIpAddress = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
179
env.nix
Normal file
179
env.nix
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
{
|
||||
nixpkgs ? import <nixpkgs> {}
|
||||
, haskellCompiler ? "ghc7103"
|
||||
, pythonCompiler ? "python34"
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (nixpkgs) stdenv pkgs;
|
||||
|
||||
pythonOverrides =
|
||||
{
|
||||
overrides = self: super:
|
||||
{
|
||||
nbstripout = super.buildPythonPackage rec {
|
||||
version = "0.2.4a";
|
||||
name = "nbstripout-${version}";
|
||||
|
||||
/* Note that 0.2.4 is currently broken with python3 */
|
||||
/* src = pkgs.fetchurl { */
|
||||
/* url = "https://pypi.python.org/packages/source/n/nbstripout/${name}.tar.gz"; */
|
||||
/* sha256 = "1gphp7dl8cw5wmylk90vc2jbq4lgp680w3ybv9k0qq6ra2balcyk"; */
|
||||
/* }; */
|
||||
src = pkgs.fetchgit
|
||||
{
|
||||
url = "git://github.com/kynan/nbstripout.git";
|
||||
sha256 = "12spiwh9wncbrvqhcfnv74zy6qnsxyjl6ma0zknkaxk8ga0bss1z";
|
||||
rev = "fe1f767053462254a13c09bed8c5e24ec216a728";
|
||||
};
|
||||
|
||||
# TODO: what should build inputs look like?
|
||||
buildInputs = with self; [ /*pytest*/ /*nbformat*/ /*jupyter*/ ];
|
||||
propagatedBuildInputs = with self; [ ];
|
||||
doCheck = false;
|
||||
|
||||
meta =
|
||||
{
|
||||
description = "strip output from Jupyter and IPython notebooks";
|
||||
homepage = "https://github.com/kynan/nbstripout";
|
||||
license = pkgs.licenses.mit;
|
||||
maintainers = [ ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
pythonPackages = pkgs.${pythonCompiler + "Packages"} // pythonOverrides.overrides pythonPackages pkgs.${pythonCompiler + "Packages"};
|
||||
|
||||
# It appears to be necessary to use python.buildEnv instead of pkgs.buildEnv in order to maintain the correct PYTHONPATH
|
||||
ipython-env = pythonPackages.python.buildEnv.override
|
||||
{
|
||||
extraLibs =
|
||||
with pythonPackages;
|
||||
[
|
||||
jupyter
|
||||
|
||||
# Python packages (uncomment as needed)
|
||||
/* scipy */
|
||||
/* toolz */
|
||||
/* numpy */
|
||||
/* matplotlib */
|
||||
/* networkx */
|
||||
/* pandas */
|
||||
/* seaborn */
|
||||
|
||||
# Utilities (notebook formatting, cleaning for git etc)
|
||||
nbstripout # use this with .gitattributes for clean git commits
|
||||
];
|
||||
};
|
||||
|
||||
filterDist = src:
|
||||
let f = name: type: !(type == "directory" && baseNameOf (toString name) == "dist");
|
||||
in builtins.filterSource f src;
|
||||
|
||||
haskellOverrides =
|
||||
{
|
||||
overrides = self: super:
|
||||
let
|
||||
callLocalPackage = path: self.callPackage (filterDist path) {};
|
||||
in
|
||||
{
|
||||
# Overides for bleeding-edge ghc 8.0.x (Unfortunately this does not work yet)
|
||||
/* monads-tf = pkgs.haskell.lib.doJailbreak super.monads-tf; */
|
||||
/* ihaskell = */
|
||||
/* self.callPackage */
|
||||
/* ({ mkDerivation, aeson, base, base64-bytestring # , bin-package-db */
|
||||
/* , bytestring, cereal, cmdargs, containers, directory, filepath, ghc */
|
||||
/* , ghc-parser, ghc-paths, haskeline, haskell-src-exts, hlint, hspec */
|
||||
/* , http-client, http-client-tls, HUnit, sa-kernel, mtl, parsec */
|
||||
/* , process, random, setenv, shelly, split, stm, strict, system-argv0 */
|
||||
/* , text, transformers, unix, unordered-containers, utf8-string, uuid */
|
||||
/* , vector */
|
||||
/* }: */
|
||||
/* mkDerivation { */
|
||||
/* pname = "ihaskell"; */
|
||||
/* version = "0.8.3.0"; */
|
||||
/* sha256 = "c486e0b6342fa6261c671ad6a891f5763f7979bc225781329fe9f913a3833107"; */
|
||||
/* revision = "1"; */
|
||||
/* editedCabalFile = "4079263fe3b633e589775753fe7e3bbab21c800fd7d54c2aa6761478c5019654"; */
|
||||
/* isLibrary = true; */
|
||||
/* isExecutable = true; */
|
||||
/* libraryHaskellDepends = [ */
|
||||
/* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */
|
||||
/* cmdargs containers directory filepath ghc ghc-parser ghc-paths */
|
||||
/* haskeline haskell-src-exts hlint http-client http-client-tls */
|
||||
/* sa-kernel mtl parsec process random shelly split stm strict */
|
||||
/* system-argv0 text transformers unix unordered-containers */
|
||||
/* utf8-string uuid vector */
|
||||
/* ]; */
|
||||
/* ds = [ */
|
||||
/* aeson base /*bin-package-db* / bytestring containers directory ghc */
|
||||
/* sa-kernel process strict text transformers unix */
|
||||
/* ]; */
|
||||
/* testHaskellDepends = [ */
|
||||
/* aeson base base64-bytestring /*bin-package-db* / bytestring cereal */
|
||||
/* cmdargs containers directory filepath ghc ghc-parser ghc-paths */
|
||||
/* haskeline haskell-src-exts hlint hspec http-client http-client-tls */
|
||||
/* HUnit sad-kernel mtl parsec process random setenv shelly split */
|
||||
/* stm strict system-argv0 text transformers unix unordered-containers */
|
||||
/* utf8-string uuid vector */
|
||||
/* ]; */
|
||||
/* doCheck = false; */
|
||||
/* homepage = "http://github.com/gibiansky/IHaskell"; */
|
||||
/* description = "A Haskell backend kernel for the IPython project"; */
|
||||
/* license = stdenv.lib.licenses.mit; */
|
||||
/* }) {}; */
|
||||
};
|
||||
};
|
||||
haskellPackages = pkgs.haskell.packages.${haskellCompiler}.override haskellOverrides;
|
||||
ihaskellWithPackages = packages: haskellPackages.ghcWithPackages
|
||||
(
|
||||
self:
|
||||
with self;
|
||||
[
|
||||
ihaskell
|
||||
|
||||
# IHaskell packages (uncomment as needed)
|
||||
/* ihaskell-blaze */
|
||||
/* ihaskell-diagrams */
|
||||
/* ihaskell-display */
|
||||
|
||||
# Haskell packages (uncomment as needed)
|
||||
/* opaleye */
|
||||
/* cassava */
|
||||
] ++ packages self
|
||||
);
|
||||
ihaskell-env =
|
||||
(
|
||||
pkgs.buildEnv
|
||||
{
|
||||
name = "ihaskell-env";
|
||||
paths =
|
||||
[
|
||||
(
|
||||
ihaskellWithPackages
|
||||
(
|
||||
self:
|
||||
with self;
|
||||
[
|
||||
# Add your own haskell packages here...
|
||||
]
|
||||
)
|
||||
)
|
||||
];
|
||||
}
|
||||
);
|
||||
|
||||
in
|
||||
|
||||
{
|
||||
inherit pythonPackages haskellPackages;
|
||||
jupyter-env = pkgs.buildEnv
|
||||
{
|
||||
name = "jupyter-env";
|
||||
paths =
|
||||
[
|
||||
ipython-env
|
||||
ihaskell-env
|
||||
];
|
||||
};
|
||||
}
|
||||
61
notebooks/test.ipynb
Normal file
61
notebooks/test.ipynb
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"{-# LANGUAGE NoImplicitPrelude #-}\n",
|
||||
"import Prelude hiding (sum)\n",
|
||||
"\n",
|
||||
"import Diagrams.Prelude\n",
|
||||
"import Opaleye\n",
|
||||
"\n",
|
||||
"testTable :: Table (Column PGText)\n",
|
||||
"testTable = Table \"test\" \n",
|
||||
" ( p2 \n",
|
||||
" ( required \"name\"\n",
|
||||
" )\n",
|
||||
" )\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\"Test\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Haskell",
|
||||
"language": "haskell",
|
||||
"name": "haskell"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": "ihaskell",
|
||||
"file_extension": ".hs",
|
||||
"name": "haskell",
|
||||
"version": "7.10.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
151
shell.nix
Normal file
151
shell.nix
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
{
|
||||
nixpkgs ?
|
||||
let
|
||||
inherit (import <nixpkgs> {}) fetchgit;
|
||||
in
|
||||
import
|
||||
( fetchgit
|
||||
{
|
||||
url = "git://github.com/NixOS/nixpkgs.git";
|
||||
sha256 = "1zfk41ddixdycq4w6sxbn9qzgmahfp98niilzlzp6yfg88xpp5v1";
|
||||
# Using the latest unstable nixpkgs for now to get access to recent updates
|
||||
# TODO: should probably peg to a stable version or unpeg in future
|
||||
rev = "2737365741406a2726e755c76390d8d2081119d7";
|
||||
}
|
||||
) {}
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (nixpkgs) pkgs;
|
||||
in
|
||||
pkgs.stdenv.mkDerivation
|
||||
{
|
||||
name = "jupyter-env";
|
||||
src = ./.;
|
||||
buildInputs =
|
||||
(
|
||||
with import ./env.nix { inherit nixpkgs; };
|
||||
with pkgs;
|
||||
[
|
||||
jupyter-env
|
||||
|
||||
# Utilities (uncomment as needed)
|
||||
/* cabal2nix */
|
||||
/* nixops */
|
||||
/* pythonPackages.pgcli */
|
||||
/* pythonPackages.mycli */
|
||||
/* pythonPackages.virtualenv */
|
||||
/* pythonPackages.pip */
|
||||
|
||||
]
|
||||
);
|
||||
shellHook =
|
||||
let
|
||||
nc="\\e[0m"; # No Color
|
||||
white="\\e[1;37m";
|
||||
black="\\e[0;30m";
|
||||
blue="\\e[0;34m";
|
||||
light_blue="\\e[1;34m";
|
||||
green="\\e[0;32m";
|
||||
light_green="\\e[1;32m";
|
||||
cyan="\\e[0;36m";
|
||||
light_cyan="\\e[1;36m";
|
||||
red="\\e[0;31m";
|
||||
light_red="\\e[1;31m";
|
||||
purple="\\e[0;35m";
|
||||
light_purple="\\e[1;35m";
|
||||
brown="\\e[0;33m";
|
||||
yellow="\\e[1;33m";
|
||||
grey="\\e[0;30m";
|
||||
light_grey="\\e[0;37m";
|
||||
in
|
||||
# localstate.nixops keeps state (IP addresses etc) local, but we're not checking this into git at present
|
||||
# See https://blog.wearewizards.io/how-to-use-nixops-in-a-team
|
||||
''
|
||||
echoline() { echo "------------------------------------------------------------------------------------------------------------------------"; }
|
||||
fail() {
|
||||
echo ""
|
||||
echoline
|
||||
cleanup
|
||||
exit 1
|
||||
}
|
||||
assert_exist() {
|
||||
if [ ! -e "$1" ]; then
|
||||
printf "$2\n"
|
||||
fail
|
||||
fi
|
||||
}
|
||||
assert_nonempty() {
|
||||
if [ -z "$1" ]; then
|
||||
printf "$2\n"
|
||||
fail
|
||||
fi
|
||||
}
|
||||
cleanup() {
|
||||
unset -f echoline
|
||||
unset -f fail
|
||||
unset -f assert_exist
|
||||
unset -f assert_nonempty
|
||||
unset -f cleanup
|
||||
}
|
||||
|
||||
echoline
|
||||
echo "Jupyter shell environment"
|
||||
echo ""
|
||||
|
||||
# if [ ! -e secrets.nix ]; then
|
||||
# printf "{ ... }:\n\n{\n}" > secrets.nix
|
||||
# fi
|
||||
|
||||
# assert_exist ".env" "Missing ${white}.env${nc} file detected. Please create one with your EC2 credentials before getting started."
|
||||
# (
|
||||
# export $(xargs < .env)
|
||||
# assert_nonempty "$EC2_ACCESS_KEY" "Environment variable ${white}\$EC2_ACCESS_KEY${nc} is empty. Please add it to your ${white}.env${nc} file."
|
||||
# assert_nonempty "$EC2_SECRET_KEY" "Environment variable ${white}\$EC2_SECRET_KEY${nc} is empty. Please add it to your ${white}.env${nc} file."
|
||||
# ) || exit
|
||||
|
||||
echo "Usage:"
|
||||
printf " ${white}nixops${nc} is aliased to use your ${white}.env${nc} file in order to deploy using EC2 keys.\n"
|
||||
printf " ${white}jupyter-remote${nc} is an alias for ${white}nixops <command> -d jupyter-remote${nc}.\n"
|
||||
printf " ${white}notebook${nc} is an alias for ${white}jupyter-notebook${nc}.\n"
|
||||
printf " ${white}livenotebook${nc} is an alias for ${white}jupyter-notebook${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
|
||||
printf " ${white}liveipython${nc} is an alias for ${white}ipython${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
|
||||
printf " ${white}liveihaskell${nc} is an alias for ${white}ihaskell${nc} using your ${white}.env${nc} file in order to get a db connection.\n"
|
||||
echo ""
|
||||
echo "E.g."
|
||||
printf " ${white}$ jupyter-remote create ./ec2-spot.nix ${light_grey}# create your deployment${nc}\n"
|
||||
printf " ${white}$ jupyter-remote deploy ${light_grey}# deploy a spot instance to amazon EC2${nc}\n"
|
||||
printf " ${white}$ jupyter-remote destroy ${light_grey}# destroy your running instance${nc}\n"
|
||||
printf " ${white}$ notebook ${light_grey}# run jupyter notebook without a db connection${nc}\n"
|
||||
printf " ${white}$ livenotebook ${light_grey}# run jupyter notebook with a db connection${nc}\n"
|
||||
printf " ${white}$ liveipython ${light_grey}# run ipython console with a db connection${nc}\n"
|
||||
printf " ${white}$ liveihaskell ${light_grey}# run ihaskell console with a db connection${nc}\n"
|
||||
echo ""
|
||||
|
||||
export NIXOPS_STATE=.localstate.nixops
|
||||
alias withenv='env $(xargs < .env)'
|
||||
# alias nixops='withenv \nixops' # The slash is used to avoid recursion
|
||||
alias jupyter-remote='withenv NIXOPS_DEPLOYMENT=jupyter-remote nixops'
|
||||
alias notebook='jupyter notebook'
|
||||
alias livenotebook='withenv jupyter notebook'
|
||||
alias liveipython='withenv ipython console'
|
||||
alias liveihaskell='withenv ipython console --kernel haskell'
|
||||
|
||||
echo "Install git filters..."
|
||||
git config filter.nbstripout.clean 'nbstripout'
|
||||
git config filter.nbstripout.smudge cat
|
||||
git config filter.nbstripout.required true
|
||||
# TODO: lhs would be very easy to manage in git, but unfortunately ihaskell convert doesn't handle stdout/stdin
|
||||
# git config filter.lhsconvert.clean 'ihaskell convert'
|
||||
# git config filter.lhsconvert.smudge cat
|
||||
# git config filter.lhsconvert.required true
|
||||
|
||||
# echo "Run in a local environment so that we can use pip as needed..."
|
||||
# virtualenv --python=python3.4 .venv
|
||||
# source .venv/bin/activate
|
||||
|
||||
echoline
|
||||
cleanup
|
||||
'';
|
||||
|
||||
}
|
||||
4
sources.nix
Normal file
4
sources.nix
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
# Download your own sources from github
|
||||
# TODO
|
||||
}
|
||||
18
vm.nix
Normal file
18
vm.nix
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
...
|
||||
}:
|
||||
|
||||
{
|
||||
jupyter-vm = { resources, ... }:
|
||||
{
|
||||
deployment =
|
||||
{
|
||||
targetEnv = "virtualbox";
|
||||
virtualbox =
|
||||
{
|
||||
memorySize = 1024;
|
||||
headless = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue