Integration of Webui with Ray (#32)

* Initial integration of webui with ray

* Re-organized calling of build-webui in setup.py

* Fixed Lint comments on js code

* Fixed more lint issues

* Fixed various issues

* Fixed directory in services.py

* Small changes.

* Changes to match lint
This commit is contained in:
Wapaul1 2016-11-17 22:33:29 -08:00 committed by Robert Nishihara
parent 7babe0d22f
commit 08707f9408
16 changed files with 693 additions and 4 deletions

32
build-webui.sh Executable file
View file

@ -0,0 +1,32 @@
#!/usr/bin/env bash
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
platform="linux"
elif [[ "$unamestr" == "Darwin" ]]; then
platform="macosx"
else
echo "Unrecognized platform."
exit 1
fi
WEBUI_DIR="$ROOT_DIR/webui"
PYTHON_DIR="$ROOT_DIR/lib/python"
PYTHON_WEBUI_DIR="$PYTHON_DIR/webui"
pushd "$WEBUI_DIR"
npm install
if [[ $platform == "linux" ]]; then
nodejs ./node_modules/.bin/webpack -g
elif [[ $platform == "macosx" ]]; then
node ./node_modules/.bin/webpack -g
fi
pushd node_modules
rm -rf react* babel* classnames dom-helpers jsesc webpack .bin
popd
popd
cp -r $WEBUI_DIR/* $PYTHON_WEBUI_DIR/

View file

@ -9,7 +9,7 @@ To install Ray, first install the following dependencies. We recommend using
```
brew update
brew install cmake automake autoconf libtool boost
brew install cmake automake autoconf libtool boost node
sudo easy_install pip # If you're using Anaconda, then this is unnecessary.
pip install numpy funcsigs colorama psutil redis --ignore-installed six

View file

@ -10,7 +10,7 @@ To install Ray, first install the following dependencies. We recommend using
```
sudo apt-get update
sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip # If you're using Anaconda, then python-dev and python-pip are unnecessary.
sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-pip libboost-all-dev unzip nodejs npm # If you're using Anaconda, then python-dev and python-pip are unnecessary.
pip install numpy funcsigs colorama psutil redis
pip install --upgrade git+git://github.com/cloudpipe/cloudpickle.git@0d225a4695f1f65ae1cbb2e0bbc145e10167cce4 # We use the latest version of cloudpickle because it can serialize named tuples.

View file

@ -18,7 +18,7 @@ fi
if [[ $platform == "linux" ]]; then
# These commands must be kept in sync with the installation instructions.
sudo apt-get update
sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip
sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip nodejs npm
sudo pip install funcsigs colorama psutil redis
elif [[ $platform == "macosx" ]]; then
# check that brew is installed
@ -31,7 +31,7 @@ elif [[ $platform == "macosx" ]]; then
brew update
fi
# These commands must be kept in sync with the installation instructions.
brew install cmake automake autoconf libtool boost
brew install cmake automake autoconf libtool boost node
sudo easy_install pip
sudo pip install numpy funcsigs colorama psutil redis --ignore-installed six
fi

1
lib/python/MANIFEST.in Normal file
View file

@ -0,0 +1 @@
recursive-include webui *

View file

@ -161,6 +161,22 @@ def start_worker(address_info, worker_path, cleanup=True):
if cleanup:
all_processes.append(p)
def start_webui(redis_port, cleanup=True):
"""This method starts the web interface.
Args:
redis_port (int): The redis server's port
cleanup (bool): True if using Ray in local mode. If cleanup is true, then
this process will be killed by services.cleanup() when the Python process
that imported services exits. This is True by default.
"""
executable = "nodejs" if sys.platform == "linux" or sys.platform == "linux2" else "node"
command = [executable, os.path.join(os.path.abspath(os.path.dirname(__file__)), "../webui/index.js"), str(redis_port)]
with open("/tmp/webui_out.txt", "wb") as out:
p = subprocess.Popen(command, stdout=out)
if cleanup:
all_processes.append(p)
def start_ray_local(node_ip_address="127.0.0.1", num_workers=0, worker_path=None):
"""Start Ray in local mode.
@ -196,4 +212,5 @@ def start_ray_local(node_ip_address="127.0.0.1", num_workers=0, worker_path=None
start_worker(address_info, worker_path, cleanup=True)
time.sleep(0.3)
# Return the addresses of the relevant processes.
start_webui(redis_port)
return address_info

View file

@ -1,10 +1,15 @@
from __future__ import print_function
import os
import subprocess
from setuptools import setup, find_packages
import setuptools.command.install as _install
subprocess.check_call(["../../build-webui.sh"])
datafiles = [(root, [os.path.join(root, f) for f in files])
for root, dirs, files in os.walk("./webui")]
class install(_install.install):
def run(self):
subprocess.check_call(["../../build.sh"])
@ -22,6 +27,7 @@ setup(name="ray",
"lib/python/libplasma.so"],
"photon": ["build/photon_scheduler",
"libphoton.so"]},
data_files=datafiles,
cmdclass={"install": install},
install_requires=["numpy",
"funcsigs",

View file

3
webui/.babelrc Normal file
View file

@ -0,0 +1,3 @@
{
"presets": ["es2015", "react"]
}

305
webui/client/app/index.jsx Normal file
View file

@ -0,0 +1,305 @@
import React from 'react';
import {render} from 'react-dom';
import {AutoSizer, InfiniteLoader, List} from 'react-virtualized';
import classNames from 'classnames/bind';
import io from 'socket.io-client';
import scrollbarSize from 'dom-helpers/util/scrollbarSize';
class RayUI extends React.Component {
constructor(props) {
super(props);
this.state = {
table_one: "object_table",
table_two: "task_table",
table_three: "failure_table",
table_four: "Remote_table",
table_one_channel:"object",
table_two_channel:"task",
table_three_channel:"failure",
table_four_channel:"remote",
websocket_connection: io()
};
}
render() {
return (
<div>
<TableView table={this.state.table_one} channel={this.state.table_one_channel} socket={this.state.websocket_connection}/>
<TableView table={this.state.table_two} channel={this.state.table_two_channel} socket={this.state.websocket_connection}/>
<TableView table={this.state.table_three} channel={this.state.table_three_channel} socket={this.state.websocket_connection}/>
<TableView table={this.state.table_four} channel={this.state.table_four_channel} socket={this.state.websocket_connection}/>
</div>
);
}
}
class TableView extends React.Component {
constructor(props) {
super(props);
this.state= {
press: false
};
this._toggle=this._toggle.bind(this)
}
_toggle(e){
this.setState({press: !this.state.press});
}
render() {
return (
<div>
<button onClick={this._toggle}>{this.props.table}</button>
<TableScroll press={this.state.press} table={this.props.table} channel={this.props.channel} socket={this.props.socket}/>
</div>
);
}
}
class TableScroll extends React.Component{
constructor(props) {
super(props);
this.state = {
messages: [],
filteredmsg: [],
loadedRowsMap: {},
content: "This is the view pane.",
atEnd: true,
currentpos: 0,
filter: ""
};
this._onfilter = this._onfilter.bind(this);
this.filterdata = this.filterdata.bind(this);
this._isRowLoaded = this._isRowLoaded.bind(this);
this._rowRenderer = this._rowRenderer.bind(this);
this._loadMoreRows = this._loadMoreRows.bind(this);
this.objectselect = this.objectselect.bind(this);
this._objectrenderer = this._objectrenderer.bind(this);
this.taskselect = this.taskselect.bind(this);
this._taskrenderer = this._taskrenderer.bind(this);
this.computationselect = this.computationselect.bind(this);
this._computationrenderer = this._computationrenderer.bind(this);
this.failureselect = this.failureselect.bind(this);
this._failurerenderer = this._failurerenderer.bind(this);
this.remoteselect = this.remoteselect.bind(this);
this._remoterenderer = this._remoterenderer.bind(this);
this.scrollcontroller = this.scrollcontroller.bind(this);
switch (this.props.channel) {
case "object": this.renderfunction = this._objectrenderer;
this.header = (<div className={classNames("evenRow", "cell", "centeredCell")}>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Object ID"}</div>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Plasma Store ID"}</div>
</div>);
break;
case "failure": this.renderfunction = this._failurerenderer;
this.header = (<div className={classNames("evenRow", "cell", "centeredCell")}>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Failed Function"}</div>
</div>);
break;
case "computation": this.renderfunction = this._computationrenderer;
this.header = (<div className={classNames("evenRow", "cell", "centeredCell")}>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Task iid"}</div>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Function_id"}</div>
</div>);
break;
case "task": this.renderfunction = this._taskrenderer;
this.header = (<div className={classNames("evenRow", "cell", "centeredCell")}>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Node id"}</div>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Function_id"}</div>
</div>);
break;
case "remote": this.renderfunction = this._remoterenderer;
this.header = (<div className={classNames("evenRow", "cell", "centeredCell")}>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Function id"}</div>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Module"}</div>
<div className={classNames("evenRow", "cell", "centeredCell")}>{"Function Name"}</div>
</div>);
break;
default: break;
}
}
componentDidMount() {
var self = this;
console.log("port" + location.port);
this.props.socket.emit('getall', {table:this.props.channel});
var arr = [];
this.props.socket.on(this.props.channel, function(message) {
console.log("got message " + message);
var filteredarray = self.state.filteredmsg;
message.forEach(function(msg,i){
// console.log("Content is " + JSON.stringify(msg));
arr.push(msg);
if (self.filterdata(msg, self.state.filter)) {filteredarray.push(msg);}
});
self.setState({messages: arr, filteredmsg: filteredarray});
});
}
filterdata(data, filter) {
console.log(data);
var self = this;
return filter != "" ? Object.values(data).filter(function(data){return data === Object(data) ? self.filterdata(data, filter) : String(data).includes(filter)}).length != 0 : true;
}
_onfilter(e) {
var self = this;
this.setState({filteredmsg: e.target.value != "" ? self.state.messages.filter(function(data){return self.filterdata(data, e.target.value)}) : self.state.messages, filter:e.target.value});
}
_isRowLoaded({ index }) {
return !!this.state.loadedRowsMap[index];
}
objectselect(data, e) {
console.log(data);
this.setState({content:JSON.stringify(data)})
}
_objectrenderer(record, key, style) {
const className = classNames("evenRow", "cell", "centeredCell");
return (
<button
onClick={this.objectselect.bind(null, record)}
className={classNames("evenRow", "cell", "rowbutton", "centeredCell")}
key={key}
style={style}
>
<div className={className}>{record.ObjectId}</div>
<div className={className}>{record.PlasmaStoreId}</div>
</button>
);
}
failureselect(data, e) {
console.log(data);
this.setState({content:data.error})
}
_failurerenderer(record, key, style) {
const className = classNames("evenRow", "cell", "centeredCell");
return (
<button
onClick={this.failureselect.bind(null, record)}
className={classNames("evenRow", "cell", "rowbutton", "centeredCell")}
key={key}
style={style}
>
<div className={className}>{record.functionname}</div>
</button>
);
}
computationselect(data, e) {
console.log(data);
this.setState({content: JSON.stringify});
}
_computationrenderer(record, key, style) {
const className = classNames("evenRow", "cell", "centeredCell");
return (
<button
onClick={this.computationselect.bind(null, record)}
className={classNames("evenRow", "cell", "rowbutton", "centeredCell")}
key={key}
style={style}
>
<div className={className}>{record.task_iid.toString().substring(0,8)}</div>
<div className={className}>{record.function_id.toString().substring(0,8)}</div>
</button>
);
}
taskselect(data, e) {
console.log(data);
this.setState({content: JSON.stringify(data)});
}
_taskrenderer(record, key, style) {
const className = classNames("evenRow", "cell", "centeredCell");
return (
<button
onClick={this.taskselect.bind(null, record)}
className={classNames("evenRow", "cell", "rowbutton", "centeredCell")}
key={key}
style={style}
>
<div className={className}>{record.node_id}</div>
<div className={className}>{record.function_id.toString().substring(0,8)}</div>
</button>
);
}
remoteselect(data, e) {
console.log(data);
this.setState({content: JSON.stringify(data)});
}
_remoterenderer(record, key, style) {
const className = classNames("evenRow", "cell", "centeredCell");
return (
<button
onClick={this.remoteselect.bind(null, record)}
className={classNames("evenRow", "cell", "rowbutton", "centeredCell")}
key={key}
style={style}
>
<div className={className}>{record.function_id}</div>
<div className={className}>{record.module}</div>
<div className={className}>{record.name}</div>
</button>
);
}
_rowRenderer({ index, key, style, isScrolling}){
const record = this.state.filteredmsg[index];
return this.renderfunction(record, key, style);
}
_loadMoreRows({ startIndex, stopIndex }) {
for (var i = startIndex; i <= stopIndex; i++) {
this.state.loadedRowsMap[i] = 1;
}
let promiseResolver;
return new Promise(resolve => {
promiseResolver = resolve;
})
}
scrollcontroller({clientHeight, scrollHeight, scrollTop}){
this.setState({atEnd:scrollTop >= scrollHeight- clientHeight, currentpos: Math.floor(scrollTop/20)-1});
}
render() {
if (!this.props.press) {
var style = {display:'none'}
}
var self = this;
return (
<AutoSizer disableHeight>
{({ width }) => (
<div style={style}>
<input type="text" onChange={this._onfilter} />
<div style={{width:width-2*scrollbarSize(), height:20}}>{this.header}</div>
<InfiniteLoader
isRowLoaded={this._isRowLoaded}
loadMoreRows={this._loadMoreRows}
rowCount={this.state.filteredmsg.length}
style={{display: "inline-block"}}>
{({ onRowsRendered, registerChild }) => (
<List
ref={registerChild}
rowRenderer={this._rowRenderer}
onRowsRendered={onRowsRendered}
className={"BodyGrid"}
width={width}
height={300}
onScroll={this.scrollcontroller}
overscanRowCount={20}
scrollToIndex={this.state.atEnd ? this.state.filteredmsg.length-1: this.currentpos}
scrollToAlignment={"end"}
rowCount={this.state.filteredmsg.length}
rowHeight={20}
/>
)}
</InfiniteLoader>
<textarea style={{width:width, height:300, display: "inline-block"}} value={this.state.content}></textarea>
</div>
)}
</AutoSizer>
);}
}
render(<RayUI/>, document.getElementById('mount-point'));

10
webui/client/index.html Normal file
View file

@ -0,0 +1,10 @@
<head>
<title>Ray UI</title>
</head>
<link rel=stylesheet type=text/css href="/static/rayui.css">
<body>
<div id="mount-point"></div>
<script src="/public/bundle.js" type="text/javascript"></script>
</body>

View file

@ -0,0 +1,84 @@
.GridContainer {
margin-top: 15px;
border: 1px solid #e0e0e0;
}
.GridRow {
margin-top: 15px;
display: flex;
flex-direction: row;
}
.GridColumn {
display: flex;
flex-direction: column;
flex: 1 1 auto;
}
.LeftSideGridContainer {
flex: 0 0 50px;
}
.BodyGrid {
width: 100%;
border: 1px solid #e0e0e0;
}
.evenRow,
.oddRow {
border-bottom: 1px solid #e0e0e0;
}
.oddRow {
background-color: #fafafa;
}
.cell,
.headerCell {
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
justify-content: center;
padding: 0 .5em;
}
.cell {
border-right: 1px solid #e0e0e0;
border-bottom: 1px solid #e0e0e0;
display: flex;
flex-direction: row;
background-color: white;
}
.rowbutton:hover {
background-color: blue;
}
.rowbutton:hover *{
background-color: gold;
}
.headerCell {
font-weight: bold;
border-right: 1px solid #e0e0e0;
}
.centeredCell {
align-items: center;
text-align: center;
}
.table {
display: flex;
flex-direction: row;
}
.letterCell {
font-size: 1.5em;
color: #fff;
text-align: center;
}
.noCells {
position: absolute;
top: 0;
bottom: 0;
left: 0;
right: 0;
display: flex;
align-items: center;
justify-content: center;
font-size: 1em;
color: #bdbdbd;
}

112
webui/index.js Normal file
View file

@ -0,0 +1,112 @@
var express = require('express');
var app = express();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var redis = require("redis");
var task = require('./task.js');
if (process.argv.length > 2) {
var port = process.argv[2];
var sub = redis.createClient(port, {return_buffers: true});
var db = redis.createClient(port, {return_buffers: true});
} else {
var sub = redis.createClient({return_buffers: true});
var db = redis.createClient({return_buffers: true});
}
const assert = require('assert');
app.use(express.static(__dirname + '/client'));
app.get('/', function(req, res) {
res.sendFile(__dirname + '/client/index.html');
});
sub.config("SET", "notify-keyspace-events", "AKE");
sub.psubscribe("task_log:*");
sub.psubscribe("__keyspace@0__:obj:*");
sub.psubscribe("__keyspace@0__:Failures*");
sub.psubscribe("__keyspace@0__:RemoteFunction*");
io.on('connection', function(socket) {
console.log('a user connected');
socket.on('disconnect', function() { console.log('user disconnected'); });
sub.on('psubscribe', function(channel, count) { console.log("Subscribed"); });
});
backlogobject = [];
backlogtask = [];
backlogfailures = [];
backlogremotefunction = [];
var failureindex;
db.llen("Failures", function(err, result) { failureindex = result; });
sub.on('pmessage', function(pattern, channel, message) {
if (channel.toString().split(":")[0] === "__keyspace@0__") {
console.log(channel.toString());
switch (channel.toString().split(":")[1]) {
case "Failures":
db.lindex("Failures", failureindex++, function(err, result) {
backlogfailures.push({
"functionname": result.toString().split(" ")[2].slice(5, -5),
"error": result.toString()
});
});
break;
case "obj":
db.smembers(channel.slice(15), function(err, result) {
console.log(result);
backlogobject.push({
"ObjectId": channel.slice(19).toString('hex'),
"PlasmaStoreId": result[0].toString()
});
});
break;
case "RemoteFunction":
db.hgetall(channel.slice(15), function(err, result) {
backlogremotefunction.push({
"function_id": result.function_id.toString('hex'),
"module": result.module.toString(),
"name": result.name.toString()
});
});
break;
default:
console.log(channel.toString());
break;
}
} else {
backlogtask.push(task.parse_task_instance(message));
}
});
setInterval(function() {
if (backlogfailures.length > 0) {
console.log("Sending ", backlogfailures.length, " objects on failure");
console.log(backlogfailures);
io.sockets.emit('failure', backlogfailures);
}
backlogfailures = [];
}, 30);
setInterval(function() {
if (backlogobject.length > 0) {
console.log("Sending ", backlogobject.length, " objects on object");
console.log(backlogobject);
io.sockets.emit('object', backlogobject);
}
backlogobject = [];
}, 30);
setInterval(function() {
if (backlogtask.length > 0) {
console.log("Sending ", backlogtask.length, " objects on task");
io.sockets.emit('task', backlogtask);
}
backlogtask = [];
}, 30);
setInterval(function() {
if (backlogremotefunction.length > 0) {
console.log("Sending ", backlogremotefunction.length, " objects on remote");
console.log(backlogremotefunction);
io.sockets.emit('remote', backlogremotefunction);
}
backlogremotefunction = [];
}, 30);
http.listen(3000, function() { console.log('listening on *:3000'); });

32
webui/package.json Normal file
View file

@ -0,0 +1,32 @@
{
"name": "RayWebUI",
"version": "0.0.1",
"description": "A Web UI for Ray",
"repository": {
"type": "git",
"url": "git://github.com/ray-project/ray.git"
},
"private": true,
"dependencies": {
"babel-core": "~6.17.0",
"babel-loader": "~6.2.5",
"babel-preset-es2015": "~6.16.0",
"babel-preset-react": "~6.16.0",
"classnames": "~2.2.5",
"express": "^4.10.2",
"jbinary": "^2.1.3",
"react": "~15.3.2",
"react-addons-shallow-compare": "~15.3.2",
"react-dom": "~15.3.2",
"react-virtualized": "~8.0.12",
"redis": "^2.6.2",
"socket.io": "^1.5.0",
"socket.io-client": "~1.5.0",
"webpack": "~1.13.2",
"dom-helpers": "~3.0.0",
"jsesc": "~2.2.0"
},
"devDependencies": {
"webpack": "~1.13.3"
}
}

68
webui/task.js Normal file
View file

@ -0,0 +1,68 @@
var jb = require('jbinary');
var stream = require('stream');
var task_argument = {
'jBinary.littleEndian': true,
is_ref: ['enum', 'int8', [true, false]],
padding: ['array', 'int8', 7],
reference: [
'if', 'is_ref',
{object_id: ['array', 'uint8', 20], padding: ['array', 'int8', 4]}
],
value: [
'if_not', 'is_ref',
{offset: 'int64', length: 'int64', padding: ['array', 'int8', 8]}
]
};
var task_spec_header = {
'jBinary.littleEndian': true,
function_id: ['array', 'uint8', 20],
padding: ['array', 'uint8', 4],
num_args: ['int64'],
arg_index: ['int64'],
num_returns: ['int64'],
args_value_size: ['int64'],
args_value_offset: ['int64'],
arguments: ['array', task_argument, 'num_args']
};
var task_instance = {
'jBinary.littleEndian': true,
task_iid: ['array', 'uint8', 20],
state: 'int32',
node_id: ['array', 'uint8', 20],
padding: ['array', 'uint8', 4],
task_spec_header: ['object', task_spec_header],
};
// Convert string or byte buffer of an object ID to hex string.
function id_to_hex(id) {
return new Buffer(id).toString('hex');
}
module.exports = {
parse_task_instance: function(buffer) {
var binary = new jb(buffer, task_instance);
binary.read('padding');
var task_spec = binary.read('task_spec_header');
var arguments = [];
for (var i = 0; i < task_spec['arguments'].length; i++) {
var arg = task_spec['arguments'][i];
if (arg['is_ref']) {
console.log(arg['reference']['object_id']);
arguments.push(id_to_hex(arg['reference']['object_id']));
} else {
arguments.push("value");
}
}
var state = binary.read('state');
var node_id = binary.read('node_id');
return {
state: state, node_id: id_to_hex(node_id),
function_id: id_to_hex(task_spec['function_id']), arguments: arguments
}
}
}

19
webui/webpack.config.js Normal file
View file

@ -0,0 +1,19 @@
var webpack = require('webpack');
var path = require('path');
var BUILD_DIR = path.resolve(__dirname, 'client/public');
var APP_DIR = path.resolve(__dirname, 'client/app');
var config = {
entry: APP_DIR + '/index.jsx',
output: {path: BUILD_DIR, filename: 'bundle.js'},
module: {
loaders: [{
test: /\.jsx?/,
include: APP_DIR,
loader: 'babel',
query: {presets: ['react']}
}]
}
};
module.exports = config;