mirror of
https://github.com/vale981/ray
synced 2025-03-06 10:31:39 -05:00
60d4d5e1aa
* Remove all __future__ imports from RLlib. * Remove (object) again from tf_run_builder.py::TFRunBuilder. * Fix 2xLINT warnings. * Fix broken appo_policy import (must be appo_tf_policy) * Remove future imports from all other ray files (not just RLlib). * Remove future imports from all other ray files (not just RLlib). * Remove future import blocks that contain `unicode_literals` as well. Revert appo_tf_policy.py to appo_policy.py (belongs to another PR). * Add two empty lines before Schedule class. * Put back __future__ imports into determine_tests_to_run.py. Fails otherwise on a py2/print related error.
108 lines
3.9 KiB
Python
108 lines
3.9 KiB
Python
"""Example of using a custom ModelV2 Keras-style model."""
|
|
|
|
import argparse
|
|
|
|
import ray
|
|
from ray import tune
|
|
from ray.rllib.models import ModelCatalog
|
|
from ray.rllib.models.tf.misc import normc_initializer
|
|
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
|
|
from ray.rllib.agents.dqn.distributional_q_model import DistributionalQModel
|
|
from ray.rllib.utils import try_import_tf
|
|
from ray.rllib.models.tf.visionnet_v2 import VisionNetwork as MyVisionNetwork
|
|
|
|
tf = try_import_tf()
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--run", type=str, default="DQN") # Try PG, PPO, DQN
|
|
parser.add_argument("--stop", type=int, default=200)
|
|
parser.add_argument("--use_vision_network", action="store_true")
|
|
|
|
|
|
class MyKerasModel(TFModelV2):
|
|
"""Custom model for policy gradient algorithms."""
|
|
|
|
def __init__(self, obs_space, action_space, num_outputs, model_config,
|
|
name):
|
|
super(MyKerasModel, self).__init__(obs_space, action_space,
|
|
num_outputs, model_config, name)
|
|
self.inputs = tf.keras.layers.Input(
|
|
shape=obs_space.shape, name="observations")
|
|
layer_1 = tf.keras.layers.Dense(
|
|
128,
|
|
name="my_layer1",
|
|
activation=tf.nn.relu,
|
|
kernel_initializer=normc_initializer(1.0))(self.inputs)
|
|
layer_out = tf.keras.layers.Dense(
|
|
num_outputs,
|
|
name="my_out",
|
|
activation=None,
|
|
kernel_initializer=normc_initializer(0.01))(layer_1)
|
|
value_out = tf.keras.layers.Dense(
|
|
1,
|
|
name="value_out",
|
|
activation=None,
|
|
kernel_initializer=normc_initializer(0.01))(layer_1)
|
|
self.base_model = tf.keras.Model(self.inputs, [layer_out, value_out])
|
|
self.register_variables(self.base_model.variables)
|
|
|
|
def forward(self, input_dict, state, seq_lens):
|
|
model_out, self._value_out = self.base_model(input_dict["obs"])
|
|
return model_out, state
|
|
|
|
def value_function(self):
|
|
return tf.reshape(self._value_out, [-1])
|
|
|
|
|
|
class MyKerasQModel(DistributionalQModel):
|
|
"""Custom model for DQN."""
|
|
|
|
def __init__(self, obs_space, action_space, num_outputs, model_config,
|
|
name, **kw):
|
|
super(MyKerasQModel, self).__init__(
|
|
obs_space, action_space, num_outputs, model_config, name, **kw)
|
|
|
|
# Define the core model layers which will be used by the other
|
|
# output heads of DistributionalQModel
|
|
self.inputs = tf.keras.layers.Input(
|
|
shape=obs_space.shape, name="observations")
|
|
layer_1 = tf.keras.layers.Dense(
|
|
128,
|
|
name="my_layer1",
|
|
activation=tf.nn.relu,
|
|
kernel_initializer=normc_initializer(1.0))(self.inputs)
|
|
layer_out = tf.keras.layers.Dense(
|
|
num_outputs,
|
|
name="my_out",
|
|
activation=tf.nn.relu,
|
|
kernel_initializer=normc_initializer(1.0))(layer_1)
|
|
self.base_model = tf.keras.Model(self.inputs, layer_out)
|
|
self.register_variables(self.base_model.variables)
|
|
|
|
# Implement the core forward method
|
|
def forward(self, input_dict, state, seq_lens):
|
|
model_out = self.base_model(input_dict["obs"])
|
|
return model_out, state
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ray.init()
|
|
args = parser.parse_args()
|
|
ModelCatalog.register_custom_model(
|
|
"keras_model", MyVisionNetwork
|
|
if args.use_vision_network else MyKerasModel)
|
|
ModelCatalog.register_custom_model(
|
|
"keras_q_model", MyVisionNetwork
|
|
if args.use_vision_network else MyKerasQModel)
|
|
tune.run(
|
|
args.run,
|
|
stop={"episode_reward_mean": args.stop},
|
|
config={
|
|
"env": "BreakoutNoFrameskip-v4"
|
|
if args.use_vision_network else "CartPole-v0",
|
|
"num_gpus": 0,
|
|
"model": {
|
|
"custom_model": "keras_q_model"
|
|
if args.run == "DQN" else "keras_model"
|
|
},
|
|
})
|