# This file is generated by `ray project create`.

# A unique identifier for the head node and workers of this cluster.
cluster_name: horovod-release-tests

# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 3
# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers. min_workers defaults to 0.
max_workers: 3

# If a node is idle for this many minutes, it will be removed.
idle_timeout_minutes: 5

# Cloud-provider specific configuration.
provider:
    type: aws
    region: us-west-2
    availability_zone: us-west-2a
    cache_stopped_nodes: False

# How Ray will authenticate with newly launched nodes.
auth:
    ssh_user: ubuntu

# By default Ray creates a new private keypair, but you can also use your own.
# If you do so, make sure to also set "KeyName" in the head and worker node
# configurations below.
#    ssh_private_key: /path/to/your/key.pem
available_node_types:
    gpu_2_32:
        min_workers: 3
        max_workers: 3
        resources: {"CPU": 32, "GPU": 2}
        node_config:
            InstanceType: g3.8xlarge
            ImageId: ami-0a2363a9cff180a64
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs: 
                      VolumeSize: 150
            TagSpecifications:
                - ResourceType: "instance"
                  Tags:
                    - Key: anyscale-user
                      Value: '{{env["ANYSCALE_USER"]}}'
                    - Key: anyscale-expiration
                      Value: '{{env["ANSYCALE_EXPIRATION"]}}'

# Provider-specific config for the head node, e.g. instance type. By default
# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
# For more documentation on available fields, see:
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
head_node:
    InstanceType: g3.8xlarge
    ImageId: latest_dlami
    BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
              VolumeSize: 150

worker_nodes:
  InstanceType: g3.8xlarge
  ImageId: latest_dlami
  BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
              VolumeSize: 150
  InstanceMarketOptions:
    MarketType: spot


file_mounts: {}

setup_commands:
    - pip install -U {{env["RAY_WHEEL"]}}
    - pip install 'ray[rllib]'
    - pip install torch torchvision
    - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip install -U git+https://github.com/horovod/horovod.git

# Command to start ray on the head node. You don't need to change this.
head_start_ray_commands:
    - ray stop
    - export RAY_BACKEND_LOG_LEVEL=debug
    - ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml

# Command to start ray on worker nodes. You don't need to change this.
worker_start_ray_commands:
    - ray stop
    - export RAY_BACKEND_LOG_LEVEL=debug
    - ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076