cluster_name: big_data_ingestion.yaml 

max_workers: 86

provider:
    type: aws
    region: us-west-1

auth:
    ssh_user: ubuntu

available_node_types:
    head:
        node_config:
            InstanceType: i3.8xlarge
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs:
                      VolumeSize: 300
        resources: { }

    gpu_nodes:
        min_workers: 16
        max_workers: 16
        node_config:
            InstanceType: i3.8xlarge
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs:
                      VolumeSize: 300
        resources:
          GPU: 1

    memory_nodes:
        min_workers: 70
        max_workers: 70
        node_config:
            InstanceType: i3.8xlarge
            BlockDeviceMappings:
                - DeviceName: /dev/sda1
                  Ebs:
                      VolumeSize: 300
        resources: { }

head_node_type: head

setup_commands:
    - pip install -U ray ray[default] pyarrow pandas

head_start_ray_commands:
    - ray start --head --port=6379 --object-manager-port=8076 --object-store-memory=90000000000 --autoscaling-config=~/ray_bootstrap_config.yaml 

worker_start_ray_commands:
    - ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=90000000000