frozenlake-appo-vtrace:
    env: FrozenLake-v1
    run: APPO
    stop:
        episode_reward_mean: 0.99
        timesteps_total: 1000000
    config:
        # Works for both torch and tf.
        framework: tf

        # Sparse reward environment (short horizon).
        env_config:
            desc:
                - SFFFFFFF
                - FFFFFFFF
                - FFFFFFFF
                - FFFFFFFF
                - FFFFFFFF
                - FFFFFFFF
                - FFFFFFFF
                - FFFFFFFG
            is_slippery: false
        horizon: 20
        rollout_fragment_length: 10
        batch_mode: complete_episodes
        vtrace: true
        vtrace_drop_last_ts: false

        num_envs_per_worker: 5
        num_workers: 4
        num_gpus: 0
        num_sgd_iter: 1
        vf_loss_coeff: 0.01