diff --git a/python/ray/autoscaler/gcp/config.py b/python/ray/autoscaler/gcp/config.py
index 55870a699..277ebebac 100644
--- a/python/ray/autoscaler/gcp/config.py
+++ b/python/ray/autoscaler/gcp/config.py
@@ -309,6 +309,10 @@ def _configure_key_pair(config, compute):
             _create_project_ssh_key_pair(project, public_key, ssh_user,
                                          compute)
 
+            # Create the directory if it doesn't exists
+            private_key_dir = os.path.dirname(private_key_path)
+            os.makedirs(private_key_dir, exist_ok=True)
+
             # We need to make sure to _create_ the file with the right
             # permissions. In order to do that we need to change the default
             # os.open behavior to include the mode we want.