[tune] Run SigOpt tests in CI (#28225)

.buildkite/pipeline.ml.yml

@@ -282,6 +282,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
+    - python ./ci/env/setup_credentials.py sigopt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
       --test_tag_filters=tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-example,-py37,-soft_imports,-gpu_only,-rllib
       python/ray/tune/...

ci/env/setup_credentials.py

@@ -3,8 +3,10 @@ This script is used to set up credentials for some services in the
 CI environment. For instance, it can fetch WandB API tokens and write
 the WandB configuration file so test scripts can use the service.
 """
+import json
 import os
 import sys
+from pathlib import Path
 
 import boto3
 
@@ -14,6 +16,9 @@ AWS_WANDB_SECRET_ARN = (
 AWS_COMET_SECRET_ARN = (
     "arn:aws:secretsmanager:us-west-2:029272617770:secret:oss-ci/comet-ml-token-vw81C3"
 )
+AWS_SIGOPT_SECRET_ARN = (
+    "arn:aws:secretsmanager:us-west-2:029272617770:secret:oss-ci/sigopt-key-qEqYrk"
+)
 
 
 def get_and_write_wandb_api_key(client):
@@ -28,9 +33,26 @@ def get_and_write_comet_ml_api_key(client):
         fp.write(f"[comet]\napi_key={api_key}\n")
 
 
+def get_and_write_sigopt_api_key(client):
+    api_key = client.get_secret_value(SecretId=AWS_SIGOPT_SECRET_ARN)["SecretString"]
+
+    sigopt_config_file = Path("~/.sigopt/client/config.json").expanduser()
+    sigopt_config_file.parent.mkdir(parents=True, exist_ok=True)
+    with open(sigopt_config_file, "wt") as f:
+        json.dump(
+            {
+                "api_token": api_key,
+                "code_tracking_enabled": False,
+                "log_collection_enabled": False,
+            },
+            f,
+        )
+
+
 SERVICES = {
     "wandb": get_and_write_wandb_api_key,
     "comet_ml": get_and_write_comet_ml_api_key,
+    "sigopt": get_and_write_sigopt_api_key,
 }
 
 

python/ray/tune/examples/sigopt_example.py

@@ -11,6 +11,7 @@ from ray import air, tune
 from ray.air import session
 from ray.tune.schedulers import AsyncHyperBandScheduler
 from ray.tune.search.sigopt import SigOptSearch
+from ray.tune.search.sigopt.sigopt_search import load_sigopt_key
 
 
 def evaluate(step, width, height):
@@ -39,6 +40,8 @@ if __name__ == "__main__":
     )
     args, _ = parser.parse_known_args()
 
+    load_sigopt_key()
+
     if "SIGOPT_KEY" not in os.environ:
         if args.smoke_test:
             print("SigOpt API Key not found. Skipping smoke test.")

python/ray/tune/search/sigopt/__init__.py

@@ -1,3 +1,3 @@
-from ray.tune.search.sigopt.sigopt_search import SigOptSearch
+from ray.tune.search.sigopt.sigopt_search import SigOptSearch, load_sigopt_key
 
-__all__ = ["SigOptSearch"]
+__all__ = ["SigOptSearch", "load_sigopt_key"]

python/ray/tune/search/sigopt/sigopt_search.py

@@ -1,7 +1,9 @@
 import copy
+import json
 import os
 import logging
 import pickle
+from pathlib import Path
 from typing import Dict, List, Optional, Union
 
 try:
@@ -18,6 +20,31 @@ from ray.tune.search import Searcher
 logger = logging.getLogger(__name__)
 
 
+def load_sigopt_key(overwrite: bool = False) -> bool:
+    """Load SigOpt key from config file and save in environment.
+
+    Args:
+        overwrite: If True, will overwrite an existing SIGOPT_KEY env variable.
+
+    Returns:
+        True if a key was loaded into the environment.
+    """
+    if "SIGOPT_KEY" in os.environ and not overwrite:
+        return False
+
+    sigopt_key_file = Path("~/.sigopt/client/config.json").expanduser()
+    if not sigopt_key_file.exists():
+        return False
+
+    try:
+        with open(sigopt_key_file, "rt") as f:
+            config = json.load(f)
+        os.environ["SIGOPT_KEY"] = config["api_token"]
+        return True
+    except Exception:
+        return False
+
+
 class SigOptSearch(Searcher):
     """A wrapper around SigOpt to provide trial suggestions.
 
@@ -184,6 +211,7 @@ class SigOptSearch(Searcher):
             ), """SigOpt must be installed!
                 You can install SigOpt with the command:
                 `pip install -U sigopt`."""
+            load_sigopt_key()
             assert (
                 "SIGOPT_KEY" in os.environ
             ), "SigOpt API key must be stored as environ variable at SIGOPT_KEY"

python/ray/tune/tests/test_tune_restore_warm_start.py

@@ -22,7 +22,7 @@ from ray.tune.search.flaml import CFO, BlendSearch
 from ray.tune.search.skopt import SkOptSearch
 from ray.tune.search.nevergrad import NevergradSearch
 from ray.tune.search.optuna import OptunaSearch
-from ray.tune.search.sigopt import SigOptSearch
+from ray.tune.search.sigopt import SigOptSearch, load_sigopt_key
 from ray.tune.search.zoopt import ZOOptSearch
 from ray.tune.search.hebo import HEBOSearch
 from ray.tune.search.ax import AxSearch
@@ -316,6 +316,10 @@ class DragonflyWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
 
 
 class SigOptWarmStartTest(AbstractWarmStartTest, unittest.TestCase):
+    def setUp(self):
+        AbstractWarmStartTest.setUp(self)
+        load_sigopt_key()
+
     def set_basic_conf(self):
         space = [
             {