[Serve][Nightly] Environment for Nightly K8s Tests (#27126)

release/k8s_tests/app_config.yaml

@@ -0,0 +1,22 @@
+base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] | default("anyscale/ray:nightly-py37") }}
+env_vars: {}
+debian_packages:
+  - curl
+  - unzip
+  - jq
+  - apt-transport-https
+  - ca-certificates
+  - gnupg
+
+python:
+  pip_packages:
+  - kubernetes
+  conda_packages: []
+
+post_build_cmds:
+  # Install gcloud tools
+  - sudo apt-get remove -y google-cloud-sdk
+  - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+  - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+  - sudo apt-get update && sudo apt-get install google-cloud-cli
+  - sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin

release/k8s_tests/compute_tpl.yaml

@@ -0,0 +1,29 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 0
+
+head_node_type:
+    name: head_node
+    instance_type: m5.2xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: m5.2xlarge
+      min_workers: 0
+      max_workers: 0
+      use_spot: false
+
+aws:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: anyscale-user
+          Value: '{{env["ANYSCALE_USER"]}}'
+        - Key: anyscale-expiration
+          Value: '{{env["EXPIRATION_2D"]}}'
+
+  BlockDeviceMappings:
+    - DeviceName: /dev/sda1
+      Ebs:
+        VolumeSize: 200

release/k8s_tests/prepare.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+aws secretsmanager get-secret-value \
+    --secret-id oss-nightly-ci-gke-cluster-service-account \
+    --region us-west-2 |
+    jq -r .SecretString |
+    jq -r .json_text \
+        >gke_service_account.json
+
+gcloud auth activate-service-account \
+    anyscale-ci@oss-nightly-test.iam.gserviceaccount.com \
+    --key-file=gke_service_account.json
+
+USE_GKE_GCLOUD_AUTH_PLUGIN=True \
+    gcloud container clusters get-credentials \
+    serve-ha-nightly \
+    --zone us-central1-c \
+    --project oss-nightly-test
+
+kubectl get nodes

release/k8s_tests/run_gcs_ft_on_k8s.py

@@ -0,0 +1,10 @@
+from kubernetes import client, config
+
+# Configs can be set in Configuration class directly or using helper utility
+config.load_kube_config()
+
+v1 = client.CoreV1Api()
+print("Listing pods with their IPs:")
+ret = v1.list_pod_for_all_namespaces(watch=False)
+for i in ret.items:
+    print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))

release/release_tests.yaml

@@ -4581,3 +4581,25 @@
       num_nodes: 20
     type: sdk_command
     file_manager: sdk
+
+- name: k8s_serve_ha_test
+  group: k8s-test
+  working_dir: k8s_tests
+  legacy:
+    test_name: k8s_serve_ha_test
+    test_suite: k8s_tests
+
+  stable: false
+
+  frequency: nightly
+  team: serve
+  cluster:
+    cluster_env: app_config.yaml
+    cluster_compute: compute_tpl.yaml
+
+  run:
+    timeout: 1800
+    prepare: bash prepare.sh
+    script: python run_gcs_ft_on_k8s.py
+    type: sdk_command
+    file_manager: sdk