[core][autoscaler] Set timeout for requests sent to K8s API server (r…

…ay-project#48513)

Signed-off-by: kaihsun <[email protected]>

python/ray/autoscaler/_private/kuberay/autoscaling_config.py

@@ -82,7 +82,10 @@ def _fetch_ray_cr_from_k8s_with_retries(self) -> Dict[str, Any]:
 
     def _fetch_ray_cr_from_k8s(self) -> Dict[str, Any]:
         result = requests.get(
-            self._ray_cr_url, headers=self._headers, verify=self._verify
+            self._ray_cr_url,
+            headers=self._headers,
+            timeout=node_provider.KUBERAY_REQUEST_TIMEOUT_S,
+            verify=self._verify,
         )
         if not result.status_code == 200:
             result.raise_for_status()

python/ray/autoscaler/_private/kuberay/node_provider.py

@@ -43,6 +43,8 @@
 # KubeRay CRD version
 KUBERAY_CRD_VER = os.getenv("KUBERAY_CRD_VER", "v1alpha1")
 
+KUBERAY_REQUEST_TIMEOUT_S = int(os.getenv("KUBERAY_REQUEST_TIMEOUT_S", 60))
+
 RAY_HEAD_POD_NAME = os.getenv("RAY_HEAD_POD_NAME")
 
 # Key for GKE label that identifies which multi-host replica a pod belongs to
@@ -268,7 +270,12 @@ def get(self, path: str) -> Dict[str, Any]:
             path=path,
             kuberay_crd_version=self._kuberay_crd_version,
         )
-        result = requests.get(url, headers=self._headers, verify=self._verify)
+        result = requests.get(
+            url,
+            headers=self._headers,
+            timeout=KUBERAY_REQUEST_TIMEOUT_S,
+            verify=self._verify,
+        )
         if not result.status_code == 200:
             result.raise_for_status()
         return result.json()