已合并
[test] 优化测试用例耗时 & 修复bug #178
[test] 优化测试用例耗时 & 修复bug #178
已合并
jason lyu创建于 5月27日
13 个文件变更+127-48
@@ -26,15 +26,16 @@ YAML_DIR=./output_yamls
26 26 
27for yaml_file in "$YAML_DIR"/*.yaml; do27for yaml_file in "$YAML_DIR"/*.yaml; do
28 if [ -f "$yaml_file" ]; then28 if [ -f "$yaml_file" ]; then
29- kubectl delete -f "$yaml_file" -n "$NAMESPACE"29+ kubectl delete -f "$yaml_file"
30 fi30 fi
31done31done
32 32 
33# keep the same with yaml_template/engine_template.yaml terminationGracePeriodSeconds: 1033# keep the same with yaml_template/engine_template.yaml terminationGracePeriodSeconds: 10
34for ((i=10; i>=1; i--)); do34for ((i=10; i>=1; i--)); do
35- echo "Waiting for pods to terminate gracefully... ${i}s remaining"35+ printf "\r\033[KWaiting for pods to terminate gracefully... %2ds remaining" "$i"
36 sleep 136 sleep 1
37done37done
38+echo ""
38 39 
39# Terminating is not a status.phase value; stuck terminating pods have metadata.deletionTimestamp set.40# Terminating is not a status.phase value; stuck terminating pods have metadata.deletionTimestamp set.
40kubectl get pods -n "$NAMESPACE" -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' | while read -r pod; do41kubectl get pods -n "$NAMESPACE" -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' | while read -r pod; do
@@ -143,6 +143,8 @@ ENV_ROLE = "ROLE"
143ENV_JOB_NAME = "JOB_NAME"143ENV_JOB_NAME = "JOB_NAME"
144ENV_CONTROLLER_SERVICE = "CONTROLLER_SERVICE"144ENV_CONTROLLER_SERVICE = "CONTROLLER_SERVICE"
145ENV_COORDINATOR_SERVICE = "COORDINATOR_SERVICE"145ENV_COORDINATOR_SERVICE = "COORDINATOR_SERVICE"
146+ENV_COORDINATOR_INFER_SERVICE = "COORDINATOR_INFER_SERVICE"
147+ENV_COORDINATOR_OBS_SERVICE = "COORDINATOR_OBS_SERVICE"
146ENV_KVP_MASTER_SERVICE = "KVP_MASTER_SERVICE"148ENV_KVP_MASTER_SERVICE = "KVP_MASTER_SERVICE"
147ENV_KV_CONDUCTOR_SERVICE = "KV_CONDUCTOR_SERVICE"149ENV_KV_CONDUCTOR_SERVICE = "KV_CONDUCTOR_SERVICE"
148ENV_KV_POOL_PORT = "KV_POOL_PORT"150ENV_KV_POOL_PORT = "KV_POOL_PORT"
@@ -54,7 +54,9 @@ def modify_controller_deployment(deployment_data, user_config):
54 54 
55 container[C.ENV].extend([55 container[C.ENV].extend([
56 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},56 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},
57- {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}57+ {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service},
58+ {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service},
59+ {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service}
58 ])60 ])
59 61 
60 modify_controller_replicas(deployment_data, user_config)62 modify_controller_replicas(deployment_data, user_config)
@@ -53,7 +53,9 @@ def modify_coordinator_deployment(deployment_data, user_config):
53 53 
54 container[C.ENV].extend([54 container[C.ENV].extend([
55 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},55 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},
56- {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}56+ {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service},
57+ {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service},
58+ {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service}
57 ])59 ])
58 60 
59 if k8s_utils.g_kv_conductor_enabled:61 if k8s_utils.g_kv_conductor_enabled:
@@ -35,7 +35,9 @@ def build_engine_env_items(role, deploy_config, job_name, include_kv_pool=False)
35 {C.NAME: C.ENV_ROLE, C.VALUE: role},35 {C.NAME: C.ENV_ROLE, C.VALUE: role},
36 {C.NAME: C.ENV_JOB_NAME, C.VALUE: job_name},36 {C.NAME: C.ENV_JOB_NAME, C.VALUE: job_name},
37 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},37 {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service},
38- {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}38+ {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service},
39+ {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service},
40+ {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service}
39 ]41 ]
40 if include_kv_pool and k8s_utils.g_kv_pool_enabled:42 if include_kv_pool and k8s_utils.g_kv_pool_enabled:
41 env_items.append({C.NAME: C.ENV_KVP_MASTER_SERVICE, C.VALUE: k8s_utils.g_kv_pool_service})43 env_items.append({C.NAME: C.ENV_KVP_MASTER_SERVICE, C.VALUE: k8s_utils.g_kv_pool_service})
@@ -15,9 +15,9 @@ from lib.utils import (
15)15)
16from lib.generator import k8s_utils16from lib.generator import k8s_utils
17from lib.generator.k8s_utils import (17from lib.generator.k8s_utils import (
18- set_controller_service, set_coordinator_service, set_kv_pool_service,18+ set_controller_service, set_coordinator_service, set_coordinator_infer_service,
19- set_kv_conductor_service, set_rbac_namespace,19+ set_coordinator_obs_service, set_kv_pool_service, set_kv_conductor_service,
20- extract_rbac_resources, apply_sp_block_annotation20+ set_rbac_namespace, extract_rbac_resources, apply_sp_block_annotation
21)21)
22from lib.generator.engine import (22from lib.generator.engine import (
23 build_engine_env_items, set_container_npu, apply_node_selector_by_hardware, set_weight_mount,23 build_engine_env_items, set_container_npu, apply_node_selector_by_hardware, set_weight_mount,
@@ -257,7 +257,7 @@ def generate_yaml_infer_service_set(input_yaml, output_file, user_config):
257 257 
258def init_infer_service_domain_name(infer_service_template_yaml, deploy_config):258def init_infer_service_domain_name(infer_service_template_yaml, deploy_config):
259 """259 """
260- Set g_controller_service and g_coordinator_service for CRD InferServiceSet mode.260+ Set g_controller_service and g_coordinator_*_service for CRD InferServiceSet mode.
261 CRD creates services with naming: {service_name}-{infer_service_set_name}-0-{role_name}261 CRD creates services with naming: {service_name}-{infer_service_set_name}-0-{role_name}
262 """262 """
263 all_docs = load_yaml(infer_service_template_yaml, False)263 all_docs = load_yaml(infer_service_template_yaml, False)
@@ -267,37 +267,47 @@ def init_infer_service_domain_name(infer_service_template_yaml, deploy_config):
267 infer_name = infer_doc.get(C.METADATA, {}).get(C.NAME, "mindie-server")267 infer_name = infer_doc.get(C.METADATA, {}).get(C.NAME, "mindie-server")
268 namespace = deploy_config[C.CONFIG_JOB_ID]268 namespace = deploy_config[C.CONFIG_JOB_ID]
269 269 
270+ def _build_fqdn(service, role_name_val):
271+ service_name = service.get(C.NAME, "")
272+ full_service_name = f"{service_name}-{infer_name}-0-{role_name_val}"
273+ return f"{full_service_name}.{namespace}.svc.cluster.local"
274+ 
270 def get_service_fqdn_for_role(role_name):275 def get_service_fqdn_for_role(role_name):
276+ """Return the first service's FQDN for non-coordinator roles."""
271 role = get_infer_role(infer_doc, role_name)277 role = get_infer_role(infer_doc, role_name)
272 if not role:278 if not role:
273 return None279 return None
274 services = role.get(C.SERVICES, [])280 services = role.get(C.SERVICES, [])
275 if not services:281 if not services:
276 return None282 return None
277- # Pick the management service (port 1026) so the FQDN resolves to the283+ service = services[0]
278- # correct ClusterIP for /readiness and /instances/refresh, even when284+ role_name_val = role.get(C.NAME, role_name)
279- # infer or obs Services appear first in the role's services list.285+ return _build_fqdn(service, role_name_val)
280- service = None286+ 
287+ def get_coordinator_fqdns():
288+ """Return a dict of port->FQDN for the coordinator role's three services."""
289+ role = get_infer_role(infer_doc, C.COORDINATOR)
290+ if not role:
291+ return {}
292+ services = role.get(C.SERVICES, [])
293+ role_name_val = role.get(C.NAME, C.COORDINATOR)
294+ result = {}
281 for svc in services:295 for svc in services:
282 for port_entry in svc.get("spec", {}).get("ports", []):296 for port_entry in svc.get("spec", {}).get("ports", []):
283- if port_entry.get("port") == 1026 or port_entry.get("targetPort") == 1026:297+ port = port_entry.get("port")
284- service = svc298+ if port in (1025, 1026, 1027):
299+ result[port] = _build_fqdn(svc, role_name_val)
285 break300 break
286- if service is not None:301+ return result
287- break
288- if service is None:
289- service = services[0]
290- service_name = service.get(C.NAME, "")
291- role_name_val = role.get(C.NAME, role_name)
292- full_service_name = f"{service_name}-{infer_name}-0-{role_name_val}"
293- return f"{full_service_name}.{namespace}.svc.cluster.local"
294 302 
295 controller_service = get_service_fqdn_for_role(C.CONTROLLER)303 controller_service = get_service_fqdn_for_role(C.CONTROLLER)
296- coordinator_service = get_service_fqdn_for_role(C.COORDINATOR)304+ coord_fqdns = get_coordinator_fqdns()
297- if not controller_service or not coordinator_service:305+ if not controller_service or not coord_fqdns:
298 raise ValueError("Controller or coordinator role not found in infer_service_template.yaml")306 raise ValueError("Controller or coordinator role not found in infer_service_template.yaml")
299 set_controller_service(controller_service)307 set_controller_service(controller_service)
300- set_coordinator_service(coordinator_service)308+ set_coordinator_service(coord_fqdns.get(1026, ""))
309+ set_coordinator_infer_service(coord_fqdns.get(1025, ""))
310+ set_coordinator_obs_service(coord_fqdns.get(1027, ""))
301 311 
302 kv_pool_service = get_service_fqdn_for_role(C.ROLE_KV_POOL)312 kv_pool_service = get_service_fqdn_for_role(C.ROLE_KV_POOL)
303 if kv_pool_service:313 if kv_pool_service:
@@ -16,6 +16,8 @@ from lib.utils import logger, safe_exec_cmd, load_yaml
16 16 
17g_controller_service = "mindie-motor-controller-service"17g_controller_service = "mindie-motor-controller-service"
18g_coordinator_service = "mindie-motor-coordinator-mgmt"18g_coordinator_service = "mindie-motor-coordinator-mgmt"
19+g_coordinator_infer_service = "mindie-motor-coordinator-infer"
20+g_coordinator_obs_service = "mindie-motor-coordinator-obs"
19g_kv_pool_service = "kvp-master"21g_kv_pool_service = "kvp-master"
20g_kv_conductor_service = "kv-conductor"22g_kv_conductor_service = "kv-conductor"
21g_kv_pool_enabled = False23g_kv_pool_enabled = False
@@ -43,6 +45,16 @@ def set_coordinator_service(service_name):
43 g_coordinator_service = service_name45 g_coordinator_service = service_name
44 46 
45 47 
48+def set_coordinator_infer_service(service_name):
49+ global g_coordinator_infer_service
50+ g_coordinator_infer_service = service_name
51+ 
52+ 
53+def set_coordinator_obs_service(service_name):
54+ global g_coordinator_obs_service
55+ g_coordinator_obs_service = service_name
56+ 
57+ 
46def set_kv_pool_service(service_name):58def set_kv_pool_service(service_name):
47 global g_kv_pool_service59 global g_kv_pool_service
48 g_kv_pool_service = service_name60 g_kv_pool_service = service_name
@@ -106,25 +118,24 @@ def get_deploy_mode_from_config(deploy_config):
106 return mode118 return mode
107 119 
108 120 
109-def _pick_mgmt_service(docs: list[dict]):121+def _pick_coordinator_services(docs: list[dict]):
110- """Pick the management Service (port 1026) from a list of YAML docs.122+ """Return a dict mapping port->Service for all coordinator Services.
111 123 
112- When the coordinator template defines multiple Services (infer, mgmt, obs),124+ The coordinator template defines three separate Services:
113- the first Service in document order may not be the management one.125+ - mindie-motor-coordinator-infer (NodePort, port 1025)
114- Select by port (1026) so the COORDINATOR_SERVICE FQDN resolves to the126+ - mindie-motor-coordinator-mgmt (ClusterIP, port 1026)
115- correct ClusterIP that serves /readiness and /instances/refresh.127+ - mindie-motor-coordinator-obs (NodePort, port 1027)
116- Falls back to the first Service if none exposes port 1026.128+ Each is identified by its port for robust matching.
117 """129 """
130+ result = {}
118 for doc in docs:131 for doc in docs:
119 if doc.get(C.KIND) == C.SERVICE:132 if doc.get(C.KIND) == C.SERVICE:
120 for port_entry in doc.get("spec", {}).get("ports", []):133 for port_entry in doc.get("spec", {}).get("ports", []):
121- if port_entry.get("port") == 1026 or port_entry.get("targetPort") == 1026:134+ port = port_entry.get("port")
122- return doc135+ if port in (1025, 1026, 1027):
123- # Fallback: first Service (backward-compatible with single-service templates)136+ result[port] = doc
124- for doc in docs:137+ break
125- if doc.get(C.KIND) == C.SERVICE:138+ return result
126- return doc
127- return None
128 139 
129 140 
130def init_service_domain_name(paths, deploy_config):141def init_service_domain_name(paths, deploy_config):
@@ -140,7 +151,7 @@ def init_service_domain_name(paths, deploy_config):
140 controller_service_data = doc151 controller_service_data = doc
141 break152 break
142 153 
143- coordinator_service_data = _pick_mgmt_service(coordinator_data)154+ coord_services = _pick_coordinator_services(coordinator_data)
144 155 
145 kv_pull_service_data = None156 kv_pull_service_data = None
146 for doc in kv_pool_data:157 for doc in kv_pool_data:
@@ -162,8 +173,18 @@ def init_service_domain_name(paths, deploy_config):
162 173 
163 controller_name = controller_service_data[C.METADATA][C.NAME]174 controller_name = controller_service_data[C.METADATA][C.NAME]
164 set_controller_service(f"{controller_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local")175 set_controller_service(f"{controller_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local")
165- coordinator_name = coordinator_service_data[C.METADATA][C.NAME]176+ 
166- set_coordinator_service(f"{coordinator_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local")177+ ns = deploy_config[C.CONFIG_JOB_ID]
178+ infer_svc = coord_services.get(1025)
179+ mgmt_svc = coord_services.get(1026)
180+ obs_svc = coord_services.get(1027)
181+ if infer_svc:
182+ set_coordinator_infer_service(f"{infer_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local")
183+ if mgmt_svc:
184+ set_coordinator_service(f"{mgmt_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local")
185+ if obs_svc:
186+ set_coordinator_obs_service(f"{obs_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local")
187+ 
167 kv_pool_name = kv_pull_service_data[C.METADATA][C.NAME]188 kv_pool_name = kv_pull_service_data[C.METADATA][C.NAME]
168 set_kv_pool_service(f"{kv_pool_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local")189 set_kv_pool_service(f"{kv_pool_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local")
169 kv_conductor_name = kv_conductor_service_data[C.METADATA][C.NAME]190 kv_conductor_name = kv_conductor_service_data[C.METADATA][C.NAME]
@@ -87,6 +87,20 @@ if [ -n "$COORDINATOR_SERVICE" ]; then
87 echo "Added COORDINATOR_SERVICE: $COORDINATOR_SERVICE"87 echo "Added COORDINATOR_SERVICE: $COORDINATOR_SERVICE"
88fi88fi
89 89 
90+# Add COORDINATOR_INFER_SERVICE if set
91+if [ -n "$COORDINATOR_INFER_SERVICE" ]; then
92+ alt_names_section="${alt_names_section}"$'\n'"DNS.${dns_count} = ${COORDINATOR_INFER_SERVICE}"
93+ dns_count=$((dns_count + 1))
94+ echo "Added COORDINATOR_INFER_SERVICE: $COORDINATOR_INFER_SERVICE"
95+fi
96+ 
97+# Add COORDINATOR_OBS_SERVICE if set
98+if [ -n "$COORDINATOR_OBS_SERVICE" ]; then
99+ alt_names_section="${alt_names_section}"$'\n'"DNS.${dns_count} = ${COORDINATOR_OBS_SERVICE}"
100+ dns_count=$((dns_count + 1))
101+ echo "Added COORDINATOR_OBS_SERVICE: $COORDINATOR_OBS_SERVICE"
102+fi
103+ 
90# Add POD_IP if set104# Add POD_IP if set
91if [ -n "$POD_IP" ]; then105if [ -n "$POD_IP" ]; then
92 alt_names_section="${alt_names_section}"$'\n'"IP.${ip_count} = ${POD_IP}"106 alt_names_section="${alt_names_section}"$'\n'"IP.${ip_count} = ${POD_IP}"
@@ -22,6 +22,7 @@ import base64
22from ccae_reporter.common.logging import Log22from ccae_reporter.common.logging import Log
23from ccae_reporter.config import ConfigUtil23from ccae_reporter.config import ConfigUtil
24from motor.common.http.http_client import SafeHTTPSClient24from motor.common.http.http_client import SafeHTTPSClient
25+from motor.common.utils.env import Env
25from .base_backend import BaseBackend26from .base_backend import BaseBackend
26 27 
27 28 
@@ -40,9 +41,9 @@ class MotorBackend(BaseBackend):
40 self.probe_client = SafeHTTPSClient(address="%s:%d" % (pod_ip, controller_probe_port))41 self.probe_client = SafeHTTPSClient(address="%s:%d" % (pod_ip, controller_probe_port))
41 42 
42 # Coordinator observability API (metrics now served by Coordinator's obs server)43 # Coordinator observability API (metrics now served by Coordinator's obs server)
43- coord_dns = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_api_dns')44+ coord_obs_dns = (Env.coordinator_obs_service or pod_ip or '127.0.0.1')
44 coord_obs_port = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_obs_port')45 coord_obs_port = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_obs_port')
45- self.coord_client = SafeHTTPSClient(address="%s:%d" % (coord_dns, coord_obs_port))46+ self.coord_client = SafeHTTPSClient(address="%s:%d" % (coord_obs_dns, coord_obs_port))
46 47 
47 def fetch_alarm_info(self) -> list:48 def fetch_alarm_info(self) -> list:
48 if not self.is_alive():49 if not self.is_alive():
@@ -33,7 +33,7 @@ class Env:
33 @property33 @property
34 def user_config_path(self):34 def user_config_path(self):
35 return os.getenv("USER_CONFIG_PATH", None)35 return os.getenv("USER_CONFIG_PATH", None)
36- 36+ 
37 @property37 @property
38 def role(self):38 def role(self):
39 return os.getenv("ROLE", None)39 return os.getenv("ROLE", None)
@@ -50,6 +50,20 @@ class Env:
50 def coordinator_service(self):50 def coordinator_service(self):
51 return os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local")51 return os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local")
52 52 
53+ @property
54+ def coordinator_infer_service(self):
55+ return os.getenv(
56+ "COORDINATOR_INFER_SERVICE",
57+ os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local"),
58+ )
59+ 
60+ @property
61+ def coordinator_obs_service(self):
62+ return os.getenv(
63+ "COORDINATOR_OBS_SERVICE",
64+ os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local"),
65+ )
66+ 
53 @property67 @property
54 def controller_service(self):68 def controller_service(self):
55 return os.getenv("CONTROLLER_SERVICE", "mindie-motor-controller-service.mindie-motor.svc.cluster.local")69 return os.getenv("CONTROLLER_SERVICE", "mindie-motor-controller-service.mindie-motor.svc.cluster.local")
@@ -62,4 +76,5 @@ class Env:
62 def disaggregation_bootstrap_port(self):76 def disaggregation_bootstrap_port(self):
63 return os.getenv("DISAGGREGATION_BOOTSTRAP_PORT", "")77 return os.getenv("DISAGGREGATION_BOOTSTRAP_PORT", "")
64 78 
79+ 
65Env = Env()80Env = Env()
@@ -244,6 +244,10 @@ class ApiConfig:
244 # coordinator API configuration244 # coordinator API configuration
245 coordinator_api_host: str = field(default_factory=lambda: Env.pod_ip or '127.0.0.1')245 coordinator_api_host: str = field(default_factory=lambda: Env.pod_ip or '127.0.0.1')
246 coordinator_api_dns: str = field(default_factory=lambda: Env.coordinator_service or '127.0.0.1')246 coordinator_api_dns: str = field(default_factory=lambda: Env.coordinator_service or '127.0.0.1')
247+ coordinator_api_infer_dns: str = field(default_factory=lambda:
248+ Env.coordinator_infer_service or Env.coordinator_service or '127.0.0.1')
249+ coordinator_api_obs_dns: str = field(default_factory=lambda:
250+ Env.coordinator_obs_service or Env.coordinator_service or '127.0.0.1')
247 coordinator_api_infer_port: int = 1025251 coordinator_api_infer_port: int = 1025
248 coordinator_api_mgmt_port: int = 1026252 coordinator_api_mgmt_port: int = 1026
249 coordinator_obs_port: int = 1027253 coordinator_obs_port: int = 1027
@@ -100,5 +100,5 @@ class CoordinatorApiClient:
100 def _generate_obs_client_args(cls) -> dict[str, str]:100 def _generate_obs_client_args(cls) -> dict[str, str]:
101 tls_config = cls.controller_config.mgmt_tls_config101 tls_config = cls.controller_config.mgmt_tls_config
102 api_config = cls.coordinator_config.api_config102 api_config = cls.coordinator_config.api_config
103- address = f"{api_config.coordinator_api_dns}:{api_config.coordinator_obs_port}"103+ address = f"{api_config.coordinator_api_obs_dns}:{api_config.coordinator_obs_port}"
104 return {"address": f"{address}", "tls_config": tls_config}104 return {"address": f"{address}", "tls_config": tls_config}
@@ -591,7 +591,7 @@ class TestCoordinatorServer:
591 # Send many requests to test rate limiting591 # Send many requests to test rate limiting
592 rate_limited = False592 rate_limited = False
593 593 
594- for i in range(150): # Exceed rate limit threshold594+ for i in range(30): # Send multiple requests to verify endpoint stability under load
595 test_data = {595 test_data = {
596 "model": "gpt-3.5-turbo",596 "model": "gpt-3.5-turbo",
597 "messages": [{"role": "user", "content": f"This is the {i + 1}th rate limiting test request"}],597 "messages": [{"role": "user", "content": f"This is the {i + 1}th rate limiting test request"}],
@@ -990,6 +990,11 @@ class TestCoordinatorServerAdvanced:
990 990 
991 # Create test server shell (ManagementServer + InferenceServer)991 # Create test server shell (ManagementServer + InferenceServer)
992 self.coordinator_server = _TestServerShell(config=coordinator_config)992 self.coordinator_server = _TestServerShell(config=coordinator_config)
993+ # Replace scheduler connection with mock to avoid ZMQ connection timeout (~15s per call)
994+ self.coordinator_server._mgmt._scheduler_connection = MagicMock()
995+ self.coordinator_server._mgmt._scheduler_connection.ensure_connected = AsyncMock()
996+ self.coordinator_server._mgmt._scheduler_connection.get_client.return_value = None
997+ self.coordinator_server._mgmt._scheduler_connection.disconnect = AsyncMock()
993 self.coordinator_server.setup_rate_limiting()998 self.coordinator_server.setup_rate_limiting()
994 # Do not mock _handle_openai_request: let real handler run so validation (400), JSON/decode (500), and999 # Do not mock _handle_openai_request: let real handler run so validation (400), JSON/decode (500), and
995 # _is_available (503) are exercised; handle_request is already patched above for 200 responses.1000 # _is_available (503) are exercised; handle_request is already patched above for 200 responses.