已合并
[test] 优化测试用例耗时 & 修复bug #178
jason lyu创建于 5月27日
[test] 优化测试用例耗时 & 修复bug #178
已合并
共 13 个文件变更+127-48
| @@ -26,15 +26,16 @@ YAML_DIR=./output_yamls | |||
| 26 | 26 | ||
| 27 | for yaml_file in "$YAML_DIR"/*.yaml; do | 27 | for yaml_file in "$YAML_DIR"/*.yaml; do |
| 28 | if [ -f "$yaml_file" ]; then | 28 | if [ -f "$yaml_file" ]; then |
| 29 | - kubectl delete -f "$yaml_file" -n "$NAMESPACE" | 29 | + kubectl delete -f "$yaml_file" |
| 30 | fi | 30 | fi |
| 31 | done | 31 | done |
| 32 | 32 | ||
| 33 | # keep the same with yaml_template/engine_template.yaml terminationGracePeriodSeconds: 10 | 33 | # keep the same with yaml_template/engine_template.yaml terminationGracePeriodSeconds: 10 |
| 34 | for ((i=10; i>=1; i--)); do | 34 | for ((i=10; i>=1; i--)); do |
| 35 | - echo "Waiting for pods to terminate gracefully... ${i}s remaining" | 35 | + printf "\r\033[KWaiting for pods to terminate gracefully... %2ds remaining" "$i" |
| 36 | sleep 1 | 36 | sleep 1 |
| 37 | done | 37 | done |
| 38 | +echo "" | ||
| 38 | 39 | ||
| 39 | # Terminating is not a status.phase value; stuck terminating pods have metadata.deletionTimestamp set. | 40 | # Terminating is not a status.phase value; stuck terminating pods have metadata.deletionTimestamp set. |
| 40 | kubectl get pods -n "$NAMESPACE" -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' | while read -r pod; do | 41 | kubectl get pods -n "$NAMESPACE" -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' | while read -r pod; do |
| @@ -143,6 +143,8 @@ ENV_ROLE = "ROLE" | |||
| 143 | ENV_JOB_NAME = "JOB_NAME" | 143 | ENV_JOB_NAME = "JOB_NAME" |
| 144 | ENV_CONTROLLER_SERVICE = "CONTROLLER_SERVICE" | 144 | ENV_CONTROLLER_SERVICE = "CONTROLLER_SERVICE" |
| 145 | ENV_COORDINATOR_SERVICE = "COORDINATOR_SERVICE" | 145 | ENV_COORDINATOR_SERVICE = "COORDINATOR_SERVICE" |
| 146 | +ENV_COORDINATOR_INFER_SERVICE = "COORDINATOR_INFER_SERVICE" | ||
| 147 | +ENV_COORDINATOR_OBS_SERVICE = "COORDINATOR_OBS_SERVICE" | ||
| 146 | ENV_KVP_MASTER_SERVICE = "KVP_MASTER_SERVICE" | 148 | ENV_KVP_MASTER_SERVICE = "KVP_MASTER_SERVICE" |
| 147 | ENV_KV_CONDUCTOR_SERVICE = "KV_CONDUCTOR_SERVICE" | 149 | ENV_KV_CONDUCTOR_SERVICE = "KV_CONDUCTOR_SERVICE" |
| 148 | ENV_KV_POOL_PORT = "KV_POOL_PORT" | 150 | ENV_KV_POOL_PORT = "KV_POOL_PORT" |
| @@ -54,7 +54,9 @@ def modify_controller_deployment(deployment_data, user_config): | |||
| 54 | 54 | ||
| 55 | container[C.ENV].extend([ | 55 | container[C.ENV].extend([ |
| 56 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, | 56 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, |
| 57 | - {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service} | 57 | + {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}, |
| 58 | + {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service}, | ||
| 59 | + {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service} | ||
| 58 | ]) | 60 | ]) |
| 59 | 61 | ||
| 60 | modify_controller_replicas(deployment_data, user_config) | 62 | modify_controller_replicas(deployment_data, user_config) |
| @@ -53,7 +53,9 @@ def modify_coordinator_deployment(deployment_data, user_config): | |||
| 53 | 53 | ||
| 54 | container[C.ENV].extend([ | 54 | container[C.ENV].extend([ |
| 55 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, | 55 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, |
| 56 | - {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service} | 56 | + {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}, |
| 57 | + {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service}, | ||
| 58 | + {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service} | ||
| 57 | ]) | 59 | ]) |
| 58 | 60 | ||
| 59 | if k8s_utils.g_kv_conductor_enabled: | 61 | if k8s_utils.g_kv_conductor_enabled: |
| @@ -35,7 +35,9 @@ def build_engine_env_items(role, deploy_config, job_name, include_kv_pool=False) | |||
| 35 | {C.NAME: C.ENV_ROLE, C.VALUE: role}, | 35 | {C.NAME: C.ENV_ROLE, C.VALUE: role}, |
| 36 | {C.NAME: C.ENV_JOB_NAME, C.VALUE: job_name}, | 36 | {C.NAME: C.ENV_JOB_NAME, C.VALUE: job_name}, |
| 37 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, | 37 | {C.NAME: C.ENV_CONTROLLER_SERVICE, C.VALUE: k8s_utils.g_controller_service}, |
| 38 | - {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service} | 38 | + {C.NAME: C.ENV_COORDINATOR_SERVICE, C.VALUE: k8s_utils.g_coordinator_service}, |
| 39 | + {C.NAME: C.ENV_COORDINATOR_INFER_SERVICE, C.VALUE: k8s_utils.g_coordinator_infer_service}, | ||
| 40 | + {C.NAME: C.ENV_COORDINATOR_OBS_SERVICE, C.VALUE: k8s_utils.g_coordinator_obs_service} | ||
| 39 | ] | 41 | ] |
| 40 | if include_kv_pool and k8s_utils.g_kv_pool_enabled: | 42 | if include_kv_pool and k8s_utils.g_kv_pool_enabled: |
| 41 | env_items.append({C.NAME: C.ENV_KVP_MASTER_SERVICE, C.VALUE: k8s_utils.g_kv_pool_service}) | 43 | env_items.append({C.NAME: C.ENV_KVP_MASTER_SERVICE, C.VALUE: k8s_utils.g_kv_pool_service}) |
| @@ -15,9 +15,9 @@ from lib.utils import ( | |||
| 15 | ) | 15 | ) |
| 16 | from lib.generator import k8s_utils | 16 | from lib.generator import k8s_utils |
| 17 | from lib.generator.k8s_utils import ( | 17 | from lib.generator.k8s_utils import ( |
| 18 | - set_controller_service, set_coordinator_service, set_kv_pool_service, | 18 | + set_controller_service, set_coordinator_service, set_coordinator_infer_service, |
| 19 | - set_kv_conductor_service, set_rbac_namespace, | 19 | + set_coordinator_obs_service, set_kv_pool_service, set_kv_conductor_service, |
| 20 | - extract_rbac_resources, apply_sp_block_annotation | 20 | + set_rbac_namespace, extract_rbac_resources, apply_sp_block_annotation |
| 21 | ) | 21 | ) |
| 22 | from lib.generator.engine import ( | 22 | from lib.generator.engine import ( |
| 23 | build_engine_env_items, set_container_npu, apply_node_selector_by_hardware, set_weight_mount, | 23 | build_engine_env_items, set_container_npu, apply_node_selector_by_hardware, set_weight_mount, |
| @@ -257,7 +257,7 @@ def generate_yaml_infer_service_set(input_yaml, output_file, user_config): | |||
| 257 | 257 | ||
| 258 | def init_infer_service_domain_name(infer_service_template_yaml, deploy_config): | 258 | def init_infer_service_domain_name(infer_service_template_yaml, deploy_config): |
| 259 | """ | 259 | """ |
| 260 | - Set g_controller_service and g_coordinator_service for CRD InferServiceSet mode. | 260 | + Set g_controller_service and g_coordinator_*_service for CRD InferServiceSet mode. |
| 261 | CRD creates services with naming: {service_name}-{infer_service_set_name}-0-{role_name} | 261 | CRD creates services with naming: {service_name}-{infer_service_set_name}-0-{role_name} |
| 262 | """ | 262 | """ |
| 263 | all_docs = load_yaml(infer_service_template_yaml, False) | 263 | all_docs = load_yaml(infer_service_template_yaml, False) |
| @@ -267,37 +267,47 @@ def init_infer_service_domain_name(infer_service_template_yaml, deploy_config): | |||
| 267 | infer_name = infer_doc.get(C.METADATA, {}).get(C.NAME, "mindie-server") | 267 | infer_name = infer_doc.get(C.METADATA, {}).get(C.NAME, "mindie-server") |
| 268 | namespace = deploy_config[C.CONFIG_JOB_ID] | 268 | namespace = deploy_config[C.CONFIG_JOB_ID] |
| 269 | 269 | ||
| 270 | + def _build_fqdn(service, role_name_val): | ||
| 271 | + service_name = service.get(C.NAME, "") | ||
| 272 | + full_service_name = f"{service_name}-{infer_name}-0-{role_name_val}" | ||
| 273 | + return f"{full_service_name}.{namespace}.svc.cluster.local" | ||
| 274 | + | ||
| 270 | def get_service_fqdn_for_role(role_name): | 275 | def get_service_fqdn_for_role(role_name): |
| 276 | + """Return the first service's FQDN for non-coordinator roles.""" | ||
| 271 | role = get_infer_role(infer_doc, role_name) | 277 | role = get_infer_role(infer_doc, role_name) |
| 272 | if not role: | 278 | if not role: |
| 273 | return None | 279 | return None |
| 274 | services = role.get(C.SERVICES, []) | 280 | services = role.get(C.SERVICES, []) |
| 275 | if not services: | 281 | if not services: |
| 276 | return None | 282 | return None |
| 277 | - # Pick the management service (port 1026) so the FQDN resolves to the | 283 | + service = services[0] |
| 278 | - # correct ClusterIP for /readiness and /instances/refresh, even when | 284 | + role_name_val = role.get(C.NAME, role_name) |
| 279 | - # infer or obs Services appear first in the role's services list. | 285 | + return _build_fqdn(service, role_name_val) |
| 280 | - service = None | 286 | + |
| 287 | + def get_coordinator_fqdns(): | ||
| 288 | + """Return a dict of port->FQDN for the coordinator role's three services.""" | ||
| 289 | + role = get_infer_role(infer_doc, C.COORDINATOR) | ||
| 290 | + if not role: | ||
| 291 | + return {} | ||
| 292 | + services = role.get(C.SERVICES, []) | ||
| 293 | + role_name_val = role.get(C.NAME, C.COORDINATOR) | ||
| 294 | + result = {} | ||
| 281 | for svc in services: | 295 | for svc in services: |
| 282 | for port_entry in svc.get("spec", {}).get("ports", []): | 296 | for port_entry in svc.get("spec", {}).get("ports", []): |
| 283 | - if port_entry.get("port") == 1026 or port_entry.get("targetPort") == 1026: | 297 | + port = port_entry.get("port") |
| 284 | - service = svc | 298 | + if port in (1025, 1026, 1027): |
| 299 | + result[port] = _build_fqdn(svc, role_name_val) | ||
| 285 | break | 300 | break |
| 286 | - if service is not None: | 301 | + return result |
| 287 | - break | ||
| 288 | - if service is None: | ||
| 289 | - service = services[0] | ||
| 290 | - service_name = service.get(C.NAME, "") | ||
| 291 | - role_name_val = role.get(C.NAME, role_name) | ||
| 292 | - full_service_name = f"{service_name}-{infer_name}-0-{role_name_val}" | ||
| 293 | - return f"{full_service_name}.{namespace}.svc.cluster.local" | ||
| 294 | 302 | ||
| 295 | controller_service = get_service_fqdn_for_role(C.CONTROLLER) | 303 | controller_service = get_service_fqdn_for_role(C.CONTROLLER) |
| 296 | - coordinator_service = get_service_fqdn_for_role(C.COORDINATOR) | 304 | + coord_fqdns = get_coordinator_fqdns() |
| 297 | - if not controller_service or not coordinator_service: | 305 | + if not controller_service or not coord_fqdns: |
| 298 | raise ValueError("Controller or coordinator role not found in infer_service_template.yaml") | 306 | raise ValueError("Controller or coordinator role not found in infer_service_template.yaml") |
| 299 | set_controller_service(controller_service) | 307 | set_controller_service(controller_service) |
| 300 | - set_coordinator_service(coordinator_service) | 308 | + set_coordinator_service(coord_fqdns.get(1026, "")) |
| 309 | + set_coordinator_infer_service(coord_fqdns.get(1025, "")) | ||
| 310 | + set_coordinator_obs_service(coord_fqdns.get(1027, "")) | ||
| 301 | 311 | ||
| 302 | kv_pool_service = get_service_fqdn_for_role(C.ROLE_KV_POOL) | 312 | kv_pool_service = get_service_fqdn_for_role(C.ROLE_KV_POOL) |
| 303 | if kv_pool_service: | 313 | if kv_pool_service: |
| @@ -16,6 +16,8 @@ from lib.utils import logger, safe_exec_cmd, load_yaml | |||
| 16 | 16 | ||
| 17 | g_controller_service = "mindie-motor-controller-service" | 17 | g_controller_service = "mindie-motor-controller-service" |
| 18 | g_coordinator_service = "mindie-motor-coordinator-mgmt" | 18 | g_coordinator_service = "mindie-motor-coordinator-mgmt" |
| 19 | +g_coordinator_infer_service = "mindie-motor-coordinator-infer" | ||
| 20 | +g_coordinator_obs_service = "mindie-motor-coordinator-obs" | ||
| 19 | g_kv_pool_service = "kvp-master" | 21 | g_kv_pool_service = "kvp-master" |
| 20 | g_kv_conductor_service = "kv-conductor" | 22 | g_kv_conductor_service = "kv-conductor" |
| 21 | g_kv_pool_enabled = False | 23 | g_kv_pool_enabled = False |
| @@ -43,6 +45,16 @@ def set_coordinator_service(service_name): | |||
| 43 | g_coordinator_service = service_name | 45 | g_coordinator_service = service_name |
| 44 | 46 | ||
| 45 | 47 | ||
| 48 | +def set_coordinator_infer_service(service_name): | ||
| 49 | + global g_coordinator_infer_service | ||
| 50 | + g_coordinator_infer_service = service_name | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +def set_coordinator_obs_service(service_name): | ||
| 54 | + global g_coordinator_obs_service | ||
| 55 | + g_coordinator_obs_service = service_name | ||
| 56 | + | ||
| 57 | + | ||
| 46 | def set_kv_pool_service(service_name): | 58 | def set_kv_pool_service(service_name): |
| 47 | global g_kv_pool_service | 59 | global g_kv_pool_service |
| 48 | g_kv_pool_service = service_name | 60 | g_kv_pool_service = service_name |
| @@ -106,25 +118,24 @@ def get_deploy_mode_from_config(deploy_config): | |||
| 106 | return mode | 118 | return mode |
| 107 | 119 | ||
| 108 | 120 | ||
| 109 | -def _pick_mgmt_service(docs: list[dict]): | 121 | +def _pick_coordinator_services(docs: list[dict]): |
| 110 | - """Pick the management Service (port 1026) from a list of YAML docs. | 122 | + """Return a dict mapping port->Service for all coordinator Services. |
| 111 | 123 | ||
| 112 | - When the coordinator template defines multiple Services (infer, mgmt, obs), | 124 | + The coordinator template defines three separate Services: |
| 113 | - the first Service in document order may not be the management one. | 125 | + - mindie-motor-coordinator-infer (NodePort, port 1025) |
| 114 | - Select by port (1026) so the COORDINATOR_SERVICE FQDN resolves to the | 126 | + - mindie-motor-coordinator-mgmt (ClusterIP, port 1026) |
| 115 | - correct ClusterIP that serves /readiness and /instances/refresh. | 127 | + - mindie-motor-coordinator-obs (NodePort, port 1027) |
| 116 | - Falls back to the first Service if none exposes port 1026. | 128 | + Each is identified by its port for robust matching. |
| 117 | """ | 129 | """ |
| 130 | + result = {} | ||
| 118 | for doc in docs: | 131 | for doc in docs: |
| 119 | if doc.get(C.KIND) == C.SERVICE: | 132 | if doc.get(C.KIND) == C.SERVICE: |
| 120 | for port_entry in doc.get("spec", {}).get("ports", []): | 133 | for port_entry in doc.get("spec", {}).get("ports", []): |
| 121 | - if port_entry.get("port") == 1026 or port_entry.get("targetPort") == 1026: | 134 | + port = port_entry.get("port") |
| 122 | - return doc | 135 | + if port in (1025, 1026, 1027): |
| 123 | - # Fallback: first Service (backward-compatible with single-service templates) | 136 | + result[port] = doc |
| 124 | - for doc in docs: | 137 | + break |
| 125 | - if doc.get(C.KIND) == C.SERVICE: | 138 | + return result |
| 126 | - return doc | ||
| 127 | - return None | ||
| 128 | 139 | ||
| 129 | 140 | ||
| 130 | def init_service_domain_name(paths, deploy_config): | 141 | def init_service_domain_name(paths, deploy_config): |
| @@ -140,7 +151,7 @@ def init_service_domain_name(paths, deploy_config): | |||
| 140 | controller_service_data = doc | 151 | controller_service_data = doc |
| 141 | break | 152 | break |
| 142 | 153 | ||
| 143 | - coordinator_service_data = _pick_mgmt_service(coordinator_data) | 154 | + coord_services = _pick_coordinator_services(coordinator_data) |
| 144 | 155 | ||
| 145 | kv_pull_service_data = None | 156 | kv_pull_service_data = None |
| 146 | for doc in kv_pool_data: | 157 | for doc in kv_pool_data: |
| @@ -162,8 +173,18 @@ def init_service_domain_name(paths, deploy_config): | |||
| 162 | 173 | ||
| 163 | controller_name = controller_service_data[C.METADATA][C.NAME] | 174 | controller_name = controller_service_data[C.METADATA][C.NAME] |
| 164 | set_controller_service(f"{controller_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local") | 175 | set_controller_service(f"{controller_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local") |
| 165 | - coordinator_name = coordinator_service_data[C.METADATA][C.NAME] | 176 | + |
| 166 | - set_coordinator_service(f"{coordinator_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local") | 177 | + ns = deploy_config[C.CONFIG_JOB_ID] |
| 178 | + infer_svc = coord_services.get(1025) | ||
| 179 | + mgmt_svc = coord_services.get(1026) | ||
| 180 | + obs_svc = coord_services.get(1027) | ||
| 181 | + if infer_svc: | ||
| 182 | + set_coordinator_infer_service(f"{infer_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local") | ||
| 183 | + if mgmt_svc: | ||
| 184 | + set_coordinator_service(f"{mgmt_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local") | ||
| 185 | + if obs_svc: | ||
| 186 | + set_coordinator_obs_service(f"{obs_svc[C.METADATA][C.NAME]}.{ns}.svc.cluster.local") | ||
| 187 | + | ||
| 167 | kv_pool_name = kv_pull_service_data[C.METADATA][C.NAME] | 188 | kv_pool_name = kv_pull_service_data[C.METADATA][C.NAME] |
| 168 | set_kv_pool_service(f"{kv_pool_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local") | 189 | set_kv_pool_service(f"{kv_pool_name}.{deploy_config[C.CONFIG_JOB_ID]}.svc.cluster.local") |
| 169 | kv_conductor_name = kv_conductor_service_data[C.METADATA][C.NAME] | 190 | kv_conductor_name = kv_conductor_service_data[C.METADATA][C.NAME] |
| @@ -87,6 +87,20 @@ if [ -n "$COORDINATOR_SERVICE" ]; then | |||
| 87 | echo "Added COORDINATOR_SERVICE: $COORDINATOR_SERVICE" | 87 | echo "Added COORDINATOR_SERVICE: $COORDINATOR_SERVICE" |
| 88 | fi | 88 | fi |
| 89 | 89 | ||
| 90 | +# Add COORDINATOR_INFER_SERVICE if set | ||
| 91 | +if [ -n "$COORDINATOR_INFER_SERVICE" ]; then | ||
| 92 | + alt_names_section="${alt_names_section}"$'\n'"DNS.${dns_count} = ${COORDINATOR_INFER_SERVICE}" | ||
| 93 | + dns_count=$((dns_count + 1)) | ||
| 94 | + echo "Added COORDINATOR_INFER_SERVICE: $COORDINATOR_INFER_SERVICE" | ||
| 95 | +fi | ||
| 96 | + | ||
| 97 | +# Add COORDINATOR_OBS_SERVICE if set | ||
| 98 | +if [ -n "$COORDINATOR_OBS_SERVICE" ]; then | ||
| 99 | + alt_names_section="${alt_names_section}"$'\n'"DNS.${dns_count} = ${COORDINATOR_OBS_SERVICE}" | ||
| 100 | + dns_count=$((dns_count + 1)) | ||
| 101 | + echo "Added COORDINATOR_OBS_SERVICE: $COORDINATOR_OBS_SERVICE" | ||
| 102 | +fi | ||
| 103 | + | ||
| 90 | # Add POD_IP if set | 104 | # Add POD_IP if set |
| 91 | if [ -n "$POD_IP" ]; then | 105 | if [ -n "$POD_IP" ]; then |
| 92 | alt_names_section="${alt_names_section}"$'\n'"IP.${ip_count} = ${POD_IP}" | 106 | alt_names_section="${alt_names_section}"$'\n'"IP.${ip_count} = ${POD_IP}" |
| @@ -22,6 +22,7 @@ import base64 | |||
| 22 | from ccae_reporter.common.logging import Log | 22 | from ccae_reporter.common.logging import Log |
| 23 | from ccae_reporter.config import ConfigUtil | 23 | from ccae_reporter.config import ConfigUtil |
| 24 | from motor.common.http.http_client import SafeHTTPSClient | 24 | from motor.common.http.http_client import SafeHTTPSClient |
| 25 | +from motor.common.utils.env import Env | ||
| 25 | from .base_backend import BaseBackend | 26 | from .base_backend import BaseBackend |
| 26 | 27 | ||
| 27 | 28 | ||
| @@ -40,9 +41,9 @@ class MotorBackend(BaseBackend): | |||
| 40 | self.probe_client = SafeHTTPSClient(address="%s:%d" % (pod_ip, controller_probe_port)) | 41 | self.probe_client = SafeHTTPSClient(address="%s:%d" % (pod_ip, controller_probe_port)) |
| 41 | 42 | ||
| 42 | # Coordinator observability API (metrics now served by Coordinator's obs server) | 43 | # Coordinator observability API (metrics now served by Coordinator's obs server) |
| 43 | - coord_dns = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_api_dns') | 44 | + coord_obs_dns = (Env.coordinator_obs_service or pod_ip or '127.0.0.1') |
| 44 | coord_obs_port = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_obs_port') | 45 | coord_obs_port = ConfigUtil.get_config('motor_coordinator_config.api_config.coordinator_obs_port') |
| 45 | - self.coord_client = SafeHTTPSClient(address="%s:%d" % (coord_dns, coord_obs_port)) | 46 | + self.coord_client = SafeHTTPSClient(address="%s:%d" % (coord_obs_dns, coord_obs_port)) |
| 46 | 47 | ||
| 47 | def fetch_alarm_info(self) -> list: | 48 | def fetch_alarm_info(self) -> list: |
| 48 | if not self.is_alive(): | 49 | if not self.is_alive(): |
| @@ -33,7 +33,7 @@ class Env: | |||
| 33 | 33 | ||
| 34 | def user_config_path(self): | 34 | def user_config_path(self): |
| 35 | return os.getenv("USER_CONFIG_PATH", None) | 35 | return os.getenv("USER_CONFIG_PATH", None) |
| 36 | - | 36 | + |
| 37 | 37 | ||
| 38 | def role(self): | 38 | def role(self): |
| 39 | return os.getenv("ROLE", None) | 39 | return os.getenv("ROLE", None) |
| @@ -50,6 +50,20 @@ class Env: | |||
| 50 | def coordinator_service(self): | 50 | def coordinator_service(self): |
| 51 | return os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local") | 51 | return os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local") |
| 52 | 52 | ||
| 53 | + | ||
| 54 | + def coordinator_infer_service(self): | ||
| 55 | + return os.getenv( | ||
| 56 | + "COORDINATOR_INFER_SERVICE", | ||
| 57 | + os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local"), | ||
| 58 | + ) | ||
| 59 | + | ||
| 60 | + | ||
| 61 | + def coordinator_obs_service(self): | ||
| 62 | + return os.getenv( | ||
| 63 | + "COORDINATOR_OBS_SERVICE", | ||
| 64 | + os.getenv("COORDINATOR_SERVICE", "mindie-motor-coordinator-service.mindie-motor.svc.cluster.local"), | ||
| 65 | + ) | ||
| 66 | + | ||
| 53 | 67 | ||
| 54 | def controller_service(self): | 68 | def controller_service(self): |
| 55 | return os.getenv("CONTROLLER_SERVICE", "mindie-motor-controller-service.mindie-motor.svc.cluster.local") | 69 | return os.getenv("CONTROLLER_SERVICE", "mindie-motor-controller-service.mindie-motor.svc.cluster.local") |
| @@ -62,4 +76,5 @@ class Env: | |||
| 62 | def disaggregation_bootstrap_port(self): | 76 | def disaggregation_bootstrap_port(self): |
| 63 | return os.getenv("DISAGGREGATION_BOOTSTRAP_PORT", "") | 77 | return os.getenv("DISAGGREGATION_BOOTSTRAP_PORT", "") |
| 64 | 78 | ||
| 79 | + | ||
| 65 | Env = Env() | 80 | Env = Env() |
| @@ -244,6 +244,10 @@ class ApiConfig: | |||
| 244 | # coordinator API configuration | 244 | # coordinator API configuration |
| 245 | coordinator_api_host: str = field(default_factory=lambda: Env.pod_ip or '127.0.0.1') | 245 | coordinator_api_host: str = field(default_factory=lambda: Env.pod_ip or '127.0.0.1') |
| 246 | coordinator_api_dns: str = field(default_factory=lambda: Env.coordinator_service or '127.0.0.1') | 246 | coordinator_api_dns: str = field(default_factory=lambda: Env.coordinator_service or '127.0.0.1') |
| 247 | + coordinator_api_infer_dns: str = field(default_factory=lambda: | ||
| 248 | + Env.coordinator_infer_service or Env.coordinator_service or '127.0.0.1') | ||
| 249 | + coordinator_api_obs_dns: str = field(default_factory=lambda: | ||
| 250 | + Env.coordinator_obs_service or Env.coordinator_service or '127.0.0.1') | ||
| 247 | coordinator_api_infer_port: int = 1025 | 251 | coordinator_api_infer_port: int = 1025 |
| 248 | coordinator_api_mgmt_port: int = 1026 | 252 | coordinator_api_mgmt_port: int = 1026 |
| 249 | coordinator_obs_port: int = 1027 | 253 | coordinator_obs_port: int = 1027 |
| @@ -100,5 +100,5 @@ class CoordinatorApiClient: | |||
| 100 | def _generate_obs_client_args(cls) -> dict[str, str]: | 100 | def _generate_obs_client_args(cls) -> dict[str, str]: |
| 101 | tls_config = cls.controller_config.mgmt_tls_config | 101 | tls_config = cls.controller_config.mgmt_tls_config |
| 102 | api_config = cls.coordinator_config.api_config | 102 | api_config = cls.coordinator_config.api_config |
| 103 | - address = f"{api_config.coordinator_api_dns}:{api_config.coordinator_obs_port}" | 103 | + address = f"{api_config.coordinator_api_obs_dns}:{api_config.coordinator_obs_port}" |
| 104 | return {"address": f"{address}", "tls_config": tls_config} | 104 | return {"address": f"{address}", "tls_config": tls_config} |
| @@ -591,7 +591,7 @@ class TestCoordinatorServer: | |||
| 591 | # Send many requests to test rate limiting | 591 | # Send many requests to test rate limiting |
| 592 | rate_limited = False | 592 | rate_limited = False |
| 593 | 593 | ||
| 594 | - for i in range(150): # Exceed rate limit threshold | 594 | + for i in range(30): # Send multiple requests to verify endpoint stability under load |
| 595 | test_data = { | 595 | test_data = { |
| 596 | "model": "gpt-3.5-turbo", | 596 | "model": "gpt-3.5-turbo", |
| 597 | "messages": [{"role": "user", "content": f"This is the {i + 1}th rate limiting test request"}], | 597 | "messages": [{"role": "user", "content": f"This is the {i + 1}th rate limiting test request"}], |
| @@ -990,6 +990,11 @@ class TestCoordinatorServerAdvanced: | |||
| 990 | 990 | ||
| 991 | # Create test server shell (ManagementServer + InferenceServer) | 991 | # Create test server shell (ManagementServer + InferenceServer) |
| 992 | self.coordinator_server = _TestServerShell(config=coordinator_config) | 992 | self.coordinator_server = _TestServerShell(config=coordinator_config) |
| 993 | + # Replace scheduler connection with mock to avoid ZMQ connection timeout (~15s per call) | ||
| 994 | + self.coordinator_server._mgmt._scheduler_connection = MagicMock() | ||
| 995 | + self.coordinator_server._mgmt._scheduler_connection.ensure_connected = AsyncMock() | ||
| 996 | + self.coordinator_server._mgmt._scheduler_connection.get_client.return_value = None | ||
| 997 | + self.coordinator_server._mgmt._scheduler_connection.disconnect = AsyncMock() | ||
| 993 | self.coordinator_server.setup_rate_limiting() | 998 | self.coordinator_server.setup_rate_limiting() |
| 994 | # Do not mock _handle_openai_request: let real handler run so validation (400), JSON/decode (500), and | 999 | # Do not mock _handle_openai_request: let real handler run so validation (400), JSON/decode (500), and |
| 995 | # _is_available (503) are exercised; handle_request is already patched above for 200 responses. | 1000 | # _is_available (503) are exercised; handle_request is already patched above for 200 responses. |