"""
yr api config for user
"""
import dataclasses
import json
import logging
from dataclasses import asdict, dataclass, field
from typing import Dict, List, Union, Optional, get_origin, Any
from enum import Enum, IntEnum
from yr.affinity import Affinity
_DEFAULT_CONNECTION_NUMS = 100
_DEFAULT_ENABLE_METRICS = False
_DEFAULT_MAX_TASK_INSTANCE_NUM = -1
_DEFAULT_MAX_CONCURRENCY_CREATE_NUM = 100
_DEFAULT_CONCURRENCY = 1
_DEFAULT_RECYCLE_TIME = 2
_DEFAULT_HTTP_IOC_THREADS_NUM = 400
_DEFAULT_RPC_TIMOUT = 30 * 60
_MAX_INT = 0x7FFFFFFF
_MIN_INT = 0
NPU_RESOURCE_NAME = "NPU"
CPU_RESOURCE_NAME = "CPU"
MEMORY_RESOURCE_NAME = "Memory"
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class UserTLSConfig:
"""
The SSL/TLS configuration used by users when communicating with external clusters.
"""
root_cert_path: str
module_cert_path: str
module_key_path: str
server_name: str = None
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class DeploymentConfig:
"""
AutoDeploymentConfig.
"""
cpu: int = 0
mem: int = 0
datamem: int = 0
spill_path: str = ""
spill_limit: int = 0
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class Config:
"""
YR API config.
"""
function_id: str = ""
cpp_function_id: str = ""
cpp_auto_function_name: str = ""
function_name: str = ""
server_address: str = ""
ds_address: str = ""
is_driver: bool = True
log_level: Union[str, int] = ""
invoke_timeout: int = 900
local_mode: bool = False
code_dir: str = ""
connection_nums: int = _DEFAULT_CONNECTION_NUMS
recycle_time: int = _DEFAULT_RECYCLE_TIME
in_cluster: bool = None
job_id: str = ""
tls_config: UserTLSConfig = None
auto: bool = False
deployment_config: "DeploymentConfig" = None
rt_server_address: str = ""
log_dir: str = "./"
env_file: str = ""
log_file_size_max: int = 0
log_file_num_max: int = 0
log_flush_interval: int = 5
logger: Optional[logging.Logger] = None
runtime_id: str = "driver"
max_task_instance_num: int = _DEFAULT_MAX_TASK_INSTANCE_NUM
load_paths: list = field(default_factory=list)
rpc_timeout: int = _DEFAULT_RPC_TIMOUT
enable_mtls: bool = None
enable_tls: bool = None
private_key_path: str = ""
certificate_file_path: str = ""
verify_file_path: str = ""
private_key_paaswd: str = ""
http_ioc_threads_num: int = _DEFAULT_HTTP_IOC_THREADS_NUM
server_name: str = ""
ns: str = ""
tenant_id: str = ""
enable_metrics: bool = True
bypass_datasystem: Optional[bool] = None
custom_envs: Dict[str, str] = field(default_factory=dict)
master_addr_list: list = field(default_factory=list)
working_dir: str = ""
enable_ds_encrypt: bool = False
ds_public_key_path: str = ""
runtime_public_key_path: str = ""
runtime_private_key_path: str = ""
num_cpus: Optional[int] = None
runtime_env: Optional[Dict[str, Any]] = None
log_to_driver: bool = False
dedup_logs: bool = True
auth_token: str = ""
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class ClientInfo:
"""
Use to store yr client info.
"""
job_id: str
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class Device:
"""
Use to init xpu task
"""
name: str = ""
batch_size: int = 1
class SchedulingAffinityType(IntEnum):
"""
Bundle affinity type.
**Please use ONLY REQUIRED_AFFINITY_IN_EACH_BUNDLE.**
**All other attributes are inherited from IntEnum and should not be used directly.**
"""
REQUIRED_AFFINITY_IN_EACH_BUNDLE = 0
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class ResourceGroupOptions:
"""
Resource group options.
"""
resource_group_name: str = ""
bundle_index: int = -1
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class DebugConfig:
"""
debug instance configurations.
"""
enable: bool = False
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class FunctionGroupOptions:
"""
Function group options.
"""
cpu: Optional[int] = None
memory: Optional[int] = None
resources: Dict[str, float] = field(default_factory=dict)
scheduling_affinity_type: Optional[SchedulingAffinityType] = None
scheduling_affinity_each_bundle_size: Optional[int] = None
timeout: Optional[int] = None
concurrency: Optional[int] = None
recover_retry_times: int = 0
def function_group_enabled(opts: FunctionGroupOptions, group_size: int) -> bool:
"""
if function group enabled.
"""
if opts.scheduling_affinity_each_bundle_size is None:
return False
if opts.scheduling_affinity_each_bundle_size > 0 and (
opts.scheduling_affinity_each_bundle_size <= group_size):
return True
if group_size == 0 and opts.scheduling_affinity_each_bundle_size == 0:
return False
raise RuntimeError("enable function group failed, group_size: {}, bundle_size: {}".format(
group_size,
opts.scheduling_affinity_each_bundle_size))
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class DeviceInfo:
device_id: int = 0
device_ip: str = ""
rank_id: int = 0
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class ServerInfo:
devices: List[DeviceInfo] = field(default_factory=list)
server_id: str = ""
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class FunctionGroupContext:
"""
A context class for managing function group information.
"""
rank_id: int = 0
world_size: int = 0
server_list: List['ServerInfo'] = field(default_factory=list)
device_name: str = ""
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class GroupOptions:
"""
Configuration options for grouped instance scheduling.
The `GroupOptions` structure defines parameters for the lifecycle management of grouped instances, including timeout
settings for rescheduling when kernel resources are insufficient.
"""
timeout: int = -1
same_lifecycle: bool = True
strategy: str = ""
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class PortForwarding:
"""Defines a port to be forwarded into the sandbox.
Attributes:
port: The port number inside the sandbox. Range: [1, 65535].
protocol: The protocol type. Supported: "TCP", "UDP". Default: "TCP".
"""
port: int = 0
protocol: str = "TCP"
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class InvokeOptions:
"""Use to set the invoke options.
Examples:
>>> import yr
>>> import time
>>> yr.init()
>>> opt = yr.InvokeOptions()
>>> opt.pod_labels["k1"] = "v1"
>>> @yr.invoke(invoke_options=opt)
... def func():
... time.sleep(100)
>>> ret = func.invoke()
>>> yr.get(ret)
>>> yr.finalize()
"""
cpu: int = 500
cpu_limit: int = 0
memory: int = 500
mem_limit: int = 0
concurrency: Optional[int] = None
custom_resources: Dict[str, float] = field(default_factory=dict)
custom_extensions: Dict[str, str] = field(default_factory=dict)
"""
Specify user-defined configurations, such as function concurrency.
It can also be used as a user-defined tag for metrics to collect user information.
.. list-table:: Common `custom_extensions` configuration
* - "Concurrency"
- Concurrency. Range: [1,1000].
* - "lifecycle"
- detached, supports detached mode.
* - "DELEGATE_DIRECTORY_INFO"
- Custom directories support the ability to create and delete subdirectories. When an instance is created,
if the user-defined directory exists and has read and write permissions, a subdirectory is created under
it as the working directory; otherwise, a subdirectory is created under the `/tmp` directory as the working
directory. When the instance is destroyed, the working directory is destroyed. The user function can
obtain the working directory through the `INSTANCE_WORK_DIR` environment variable.
* - "DELEGATE_DIRECTORY_QUOTA"
- Subdirectory quota size, value range is greater than ``0 M`` and less than ``1 TB``. If this configuration
is not set, the default is ``512 M``. If the configuration is ``-1``, monitoring is not performed. Unit: MB.
* - "GRACEFUL_SHUTDOWN_TIME"
- Customize the graceful exit time, in seconds. Limit: ``>=0``, ``0`` means immediate exit, and does not
guarantee that the user's graceful exit function can be completed; if configured <0, the system
configuration at deployment time is used as the timeout time.
* - "RECOVER_RETRY_TIMEOUT"
- Customize the recover timeout time. The instance recover timeout time is in milliseconds. Limit: ``>0``,
Default to ``10 * 60 * 1000``
When used as a user-defined tag for metrics:
>>> import yr
>>> yr.init()
>>> opt = yr.InvokeOptions()
>>> opt.custom_extensions["YR_Metrics"] = "{\'endpoint\':\'127.0.0.1\', \'project_id\':\'my_project_id\'}"
In Prometheus, select `metrics name` as `yr_app_instance_billing_invoke_latency`, and you can find the custom tag
information in the collected invoke information:
.. code-block:: text
yr_app_instance_billing_invoke_latency{
...
endpoint="127.0.0.1",
...}
"""
pod_labels: Dict[str, str] = field(default_factory=dict)
"""
Pod labels only used in Kubernetes environment. When creating a function instance, pod_labels can accept key-value
pairs from the user and pass them to the function system.
* After the ActorPattern function instance specialization is completed (Running),
the Scaler applies the incoming labels to the POD.
* When an ActorPattern function instance fails or is deleted,
the Scaler sets the corresponding label of the POD to empty (Remove it);
* Constraints:
* The number of labels that can be stored in `pod_labels` cannot exceed 5.
* Constraints on the key and value in `pod_labels`:
* key:Supports uppercase and lowercase letters, numbers, and hyphens, and allows a length of 1-63.
Does not start or end with a hyphen. Empty strings are not allowed.
* value:Supports uppercase and lowercase letters, numbers, and hyphens, with a length of 1-63.
Does not start or end with a hyphen. Allows empty strings.
* Raises:
When the `pod_labels` passed by the user does not meet the constraints,
the corresponding exception and error message will be thrown.
"""
labels: List[str] = field(default_factory=list)
affinity: Dict[str, str] = field(default_factory=dict)
device: Device = field(default_factory=Device)
max_invoke_latency: int = 5000
min_instances: int = 0
max_instances: int = 0
recover_retry_times: int = 0
need_order: Optional[bool] = None
name: str = ""
namespace: str = ""
schedule_affinities: List[Affinity] = field(default_factory=dict)
is_data_affinity: bool = False
preferred_priority = True
required_priority = False
preferred_anti_other_labels = False
resource_group_options: ResourceGroupOptions = field(
default_factory=ResourceGroupOptions)
"""
Specify the ResourceGroup option, which includes resource_group_name and bundle_index.
When creating a function instance: If `resource_group_name` is set, it will be passed to the kernel to schedule to
the specified ResourceGroup. If both `resource_group_name` and `bundle_index` are set, they are passed to the
kernel to schedule the bundle to the specified ResourceGroup and index. The default value of `resource_group_name`
is empty, and the default value of `bundle_index` is ``-1``.
* Constraints:
* When `resource_group_name` is empty, the instance will not be scheduled to the specified ResourceGroup,
and the bundle_index field is not effective.
* When `resource_group_name` is not empty:
* When `bundle_index` is ``-1``, the instance is scheduled to the specified ResourceGroup.
* When ``0<= bundle_index < number of bundles in ResourceGroup``,schedule the instance to a specified bundle
in a specified ResourceGroup.
* When ``bundle_index < -1`` or ``bundle_index >= number of bundles in ResourceGroup``,raise error.
* Raises:
* There is no ResourceGroup with the `resource_group_name` provided by the user.
* The user passes a non-empty `resource_group_name` and ``bundle_index < -1``.
* When the user passes a non-empty `resource_group_name` and `bundle_index` >= the number of
bundles in the ResourceGroup
* Scheduling failed: For example, the specified ResourceGroup or the specified bundle of the specified
ResourceGroup does not have enough resources to handle instance scheduling.
"""
function_group_options: FunctionGroupOptions = field(
default_factory=FunctionGroupOptions)
env_vars: Dict[str, str] = field(default_factory=dict)
retry_times: int = 0
trace_id: str = ""
alias_params: Dict[str, str] = field(default_factory=dict)
runtime_env: Dict = field(default_factory=dict)
debug: DebugConfig = field(default_factory=DebugConfig)
"""
Configure the stateful/stateless function runtime environment with `conda`, `pip`, `working_dir`, and `env_vars`.
* `conda` provides different Python runtime environments for stateful function.
* Specify an existing conda environment (the environment exists on all nodes)
``runtime_env = {"conda":"pytorch_p39"}``
* Create and use conda environments through configuration.
``runtime_env["conda"] = {"name":"myenv","channels": ["conda-forge"], "dependencies": ["python=3.9",
"msgpack-python=1.0.5", "protobuf", "libgcc-ng", "cloudpickle=3.1.2", "cython=3.0.10", "pyyaml=6.0.2"]}``
* Create and use a conda environment through a YAML file (the YAML file meets the conda requirements).
``runtime_env = {"conda":"/home/env.yaml"}``
* `pip` installs dependencies for Python runtime environment.
* `working_dir` configure the code path of the job.
* `env_vars` configure process-level environment variables.
``runtime_env = {"env_vars":{"OMP_NUM_THREADS": "32", "TF_WARNINGS": "none"}}``
* `shared_dir` supports configuring a shared directory for some instance, with yr managing the lifecycle of this
shared directory. `shared_dir` supports two fields: name and TTL. The name field only allows numbers, letters,
"-", and "_". The TTL supports integers greater than 0 and less than INTMAX.
``runtime_env = {"shared_dir":{"name": "user_define", "TTL": 5}}``
* Constraints of `runtime_env`:
* The keys supported by runtime_env are `conda`, `env_vars`, `pip`, `working_dir`.
Other keys will not take effect and will not cause errors.
* Run the yr function with conda. The environment needs to have yr and its third-party dependencies. It is
recommended that users first create a conda environment and then specify it with `runtime_env`, for example:
``runtime_env = {"conda":"pytorch_p39"}``
* `runtime_env` supports creating and switching conda environments using configurations. The configuration needs
to install third-party dependencies for yr, for example:
``runtime_env["conda"] = {"name":"myenv","channels": ["conda-forge"], "dependencies": ["python=3.9",
"msgpack-python=1.0.5", "protobuf", "libgcc-ng", "cloudpickle=3.1.2", "cython=3.0.10", "pyyaml=6.0.2"]}``
* The environment created using conda in `runtime_env` needs to be cleaned up by the user.
* In `runtime_env`, conda can use `pip` to install dependencies, which are managed directly by conda.
``runtime_env = {"conda":{'name': 'my_project_env', 'channels': ['defaults', 'conda-forge'],
'dependencies': ['python=3.9', {'pip': ['requests==2.25.1']}]}}``
* Currently, Python 3.9 and Python 3.11 SDKs are available. The Python version of conda needs to
be consistent with the SDK version.
* If both `InvokeOptions.env_vars` and `InvokeOptions.runtime_env["env_vars"]` are configured with the same key,
the configuration in `InvokeOptions.env_vars` will be used.
* If `InvokeOptions.runtime_env["working_dir"]` is configured, use this configuration,
otherwise, use `YR.Config.working_dir` and finally use the configuration in `InvokeOptions.env_vars`.
* If you use conda, you need to specify the environment variable `YR_CONDA_HOME` to point to installation path.
* `shared_dir` has the following constraints:
1. It is not recommended to configure different TTL for the same shared directory.
2. The minimum cleanup interval for shared directories is 5 seconds.
3. When multiple yr Agents are deployed on the same node, each Agent must be configured with
different root directory to prevent conflicts in shared directory management.
"""
preempted_allowed: bool = False
instance_priority: int = 0
schedule_timeout_ms: int = 30000
group_name: str = ""
idle_timeout: int = 300
is_delete_remote_tensor: bool = False
bypass_datasystem: bool = False
skip_serialize: bool = False
port_forwardings: List[PortForwarding] = field(default_factory=list)
"""
Configure port forwarding rules for the sandbox. Each entry specifies a port to be forwarded
inside the sandbox environment.
When configured, the port forwarding rules are serialized to JSON and passed to the runtime
via ``createOptions["network"]``. Supports configuring multiple ports simultaneously.
* Constraints:
* ``port``: Must be an integer in the range [1, 65535].
* ``protocol``: Must be ``"TCP"`` or ``"UDP"``. Default is ``"TCP"``.
* Raises:
* ``TypeError``: If ``port`` is not an ``int`` or ``protocol`` is not a ``str``.
* ``ValueError``: If ``port`` is out of range or ``protocol`` is unsupported.
Example::
>>> import yr
>>> yr.init()
>>> opt = yr.InvokeOptions()
>>> opt.port_forwardings = [
... yr.PortForwarding(port=8080),
... yr.PortForwarding(port=9090, protocol="UDP"),
... ]
>>> @yr.invoke(invoke_options=opt)
... def serve():
... pass
The above configuration produces the following JSON in ``createOptions["network"]``::
{"portForwardings": [{"port": 8080, "protocol": "TCP"}, {"port": 9090, "protocol": "UDP"}]}
"""
get_if_exists: bool = False
def check_options_valid(self):
"""
Check whether the options are valid.
Raises:
TypeError: If options are invalid, throw this exception.
"""
attributes_to_check = [
("env_vars", Dict[str, str]),
("name", str),
("namespace", str),
("preferred_anti_other_labels", bool),
("preferred_priority", bool),
("trace_id", str),
("custom_resources", Dict[str, float]),
("custom_extensions", Dict[str, str]),
]
for actual, expected_type in attributes_to_check:
actual_value = getattr(self, actual)
if actual_value is None:
continue
origin = get_origin(expected_type)
check_type = origin if origin is not None else expected_type
if not isinstance(actual_value, check_type):
raise TypeError(
f"invalid type for '{actual}', actual: {type(actual_value)}, expect: {expected_type}"
)
def check_options_range(self):
"""
Check whether the options are in the valid range.
"""
attrs = [
"retry_times",
"recover_retry_times",
"max_instances",
"max_invoke_latency",
"min_instances",
]
for attr in attrs:
value = getattr(self, attr)
if attr in ["max_instances", "max_invoke_latency"] and value == -1:
continue
if not _MIN_INT <= value <= _MAX_INT:
raise ValueError(
f"{attr} 超过范围, 请输入 {_MIN_INT} 到 {_MAX_INT} 范围的值"
)
if self.cpu_limit < 0:
raise ValueError(
f"cpu_limit ({self.cpu_limit}) must be 0 or a positive value"
)
if self.cpu_limit > 0 and self.cpu_limit < self.cpu:
raise ValueError(
f"cpu_limit ({self.cpu_limit}) must be >= cpu ({self.cpu})"
)
if self.mem_limit < 0:
raise ValueError(
f"mem_limit ({self.mem_limit}) must be 0 or a positive value"
)
if self.mem_limit > 0 and self.mem_limit < self.memory:
raise ValueError(
f"mem_limit ({self.mem_limit}) must be >= memory ({self.memory})"
)
def dataclass_from_dict(klass, d):
"""
parse dataclass from dict
"""
try:
field_types = {f.name: f.type for f in dataclasses.fields(klass)}
return klass(**{f: dataclass_from_dict(field_types[f], d[f]) for f in d})
except Exception:
return d
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class MetaFunctionID:
"""
meta function id
"""
cpp: str = ""
python: str = ""
java: str = ""
@dataclass(init=True, repr=False, eq=False, order=False, unsafe_hash=False)
class MetaConfig:
"""
TaskMetadata is used to convey control information
{
"jobID": "xxx",
"codePath": "xx",
"recycleTime": 2,
"maxTaskInstanceNum": -1,
"maxConcurrencyCreateNum": 100,
"enableMetrics": false,
"threadPoolSize": 10,
"functionID": {
"cpp": "xxx",
"python": "xxx",
"java": "xxx"
}
}
"""
jobID: str
codePath: str
recycleTime: int = _DEFAULT_RECYCLE_TIME
maxTaskInstanceNum: int = _DEFAULT_MAX_TASK_INSTANCE_NUM
maxConcurrencyCreateNum: int = _DEFAULT_MAX_CONCURRENCY_CREATE_NUM
enableMetrics: bool = _DEFAULT_ENABLE_METRICS
threadPoolSize: int = 10
functionID: MetaFunctionID = field(default_factory=MetaFunctionID)
functionMasters: list = field(default_factory=list)
@classmethod
def parse(cls, data):
"""
parse TaskMetadata from json or dict
Returns: TaskMetadata object
"""
self = dataclass_from_dict(MetaConfig, json.loads(data))
return self
def to_json(self):
"""
convert to json
Returns: json
"""
return json.dumps(asdict(self))