已合并
feat: support docker deploy #50
陈卉创建于 3月30日
feat: support docker deploy #50
已合并
共 15 个文件变更+377-176
| @@ -0,0 +1,4 @@ | |||
| 1 | +[submodule "agent-studio"] | ||
| 2 | + path = agent-studio | ||
| 3 | + url = https://gitcode.com/openJiuwen/agent-studio.git | ||
| 4 | + branch = develop | ||
| @@ -0,0 +1,68 @@ | |||
| 1 | +# Stage 1: Builder environment | ||
| 2 | +ARG BASE_IMAGE | ||
| 3 | +FROM ${BASE_IMAGE} AS builder | ||
| 4 | + | ||
| 5 | +ARG INDEX_URL | ||
| 6 | + | ||
| 7 | +WORKDIR /app | ||
| 8 | +COPY foundation ./foundation | ||
| 9 | +COPY management ./management | ||
| 10 | +COPY server ./server | ||
| 11 | + | ||
| 12 | +RUN sed -i '/openjiuwen-runtime-foundation/d' /app/management/pyproject.toml | ||
| 13 | +RUN sed -i '/openjiuwen-runtime-management/d' /app/server/pyproject.toml | ||
| 14 | + | ||
| 15 | +RUN pip install uv -i ${INDEX_URL} | ||
| 16 | +RUN sed -i "s#url = \"[^\"]*\"#url = \"${INDEX_URL}\"#g" /app/foundation/pyproject.toml | ||
| 17 | +RUN sed -i "s#url = \"[^\"]*\"#url = \"${INDEX_URL}\"#g" /app/management/pyproject.toml | ||
| 18 | +RUN sed -i "s#url = \"[^\"]*\"#url = \"${INDEX_URL}\"#g" /app/server/pyproject.toml | ||
| 19 | + | ||
| 20 | +WORKDIR /app/foundation | ||
| 21 | +RUN uv sync --group dev | ||
| 22 | +RUN uv build --out-dir /app/dist | ||
| 23 | + | ||
| 24 | +WORKDIR /app/management | ||
| 25 | +RUN uv sync --group dev | ||
| 26 | +RUN uv build --out-dir /app/dist | ||
| 27 | + | ||
| 28 | +WORKDIR /app/server | ||
| 29 | +RUN uv sync --group dev | ||
| 30 | +RUN uv build --out-dir /app/dist | ||
| 31 | + | ||
| 32 | +# Stage 2: Final image - Only copy necessary files from builder stage | ||
| 33 | +ARG BASE_IMAGE | ||
| 34 | +FROM ${BASE_IMAGE} AS runtime | ||
| 35 | + | ||
| 36 | +ARG RUNTIME_FOUNDATION_VERSION | ||
| 37 | +ARG RUNTIME_MANAGEMENT_VERSION | ||
| 38 | +ARG RUNTIME_SERVER_VERSION | ||
| 39 | +ARG INDEX_URL | ||
| 40 | + | ||
| 41 | +RUN mkdir -p /app/dist /app/dist2 | ||
| 42 | + | ||
| 43 | +RUN apt-get update && \ | ||
| 44 | + apt-get install -y --no-install-recommends \ | ||
| 45 | + docker.io && \ | ||
| 46 | + rm -rf /var/lib/apt/lists/* | ||
| 47 | + | ||
| 48 | +COPY --from=builder /app/dist/openjiuwen_runtime_foundation-${RUNTIME_FOUNDATION_VERSION}-py3-none-any.whl /app/dist | ||
| 49 | +COPY --from=builder /app/dist/openjiuwen_runtime_management-${RUNTIME_MANAGEMENT_VERSION}-py3-none-any.whl /app/dist | ||
| 50 | +COPY --from=builder /app/dist/agent_runtime_server-${RUNTIME_SERVER_VERSION}-py3-none-any.whl /app/dist | ||
| 51 | + | ||
| 52 | +# 当多个包共用同一个根目录 openjiuwen_runtime，不能用pip install: 因为 pip （旧工具）有一个致命 BUG（针对 --target 安装）当目录已存在时，pip 会直接把整个目录删掉，再重新创建！ | ||
| 53 | +# 必须要用uv（新一代 Python 包管理器），专门修复了这个 bug，安装共享目录包时，只新增 / 覆盖文件，绝不删除整个目录 | ||
| 54 | + | ||
| 55 | +RUN pip install uv -i ${INDEX_URL} | ||
| 56 | +RUN uv pip install /app/dist/openjiuwen_runtime_foundation-${RUNTIME_FOUNDATION_VERSION}-py3-none-any.whl --index-url ${INDEX_URL} --target=/app/site-packages | ||
| 57 | +RUN uv pip install /app/dist/openjiuwen_runtime_management-${RUNTIME_MANAGEMENT_VERSION}-py3-none-any.whl --index-url ${INDEX_URL} --target=/app/site-packages | ||
| 58 | +RUN uv pip install /app/dist/agent_runtime_server-${RUNTIME_SERVER_VERSION}-py3-none-any.whl --index-url ${INDEX_URL} --target=/app/site-packages | ||
| 59 | + | ||
| 60 | +EXPOSE 8186 | ||
| 61 | + | ||
| 62 | +HEALTHCHECK --interval=30s --timeout=30s --start-period=30s --retries=5 \ | ||
| 63 | + CMD curl -f http://localhost:8186/health || exit 1 | ||
| 64 | + | ||
| 65 | +WORKDIR /app/ | ||
| 66 | +ENV PYTHONPATH=/app/site-packages/openjiuwen_runtime:/app/site-packages | ||
| 67 | +CMD ["python", "-m", "openjiuwen_runtime.server.main"] | ||
| 68 | + | ||
| @@ -8,6 +8,8 @@ import os | |||
| 8 | from pathlib import Path | 8 | from pathlib import Path |
| 9 | from typing import Optional | 9 | from typing import Optional |
| 10 | 10 | ||
| 11 | +from .deploy_utils import get_deploy_dir | ||
| 12 | + | ||
| 11 | logger = logging.getLogger(__name__) | 13 | logger = logging.getLogger(__name__) |
| 12 | 14 | ||
| 13 | 15 | ||
| @@ -17,17 +19,18 @@ class VirtualEnvironmentManager: | |||
| 17 | 19 | ||
| 18 | 负责为每个部署创建、管理和清理独立的虚拟环境。 | 20 | 负责为每个部署创建、管理和清理独立的虚拟环境。 |
| 19 | """ | 21 | """ |
| 20 | - | 22 | + def get_venv_path(self, deployment_id: str) -> Path: |
| 21 | - def __init__(self, venvs_root: str = "./venvs"): | ||
| 22 | """ | 23 | """ |
| 23 | - 初始化虚拟环境管理器 | 24 | + 根据部署ID获取虚拟环境路径 |
| 24 | 25 | ||
| 25 | Args: | 26 | Args: |
| 26 | - venvs_root: 虚拟环境根目录路径 | 27 | + deployment_id: 部署ID |
| 28 | + | ||
| 29 | + Returns: | ||
| 30 | + 虚拟环境根目录路径 | ||
| 27 | """ | 31 | """ |
| 28 | - self.venvs_root = Path(venvs_root).resolve() | 32 | + return get_deploy_dir(deployment_id)/".venv" |
| 29 | - self.venvs_root.mkdir(parents=True, exist_ok=True) | 33 | + |
| 30 | - logger.info(f"VirtualEnvironmentManager initialized with root: {self.venvs_root}") | ||
| 31 | 34 | ||
| 32 | def create_venv(self, deployment_id: str) -> Path: | 35 | def create_venv(self, deployment_id: str) -> Path: |
| 33 | """ | 36 | """ |
| @@ -42,7 +45,7 @@ class VirtualEnvironmentManager: | |||
| 42 | Raises: | 45 | Raises: |
| 43 | RuntimeError: 虚拟环境创建失败 | 46 | RuntimeError: 虚拟环境创建失败 |
| 44 | """ | 47 | """ |
| 45 | - venv_path = self.venvs_root / deployment_id | 48 | + venv_path = self.get_venv_path(deployment_id) |
| 46 | 49 | ||
| 47 | if venv_path.exists(): | 50 | if venv_path.exists(): |
| 48 | logger.info(f"Virtual environment already exists: {venv_path}") | 51 | logger.info(f"Virtual environment already exists: {venv_path}") |
| @@ -65,11 +68,11 @@ class VirtualEnvironmentManager: | |||
| 65 | logger.error(f"Virtual environment directory not created: {venv_path}") | 68 | logger.error(f"Virtual environment directory not created: {venv_path}") |
| 66 | raise RuntimeError(f"Failed to create venv: directory not created") | 69 | raise RuntimeError(f"Failed to create venv: directory not created") |
| 67 | 70 | ||
| 71 | + python_exe = self.get_python_executable(deployment_id) | ||
| 72 | + | ||
| 68 | if sys.platform == "win32": | 73 | if sys.platform == "win32": |
| 69 | - python_exe = venv_path / "Scripts" / "python.exe" | ||
| 70 | pip_exe = venv_path / "Scripts" / "pip.exe" | 74 | pip_exe = venv_path / "Scripts" / "pip.exe" |
| 71 | else: | 75 | else: |
| 72 | - python_exe = venv_path / "bin" / "python" | ||
| 73 | pip_exe = venv_path / "bin" / "pip" | 76 | pip_exe = venv_path / "bin" / "pip" |
| 74 | 77 | ||
| 75 | if not python_exe.exists(): | 78 | if not python_exe.exists(): |
| @@ -100,7 +103,6 @@ class VirtualEnvironmentManager: | |||
| 100 | logger.error(f"Failed to create virtual environment: {e.stderr}") | 103 | logger.error(f"Failed to create virtual environment: {e.stderr}") |
| 101 | raise RuntimeError(f"Failed to create venv: {e}") | 104 | raise RuntimeError(f"Failed to create venv: {e}") |
| 102 | 105 | ||
| 103 | - | ||
| 104 | def get_python_executable(self, deployment_id: str) -> Path: | 106 | def get_python_executable(self, deployment_id: str) -> Path: |
| 105 | """ | 107 | """ |
| 106 | 获取虚拟环境中的Python可执行文件路径 | 108 | 获取虚拟环境中的Python可执行文件路径 |
| @@ -114,7 +116,7 @@ class VirtualEnvironmentManager: | |||
| 114 | Raises: | 116 | Raises: |
| 115 | RuntimeError: Python可执行文件未找到 | 117 | RuntimeError: Python可执行文件未找到 |
| 116 | """ | 118 | """ |
| 117 | - venv_path = self.venvs_root / deployment_id | 119 | + venv_path = self.get_venv_path(deployment_id) |
| 118 | 120 | ||
| 119 | if not venv_path.exists(): | 121 | if not venv_path.exists(): |
| 120 | raise RuntimeError(f"Virtual environment not found: {venv_path}") | 122 | raise RuntimeError(f"Virtual environment not found: {venv_path}") |
| @@ -131,6 +133,7 @@ class VirtualEnvironmentManager: | |||
| 131 | 133 | ||
| 132 | return python_path | 134 | return python_path |
| 133 | 135 | ||
| 136 | + | ||
| 134 | def install_whl(self, deployment_id: str, whl_path: str) -> bool: | 137 | def install_whl(self, deployment_id: str, whl_path: str) -> bool: |
| 135 | """ | 138 | """ |
| 136 | 在虚拟环境中安装WHL包 | 139 | 在虚拟环境中安装WHL包 |
| @@ -159,7 +162,7 @@ class VirtualEnvironmentManager: | |||
| 159 | 162 | ||
| 160 | try: | 163 | try: |
| 161 | env = os.environ.copy() | 164 | env = os.environ.copy() |
| 162 | - env["VIRTUAL_ENV"] = str(self.venvs_root / deployment_id) | 165 | + env["VIRTUAL_ENV"] = str(self.get_venv_path(deployment_id)) |
| 163 | result = subprocess.run( | 166 | result = subprocess.run( |
| 164 | cmd, | 167 | cmd, |
| 165 | env=env, | 168 | env=env, |
| @@ -207,7 +210,7 @@ class VirtualEnvironmentManager: | |||
| 207 | Returns: | 210 | Returns: |
| 208 | 是否删除成功 | 211 | 是否删除成功 |
| 209 | """ | 212 | """ |
| 210 | - venv_path = self.venvs_root / deployment_id | 213 | + venv_path = self.get_venv_path(deployment_id) |
| 211 | 214 | ||
| 212 | if not venv_path.exists(): | 215 | if not venv_path.exists(): |
| 213 | logger.warning(f"Virtual environment not found: {venv_path}") | 216 | logger.warning(f"Virtual environment not found: {venv_path}") |
| @@ -222,16 +225,3 @@ class VirtualEnvironmentManager: | |||
| 222 | except Exception as e: | 225 | except Exception as e: |
| 223 | logger.error(f"Failed to delete virtual environment: {e}") | 226 | logger.error(f"Failed to delete virtual environment: {e}") |
| 224 | return False | 227 | return False |
| 225 | - | ||
| 226 | - def venv_exists(self, deployment_id: str) -> bool: | ||
| 227 | - """ | ||
| 228 | - 检查虚拟环境是否存在 | ||
| 229 | - | ||
| 230 | - Args: | ||
| 231 | - deployment_id: 部署ID | ||
| 232 | - | ||
| 233 | - Returns: | ||
| 234 | - 虚拟环境是否存在 | ||
| 235 | - """ | ||
| 236 | - venv_path = self.venvs_root / deployment_id | ||
| 237 | - return venv_path.exists() | ||
| @@ -29,7 +29,6 @@ dependencies = [ | |||
| 29 | 29 | ||
| 30 | [tool.uv] | 30 | [tool.uv] |
| 31 | default-groups = ['dev'] | 31 | default-groups = ['dev'] |
| 32 | -pip = true | ||
| 33 | 32 | ||
| 34 | [dependency-groups] | 33 | [dependency-groups] |
| 35 | dev = [ | 34 | dev = [ |
| @@ -11,6 +11,7 @@ import asyncio | |||
| 11 | import logging | 11 | import logging |
| 12 | import os | 12 | import os |
| 13 | import subprocess | 13 | import subprocess |
| 14 | +import shutil | ||
| 14 | from pathlib import Path | 15 | from pathlib import Path |
| 15 | from typing import Optional | 16 | from typing import Optional |
| 16 | 17 | ||
| @@ -19,6 +20,8 @@ from ..base.models import DeployContext, DeployResult | |||
| 19 | from .models import DockerParams | 20 | from .models import DockerParams |
| 20 | from ...models.enums import DeploymentStatus | 21 | from ...models.enums import DeploymentStatus |
| 21 | 22 | ||
| 23 | +from openjiuwen_runtime.foundation.deploy_utils import get_deploy_dir | ||
| 24 | + | ||
| 22 | logger = logging.getLogger(__name__) | 25 | logger = logging.getLogger(__name__) |
| 23 | 26 | ||
| 24 | 27 | ||
| @@ -29,14 +32,11 @@ class DockerDeployer(Deployer[DockerParams]): | |||
| 29 | self, | 32 | self, |
| 30 | default_host: str = "localhost", | 33 | default_host: str = "localhost", |
| 31 | docker_host: Optional[str] = None, | 34 | docker_host: Optional[str] = None, |
| 32 | - deploy_dir: Optional[str] = None, | ||
| 33 | ): | 35 | ): |
| 34 | self.default_host = default_host | 36 | self.default_host = default_host |
| 35 | self.docker_host = docker_host | 37 | self.docker_host = docker_host |
| 36 | - self.deploy_dir = Path(deploy_dir or os.getenv("DEPLOY_DIR", "./.deploys")).resolve() | ||
| 37 | - self.deploy_dir.mkdir(parents=True, exist_ok=True) | ||
| 38 | self._containers: dict[str, str] = {} | 38 | self._containers: dict[str, str] = {} |
| 39 | - logger.debug(f"DockerDeployer initialized: docker_host={docker_host}, deploy_dir={self.deploy_dir}") | 39 | + logger.debug(f"DockerDeployer initialized: docker_host={docker_host}") |
| 40 | 40 | ||
| 41 | def _build_docker_command(self, *args: str) -> list[str]: | 41 | def _build_docker_command(self, *args: str) -> list[str]: |
| 42 | """构建 Docker 命令""" | 42 | """构建 Docker 命令""" |
| @@ -62,44 +62,70 @@ class DockerDeployer(Deployer[DockerParams]): | |||
| 62 | return True, stdout.decode().strip() | 62 | return True, stdout.decode().strip() |
| 63 | return False, stderr.decode().strip() | 63 | return False, stderr.decode().strip() |
| 64 | 64 | ||
| 65 | - def _generate_dockerfile(self, whl_path: str, package_name: str, port: str) -> str: | ||
| 66 | - """生成 Dockerfile""" | ||
| 67 | - whl_name = Path(whl_path).name | ||
| 68 | - base_image =os.getenv("BASE_IMAGE", "swr.cn-north-4.myhuaweicloud.com/openjiuwen/studio-python-tool-amd64:0.1.0") | ||
| 69 | - logger.debug(f"whl_name: {whl_name}") | ||
| 70 | 65 | ||
| 71 | - return f"""ARG BASE_IMAGE | 66 | + async def deploy_lowcode_container( |
| 72 | -FROM {base_image} | 67 | + self, |
| 68 | + container_name: str, | ||
| 69 | + env_vars: dict, | ||
| 70 | + volumes: list, | ||
| 71 | + port: str, | ||
| 72 | + ir_source_path: str | ||
| 73 | + ) -> str: | ||
| 74 | + """ | ||
| 75 | + 创建低代码Agent容器：采用 docker create → docker cp → docker start 流程。 | ||
| 73 | 76 | ||
| 74 | -ARG WHL_FILE="{whl_name}" | 77 | + 规避问题： |
| 75 | -ARG PACKAGE_NAME="{package_name}" | 78 | + 禁止使用 docker run -v 挂载容器内文件。 |
| 79 | + 因如果当前服务运行于容器中，需要挂载/var/run/docker.sock， | ||
| 80 | + 所有挂载路径均指向宿主机，直接挂载容器内文件会导致目标文件被创建为空目录。 | ||
| 81 | + """ | ||
| 82 | + # 创建容器（不启动） | ||
| 83 | + create_args = ["create", "--name", container_name] | ||
| 76 | 84 | ||
| 77 | -RUN mkdir -p /app/dist | 85 | + # 环境变量 |
| 78 | -RUN useradd --create-home --shell /bin/bash app | 86 | + if env_vars: |
| 79 | -RUN chown -R app:app /app | 87 | + for k, v in env_vars.items(): |
| 88 | + create_args.extend(["-e", f"{k}={v}"]) | ||
| 80 | 89 | ||
| 81 | -USER app | 90 | + # 挂载卷 |
| 82 | -# Copy Python packages from builder stage | 91 | + if volumes: |
| 83 | -COPY {whl_name} /app/dist | 92 | + for vol in volumes: |
| 84 | -RUN pip3 install /app/dist/{whl_name} --target=/app/site-packages --retries=5 --timeout=120 | 93 | + if "source" in vol and "target" in vol: |
| 94 | + create_args.extend(["-v", f"{vol['source']}:{vol['target']}"]) | ||
| 85 | 95 | ||
| 86 | -ENV PYTHONPATH=/app/site-packages | 96 | + create_args.extend(["-p", port]) |
| 87 | -ENV PACKAGE_NAME={package_name} | ||
| 88 | 97 | ||
| 89 | -WORKDIR /app | 98 | + # 镜像 |
| 99 | + image_name = os.getenv("LOWCODE_IMAGE") | ||
| 100 | + if not image_name: | ||
| 101 | + raise RuntimeError("Environment variable LOWCODE_IMAGE is not set") | ||
| 102 | + create_args.append(image_name) | ||
| 90 | 103 | ||
| 91 | -# Start the application | 104 | + logger.debug(f"Creating lowcode agent container: {container_name}") |
| 92 | -CMD python -m {package_name} --host 0.0.0.0 --port {port} | 105 | + success, output = await self._run_docker_command(*create_args) |
| 93 | -""" | 106 | + if not success: |
| 107 | + raise RuntimeError(f"create lowcode container failed: {output}") | ||
| 108 | + | ||
| 109 | + container_id = output.strip() | ||
| 110 | + logger.debug(f"container created: {container_id}") | ||
| 111 | + | ||
| 112 | + # 复制 ir.json 到容器 | ||
| 113 | + target_path = f"{container_name}:/app/ir.json" | ||
| 114 | + logger.debug(f"docker cp {ir_source_path} -> {target_path}") | ||
| 115 | + success, cp_out = await self._run_docker_command("cp", ir_source_path, target_path) | ||
| 116 | + if not success: | ||
| 117 | + raise RuntimeError(f"docker cp failed: {cp_out}") | ||
| 118 | + | ||
| 119 | + # 启动容器 | ||
| 120 | + logger.debug(f"Start container: {container_name}") | ||
| 121 | + success, start_out = await self._run_docker_command("start", container_id) | ||
| 122 | + if not success: | ||
| 123 | + raise RuntimeError(f"Start container failed: {start_out}") | ||
| 124 | + | ||
| 125 | + return container_id | ||
| 94 | 126 | ||
| 95 | async def deploy(self, ctx: DeployContext[DockerParams]) -> DeployResult: | 127 | async def deploy(self, ctx: DeployContext[DockerParams]) -> DeployResult: |
| 96 | """使用 Docker 容器部署应用 | 128 | """使用 Docker 容器部署应用 |
| 97 | - | ||
| 98 | - 流程: | ||
| 99 | - 1. 生成 Dockerfile | ||
| 100 | - 2. 构建 Docker 镜像 | ||
| 101 | - 3. 运行 Docker 容器 | ||
| 102 | - | ||
| 103 | Args: | 129 | Args: |
| 104 | ctx: 部署上下文参数 | 130 | ctx: 部署上下文参数 |
| 105 | 131 | ||
| @@ -112,6 +138,7 @@ CMD python -m {package_name} --host 0.0.0.0 --port {port} | |||
| 112 | try: | 138 | try: |
| 113 | docker_params = ctx.params or DockerParams() | 139 | docker_params = ctx.params or DockerParams() |
| 114 | whl_path = docker_params.whl_path | 140 | whl_path = docker_params.whl_path |
| 141 | + ir_path = docker_params.ir_path | ||
| 115 | package_name = docker_params.package_name | 142 | package_name = docker_params.package_name |
| 116 | container_name = docker_params.container_name or f"deploy_{deployment_id}" | 143 | container_name = docker_params.container_name or f"deploy_{deployment_id}" |
| 117 | env_vars = docker_params.env_vars | 144 | env_vars = docker_params.env_vars |
| @@ -119,72 +146,24 @@ CMD python -m {package_name} --host 0.0.0.0 --port {port} | |||
| 119 | host = ctx.host or self.default_host | 146 | host = ctx.host or self.default_host |
| 120 | iport = "8090" | 147 | iport = "8090" |
| 121 | 148 | ||
| 122 | - if not whl_path: | 149 | + import json |
| 123 | - raise RuntimeError("whl_path is required for docker deployment") | 150 | + logger.debug(f"docker_params 完整内容: \n{json.dumps(docker_params.__dict__, indent=2, ensure_ascii=False)}") |
| 124 | - if not package_name: | ||
| 125 | - raise RuntimeError("package_name is required for docker deployment") | ||
| 126 | 151 | ||
| 127 | - # 1. 生成 Dockerfile | 152 | + # 非低码情况 |
| 128 | - dockerfile_content = self._generate_dockerfile(whl_path, package_name, iport) | 153 | + if not ir_path: |
| 154 | + if not whl_path: | ||
| 155 | + raise RuntimeError("whl_path is required for docker deployment") | ||
| 156 | + if not package_name: | ||
| 157 | + raise RuntimeError("package_name is required for docker deployment") | ||
| 129 | 158 | ||
| 130 | - # 使用指定目录：DEPLOY_DIR/deployment_id/ | 159 | + # 创建并启动低代码Agent容器 |
| 131 | - deploy_context_dir = self.deploy_dir / deployment_id | 160 | + container_id = await self.deploy_lowcode_container( |
| 132 | - deploy_context_dir.mkdir(parents=True, exist_ok=True) | 161 | + container_name=container_name, |
| 133 | - logger.debug(f"Using deploy context directory: {deploy_context_dir}") | 162 | + env_vars=env_vars, |
| 134 | - | 163 | + volumes=volumes, |
| 135 | - dockerfile_path = deploy_context_dir / "Dockerfile" | 164 | + port=iport, |
| 136 | - dockerfile_path.write_text(dockerfile_content) | 165 | + ir_source_path=ir_path |
| 137 | - | ||
| 138 | - # 复制 WHL 文件到部署目录 | ||
| 139 | - whl_file = Path(whl_path) | ||
| 140 | - if whl_file.exists(): | ||
| 141 | - import shutil | ||
| 142 | - shutil.copy(whl_path, deploy_context_dir) | ||
| 143 | - else: | ||
| 144 | - raise RuntimeError(f"WHL file not found: {whl_path}") | ||
| 145 | - | ||
| 146 | - # 2. 构建 Docker 镜像 | ||
| 147 | - image_name = f"{deployment_id}:latest" | ||
| 148 | - logger.debug(f"Building docker image: image_name={image_name}") | ||
| 149 | - | ||
| 150 | - success, output = await self._run_docker_command( | ||
| 151 | - "build", "-t", image_name, str(deploy_context_dir) | ||
| 152 | ) | 166 | ) |
| 153 | - | ||
| 154 | - if not success: | ||
| 155 | - logger.error(f"Docker build failed: deployment_id={deployment_id}, error={output}") | ||
| 156 | - return DeployResult( | ||
| 157 | - success=False, | ||
| 158 | - deployment_id=deployment_id, | ||
| 159 | - message=f"Docker build failed: {output}" | ||
| 160 | - ) | ||
| 161 | - | ||
| 162 | - # 3. 运行 Docker 容器 | ||
| 163 | - run_args = ["run", "-d", "--name", container_name] | ||
| 164 | - | ||
| 165 | - if env_vars: | ||
| 166 | - for key, value in env_vars.items(): | ||
| 167 | - run_args.extend(["-e", f"{key}={value}"]) | ||
| 168 | - | ||
| 169 | - if volumes: | ||
| 170 | - for vol in volumes: | ||
| 171 | - if "source" in vol and "target" in vol: | ||
| 172 | - run_args.extend(["-v", f"{vol['source']}:{vol['target']}"]) | ||
| 173 | - | ||
| 174 | - run_args.extend(["-p", iport, image_name]) | ||
| 175 | - | ||
| 176 | - logger.debug(f"Running docker container: container_name={container_name}, image={image_name}") | ||
| 177 | - success, output = await self._run_docker_command(*run_args) | ||
| 178 | - | ||
| 179 | - if not success: | ||
| 180 | - logger.error(f"Docker run failed: deployment_id={deployment_id}, error={output}") | ||
| 181 | - return DeployResult( | ||
| 182 | - success=False, | ||
| 183 | - deployment_id=deployment_id, | ||
| 184 | - message=f"Docker run failed: {output}" | ||
| 185 | - ) | ||
| 186 | - | ||
| 187 | - container_id = output | ||
| 188 | self._containers[deployment_id] = container_id | 167 | self._containers[deployment_id] = container_id |
| 189 | 168 | ||
| 190 | # 获取该容器在宿主机上的port | 169 | # 获取该容器在宿主机上的port |
| @@ -239,20 +218,18 @@ CMD python -m {package_name} --host 0.0.0.0 --port {port} | |||
| 239 | logger.warning(f"Docker rm failed: deployment_id={deployment_id}, error={output}") | 218 | logger.warning(f"Docker rm failed: deployment_id={deployment_id}, error={output}") |
| 240 | 219 | ||
| 241 | # 删除镜像 | 220 | # 删除镜像 |
| 242 | - image_name = f"{deployment_id}:latest" | 221 | + # image_name = f"{deployment_id}:latest" |
| 243 | - success, output = await self._run_docker_command("rmi", image_name) | 222 | + # success, output = await self._run_docker_command("rmi", image_name) |
| 244 | - if not success: | 223 | + # if not success: |
| 245 | - logger.warning(f"Docker rmi failed: deployment_id={deployment_id}, error={output}") | 224 | + # logger.warning(f"Docker rmi failed: deployment_id={deployment_id}, error={output}") |
| 246 | 225 | ||
| 247 | if deployment_id in self._containers: | 226 | if deployment_id in self._containers: |
| 248 | del self._containers[deployment_id] | 227 | del self._containers[deployment_id] |
| 249 | 228 | ||
| 250 | # 清理部署目录 | 229 | # 清理部署目录 |
| 251 | - deploy_context_dir = self.deploy_dir / deployment_id | 230 | + deploy_context_dir = get_deploy_dir(deployment_id) |
| 252 | - if deploy_context_dir.exists(): | 231 | + logger.debug(f"Cleaning deploy directory: {deploy_context_dir}") |
| 253 | - import shutil | 232 | + shutil.rmtree(deploy_context_dir, ignore_errors=True) |
| 254 | - logger.debug(f"Cleaning deploy directory: {deploy_context_dir}") | ||
| 255 | - shutil.rmtree(deploy_context_dir, ignore_errors=True) | ||
| 256 | 233 | ||
| 257 | logger.info(f"Docker stopped: deployment_id={deployment_id}") | 234 | logger.info(f"Docker stopped: deployment_id={deployment_id}") |
| 258 | return DeployResult( | 235 | return DeployResult( |
| @@ -17,6 +17,7 @@ class DockerParams: | |||
| 17 | env_vars: Optional[dict[str, str]] = None | 17 | env_vars: Optional[dict[str, str]] = None |
| 18 | volumes: Optional[list[dict[str, str]]] = None | 18 | volumes: Optional[list[dict[str, str]]] = None |
| 19 | whl_path: Optional[str] = None | 19 | whl_path: Optional[str] = None |
| 20 | + ir_path: Optional[str] = None | ||
| 20 | package_name: Optional[str] = None | 21 | package_name: Optional[str] = None |
| 21 | port: Optional[int] = None | 22 | port: Optional[int] = None |
| 22 | 23 | ||
| @@ -33,6 +34,7 @@ class DockerInfo(BaseModel): | |||
| 33 | container_name: Optional[str] = None | 34 | container_name: Optional[str] = None |
| 34 | image: Optional[str] = None | 35 | image: Optional[str] = None |
| 35 | whl_path: Optional[str] = None | 36 | whl_path: Optional[str] = None |
| 37 | + ir_path: Optional[str] = None | ||
| 36 | package_name: Optional[str] = None | 38 | package_name: Optional[str] = None |
| 37 | created_at: Optional[datetime] = None | 39 | created_at: Optional[datetime] = None |
| 38 | updated_at: Optional[datetime] = None | 40 | updated_at: Optional[datetime] = None |
| @@ -53,6 +55,7 @@ class DockerCreate(BaseModel): | |||
| 53 | container_name: Optional[str] = Field(None, description="容器名称") | 55 | container_name: Optional[str] = Field(None, description="容器名称") |
| 54 | image: Optional[str] = Field(None, description="镜像名称") | 56 | image: Optional[str] = Field(None, description="镜像名称") |
| 55 | whl_path: Optional[str] = Field(None, description="WHL包路径") | 57 | whl_path: Optional[str] = Field(None, description="WHL包路径") |
| 58 | + ir_path: Optional[str] = Field(None, description="IR文件路径") | ||
| 56 | package_name: Optional[str] = Field(None, description="包名称") | 59 | package_name: Optional[str] = Field(None, description="包名称") |
| 57 | data: Optional[dict[str, Any]] = Field(None, description="扩展数据") | 60 | data: Optional[dict[str, Any]] = Field(None, description="扩展数据") |
| 58 | 61 | ||
| @@ -70,6 +73,7 @@ DOCKER_TABLE_DEF = TableDefinition( | |||
| 70 | ColumnDefinition("container_name", "string", length=255, nullable=True), | 73 | ColumnDefinition("container_name", "string", length=255, nullable=True), |
| 71 | ColumnDefinition("image", "string", length=512, nullable=True), | 74 | ColumnDefinition("image", "string", length=512, nullable=True), |
| 72 | ColumnDefinition("whl_path", "string", length=512, nullable=True), | 75 | ColumnDefinition("whl_path", "string", length=512, nullable=True), |
| 76 | + ColumnDefinition("ir_path", "string", length=512, nullable=True), | ||
| 73 | ColumnDefinition("package_name", "string", length=255, nullable=True), | 77 | ColumnDefinition("package_name", "string", length=255, nullable=True), |
| 74 | ColumnDefinition("created_at", "datetime", nullable=False), | 78 | ColumnDefinition("created_at", "datetime", nullable=False), |
| 75 | ColumnDefinition("updated_at", "datetime", nullable=False), | 79 | ColumnDefinition("updated_at", "datetime", nullable=False), |
| @@ -14,13 +14,16 @@ import shutil | |||
| 14 | import subprocess | 14 | import subprocess |
| 15 | import sys | 15 | import sys |
| 16 | from typing import Dict, Optional | 16 | from typing import Dict, Optional |
| 17 | +from pathlib import Path | ||
| 17 | 18 | ||
| 18 | from ..base.deployer import Deployer | 19 | from ..base.deployer import Deployer |
| 19 | from ..base.models import DeployContext, DeployResult | 20 | from ..base.models import DeployContext, DeployResult |
| 20 | from .models import SubprocessParams | 21 | from .models import SubprocessParams |
| 21 | -from openjiuwen_runtime.foundation.venv_manager import VirtualEnvironmentManager | ||
| 22 | from ...models.enums import DeploymentStatus | 22 | from ...models.enums import DeploymentStatus |
| 23 | 23 | ||
| 24 | +from openjiuwen_runtime.foundation.venv_manager import VirtualEnvironmentManager | ||
| 25 | +from openjiuwen_runtime.foundation.deploy_utils import get_deploy_dir, get_dist_dir | ||
| 26 | + | ||
| 24 | logger = logging.getLogger(__name__) | 27 | logger = logging.getLogger(__name__) |
| 25 | 28 | ||
| 26 | 29 | ||
| @@ -31,14 +34,11 @@ class LocalSubprocessDeployer(Deployer[SubprocessParams]): | |||
| 31 | self, | 34 | self, |
| 32 | default_host: str = "127.0.0.1", | 35 | default_host: str = "127.0.0.1", |
| 33 | default_port_start: int = 8000, | 36 | default_port_start: int = 8000, |
| 34 | - venv_base_path: str = "./venvs", | ||
| 35 | ): | 37 | ): |
| 36 | self.default_host = default_host | 38 | self.default_host = default_host |
| 37 | self.default_port_start = default_port_start | 39 | self.default_port_start = default_port_start |
| 38 | - self.venv_base_path = venv_base_path | 40 | + self.venv_manager = VirtualEnvironmentManager() |
| 39 | - self.venv_manager = VirtualEnvironmentManager(venv_base_path) | ||
| 40 | self._processes: Dict[str, asyncio.subprocess.Process] = {} | 41 | self._processes: Dict[str, asyncio.subprocess.Process] = {} |
| 41 | - logger.debug(f"LocalSubprocessDeployer initialized: venv_base_path={venv_base_path}") | ||
| 42 | 42 | ||
| 43 | def _kill_by_pid(self, pid: int) -> bool: | 43 | def _kill_by_pid(self, pid: int) -> bool: |
| 44 | """通过 PID 终止进程（跨进程有效）""" | 44 | """通过 PID 终止进程（跨进程有效）""" |
| @@ -127,23 +127,27 @@ class LocalSubprocessDeployer(Deployer[SubprocessParams]): | |||
| 127 | else: | 127 | else: |
| 128 | package_name = subprocess_params.package_name | 128 | package_name = subprocess_params.package_name |
| 129 | 129 | ||
| 130 | - # 1. 创建虚拟环境 | 130 | + # 创建虚拟环境 |
| 131 | venv_path = self.venv_manager.create_venv(deployment_id) | 131 | venv_path = self.venv_manager.create_venv(deployment_id) |
| 132 | logger.info(f"Virtual environment created: {venv_path}") | 132 | logger.info(f"Virtual environment created: {venv_path}") |
| 133 | 133 | ||
| 134 | - # 2. 如果有 ir_path，拷贝到虚拟环境 | 134 | + # 优先读取环境变量，未配置则使用原始默认路径 |
| 135 | - venv_ir_path = None | 135 | + dist_path = get_dist_dir() |
| 136 | - if ir_path: | 136 | + logger.info(f"dist_path: {dist_path}") |
| 137 | - import shutil | ||
| 138 | - venv_ir_path = os.path.join(venv_path, os.path.basename(ir_path)) | ||
| 139 | - shutil.copy2(ir_path, venv_ir_path) | ||
| 140 | - logger.info(f"IR file copied to venv: {ir_path} -> {venv_ir_path}") | ||
| 141 | 137 | ||
| 142 | - # 3. 安装WHL包 | 138 | + # 获取所有 .whl 文件 |
| 143 | - self.venv_manager.install_whl(deployment_id, whl_path) | 139 | + whl_files = list(dist_path.glob("*.whl")) |
| 144 | - logger.info(f"WHL package installed: {whl_path}") | 140 | + if not whl_files: |
| 141 | + raise RuntimeError(f"No .whl files found in dist directory: {dist_path}") | ||
| 145 | 142 | ||
| 146 | - # 4. 获取虚拟环境Python解释器 | 143 | + logger.info(f"whl_files: {whl_files}") |
| 144 | + | ||
| 145 | + # 循环安装所有 whl 包 | ||
| 146 | + for whl_file in whl_files: | ||
| 147 | + self.venv_manager.install_whl(deployment_id, str(whl_file)) | ||
| 148 | + logger.info(f"Installed WHL package: {whl_file}") | ||
| 149 | + | ||
| 150 | + # 获取虚拟环境Python解释器 | ||
| 147 | python_executable = self.venv_manager.get_python_executable(deployment_id) | 151 | python_executable = self.venv_manager.get_python_executable(deployment_id) |
| 148 | 152 | ||
| 149 | # 5. 构建启动命令: python -m name --host --port [--irpath ir_path] [--userdata userdata] | 153 | # 5. 构建启动命令: python -m name --host --port [--irpath ir_path] [--userdata userdata] |
| @@ -157,8 +161,8 @@ class LocalSubprocessDeployer(Deployer[SubprocessParams]): | |||
| 157 | "--host", "0.0.0.0", | 161 | "--host", "0.0.0.0", |
| 158 | "--port", str(ctx.port) | 162 | "--port", str(ctx.port) |
| 159 | ] | 163 | ] |
| 160 | - if venv_ir_path: | 164 | + if ir_path: |
| 161 | - cmd.extend(["--irpath", venv_ir_path]) | 165 | + cmd.extend(["--irpath", ir_path]) |
| 162 | if userdata: | 166 | if userdata: |
| 163 | cmd.extend(["--userdata", userdata]) | 167 | cmd.extend(["--userdata", userdata]) |
| 164 | # 不记录完整 userdata 值，避免泄露敏感信息或日志混乱 | 168 | # 不记录完整 userdata 值，避免泄露敏感信息或日志混乱 |
| @@ -170,7 +170,7 @@ class DeploymentManager: | |||
| 170 | deployment_id = kwargs.pop("deployment_id", None) or self._generate_deployment_id() | 170 | deployment_id = kwargs.pop("deployment_id", None) or self._generate_deployment_id() |
| 171 | now = datetime.utcnow() | 171 | now = datetime.utcnow() |
| 172 | 172 | ||
| 173 | - logger.debug(f"Creating deployment record: deployment_id={deployment_id}, type={deployment_type}, name={name}") | 173 | + logger.debug(f"deployment_id={deployment_id}, type={deployment_type}, name={name}") |
| 174 | 174 | ||
| 175 | create_model = DeploymentCreate( | 175 | create_model = DeploymentCreate( |
| 176 | deployment_id=deployment_id, | 176 | deployment_id=deployment_id, |