Merged
fix(registry): preserve atomic state writes on failure #385
fix(registry): preserve atomic state writes on failure #385
Merged
Yiyang ShaoAdd on 18 days ago
Total: 5 files changed+264-43
@@ -1,8 +1,8 @@
1"""Cross-platform atomic JSON file write.1"""Cross-platform atomic JSON file write.
2 2 
3-Temp-file + ``os.replace`` (atomic on POSIX and Windows), with a retry +3+Unique sibling temp-file + ``os.replace`` (atomic on POSIX and Windows),
4-direct-overwrite fallback for the Windows case where ``os.replace`` can4+with one retry if replacement is briefly denied (e.g. by an AV scanner).
5-transiently fail if the target is briefly locked (e.g. by an AV scanner).5+Failed writes leave the existing destination intact and propagate the error.
6 6 
7Shared helper so modules that persist small JSON state (cluster identity,7Shared helper so modules that persist small JSON state (cluster identity,
8etc.) don't each re-implement the dance.8etc.) don't each re-implement the dance.
@@ -12,6 +12,9 @@ from __future__ import annotations
12 12 
13import json13import json
14import os14import os
15+import stat
16+import tempfile
17+import time
15from pathlib import Path18from pathlib import Path
16from typing import Any19from typing import Any
17 20 
@@ -20,23 +23,40 @@ def atomic_write_json(path: Path, data: Any) -> None:
20 """Write ``data`` as pretty UTF-8 JSON to ``path`` atomically.23 """Write ``data`` as pretty UTF-8 JSON to ``path`` atomically.
21 24 
22 The parent directory must exist. Trailing newline is added for clean25 The parent directory must exist. Trailing newline is added for clean
23- diffs. On the rare Windows ``os.replace`` failure, retries once after a26+ diffs. Flush the temporary file before replacing the destination. If
24- short sleep, then falls back to a direct (non-atomic) overwrite.27+ replacement is denied, retry once after a short sleep, then propagate
28+ the error. Concurrent writers use separate files; the last successful
29+ replacement wins. Temporary-file cleanup is best effort on failure.
30+ On POSIX, preserve existing permission bits; new files use 0644.
31+ Windows retains the platform temporary-file ACL behavior.
25 """32 """
26 path = Path(path)33 path = Path(path)
27 content = json.dumps(data, ensure_ascii=False, indent=2) + "\n"34 content = json.dumps(data, ensure_ascii=False, indent=2) + "\n"
28- tmp_path = path.with_suffix(path.suffix + ".tmp")35+ tmp_path = None
29- with open(tmp_path, "w", encoding="utf-8") as f:
30- f.write(content)
31 try:36 try:
32- os.replace(str(tmp_path), str(path))37+ with tempfile.NamedTemporaryFile(
maoqiming
maoqimingmaoqiming16 days ago

【review】文件权限。NamedTemporaryFile 在 POSIX 下以 0600 创建临时文件,os.replace 后目标文件权限会从原来的 0644/umask 默认值变为 0600;如果集群中其他进程或用户需要读取该状态文件,升级后可能出现权限拒绝。在替换前读取原目标文件的 mode 并 chmod 到临时文件(新建文件时显式指定期望 mode,例如 0644),并补充文件权限回归测试;如果 0600 是有意收紧,请在文档或变更说明中明确。

likedislike
33- except OSError:38+ mode="w", encoding="utf-8", dir=path.parent,
34- import time39+ prefix=f".{path.name}.", suffix=".tmp", delete=False,
35- time.sleep(0.05)40+ ) as f:
41+ tmp_path = Path(f.name)
42+ f.write(content)
43+ f.flush()
44+ if os.name == "posix":
45+ try:
46+ mode = stat.S_IMODE(path.stat().st_mode)
47+ except FileNotFoundError:
48+ mode = 0o644
49+ os.chmod(tmp_path, mode)
50+ os.fsync(f.fileno())
36 try:51 try:
37 os.replace(str(tmp_path), str(path))52 os.replace(str(tmp_path), str(path))
38- except OSError:53+ except PermissionError:
39- with open(path, "w", encoding="utf-8") as f:54+ time.sleep(0.05)
40- f.write(content)55+ os.replace(str(tmp_path), str(path))
41- if tmp_path.exists():56+ finally:
57+ if tmp_path is not None:
58+ try:
42 tmp_path.unlink(missing_ok=True)59 tmp_path.unlink(missing_ok=True)
60+ except OSError:
61+ # Cleanup must not hide the original write/replace failure.
62+ pass
maoqiming
maoqimingmaoqiming18 days ago

【review】设计问题。本 PR 修复了公共原子写入实现(唯一临时文件、fsync、失败时保留原文件并向上抛错),但 register 存储模块中保留的一份同逻辑本地 _atomic_write 拷贝并未迁移:临时文件名固定为目标文件名加 .tmp(并发写入会互相覆盖临时文件)、写后没有 fsync、replace 两次失败后还会以写模式直接打开目标文件覆盖(中途失败会留下损坏状态),且整个过程不抛异常。该函数被服务状态、注册配置、向量配置等 6 处持久化调用使用,合并后这些路径仍暴露在旧实现的数据损坏风险中。建议将这些调用点迁移到 atomic_write_json 并删除本地拷贝;若短期不迁移,至少去掉直接覆盖目标文件的兜底逻辑并让失败向上传播。

likedislike
@@ -16,6 +16,8 @@ import zipfile
16from pathlib import Path16from pathlib import Path
17from typing import Any, Dict, List, Optional17from typing import Any, Dict, List, Optional
18 18 
19+from a2x_registry.common.atomic import atomic_write_json
20+ 
19from .models import AgentCard, GenericServiceData, RegistryEntry, SkillData21from .models import AgentCard, GenericServiceData, RegistryEntry, SkillData
20 22 
21logger = logging.getLogger(__name__)23logger = logging.getLogger(__name__)
@@ -103,7 +105,7 @@ class RegistryStore:
103 def _flush_api_config(self):105 def _flush_api_config(self):
104 """Serialize _api_entries to api_config.json. Must be called with lock held."""106 """Serialize _api_entries to api_config.json. Must be called with lock held."""
105 services = [_entry_to_config_dict(e) for e in self._api_entries.values()]107 services = [_entry_to_config_dict(e) for e in self._api_entries.values()]
106- _atomic_write(self._dir / API_CONFIG_FILE, {"services": services})108+ atomic_write_json(self._dir / API_CONFIG_FILE, {"services": services})
107 109 
108 # --- Skill folder I/O ---110 # --- Skill folder I/O ---
109 111 
@@ -279,7 +281,7 @@ class RegistryStore:
279 281 
280 def write_service_json(self, services: List[dict]):282 def write_service_json(self, services: List[dict]):
281 """Atomically write the output service.json."""283 """Atomically write the output service.json."""
282- _atomic_write(self._dir / SERVICE_JSON_FILE, services)284+ atomic_write_json(self._dir / SERVICE_JSON_FILE, services)
283 285 
284 # --- Register-format config ---286 # --- Register-format config ---
285 287 
@@ -305,7 +307,7 @@ class RegistryStore:
305 307 
306 def write_register_config(self, formats: Dict[str, str]) -> None:308 def write_register_config(self, formats: Dict[str, str]) -> None:
307 """Persist ``register_config.json`` with normalized formats dict."""309 """Persist ``register_config.json`` with normalized formats dict."""
308- _atomic_write(self._dir / REGISTER_CONFIG_FILE, {"formats": formats})310+ atomic_write_json(self._dir / REGISTER_CONFIG_FILE, {"formats": formats})
309 311 
310 def load_vector_config(self) -> Optional[Dict[str, Any]]:312 def load_vector_config(self) -> Optional[Dict[str, Any]]:
311 """Read ``vector_config.json`` → ``{embedding_model, embedding_dim}``.313 """Read ``vector_config.json`` → ``{embedding_model, embedding_dim}``.
@@ -327,7 +329,7 @@ class RegistryStore:
327 329 
328 def write_vector_config(self, embedding_model: str, embedding_dim: int) -> None:330 def write_vector_config(self, embedding_model: str, embedding_dim: int) -> None:
329 """Persist ``vector_config.json`` atomically."""331 """Persist ``vector_config.json`` atomically."""
330- _atomic_write(332+ atomic_write_json(
331 self._dir / VECTOR_CONFIG_FILE,333 self._dir / VECTOR_CONFIG_FILE,
332 {"embedding_model": embedding_model, "embedding_dim": embedding_dim},334 {"embedding_model": embedding_model, "embedding_dim": embedding_dim},
333 )335 )
@@ -362,7 +364,7 @@ class RegistryStore:
362 364 
363 def write_auth_config(self, required: bool) -> None:365 def write_auth_config(self, required: bool) -> None:
364 """Persist ``auth_config.json`` with ``{"required": bool, ...}``."""366 """Persist ``auth_config.json`` with ``{"required": bool, ...}``."""
365- _atomic_write(367+ atomic_write_json(
366 self._dir / AUTH_CONFIG_FILE,368 self._dir / AUTH_CONFIG_FILE,
367 {"required": bool(required), "schema_version": 1},369 {"required": bool(required), "schema_version": 1},
368 )370 )
@@ -415,7 +417,7 @@ class RegistryStore:
415 raise ValueError(f"max_ttl ({max_ttl}) must be >= min_ttl ({min_ttl})")417 raise ValueError(f"max_ttl ({max_ttl}) must be >= min_ttl ({min_ttl})")
416 if grace_period < 0:418 if grace_period < 0:
417 raise ValueError(f"grace_period must be >= 0, got {grace_period}")419 raise ValueError(f"grace_period must be >= 0, got {grace_period}")
418- _atomic_write(420+ atomic_write_json(
419 self._dir / LEASE_CONFIG_FILE,421 self._dir / LEASE_CONFIG_FILE,
420 {422 {
421 "enabled": bool(enabled),423 "enabled": bool(enabled),
@@ -578,24 +580,3 @@ def _list_skill_files(skill_dir: Path) -> List[str]:
578 full = Path(root) / fname580 full = Path(root) / fname
579 files.append(full.relative_to(skill_dir).as_posix())581 files.append(full.relative_to(skill_dir).as_posix())
580 return sorted(files)582 return sorted(files)
581- 
582- 
583-def _atomic_write(path: Path, data):
584- """Write JSON data atomically via temp file + os.replace."""
585- content = json.dumps(data, ensure_ascii=False, indent=2) + "\n"
586- tmp_path = path.with_suffix(".tmp")
587- with open(tmp_path, "w", encoding="utf-8") as f:
588- f.write(content)
589- try:
590- os.replace(str(tmp_path), str(path))
591- except OSError:
592- import time
593- time.sleep(0.05)
594- try:
595- os.replace(str(tmp_path), str(path))
596- except OSError:
597- # Fallback: direct overwrite (non-atomic but functional on Windows)
598- with open(path, "w", encoding="utf-8") as f:
599- f.write(content)
600- if tmp_path.exists():
601- tmp_path.unlink(missing_ok=True)
@@ -48,6 +48,29 @@ def test_init_writes_file(tmp_path):
48 }48 }
49 49 
50 50 
51+def test_failed_save_keeps_last_loadable_state(tmp_path, monkeypatch):
52+ from a2x_registry.common import atomic
53+ 
54+ path = tmp_path / "cluster_state.json"
55+ state = ClusterState.init(node_id="reg-preserved", path=path)
56+ original = path.read_bytes()
57+ state.version_clock = 42
58+ 
59+ def deny_replace(source, target):
60+ raise PermissionError("state file locked")
61+ 
62+ monkeypatch.setattr(atomic.os, "replace", deny_replace)
63+ monkeypatch.setattr(atomic.time, "sleep", lambda _: None)
64+ with pytest.raises(PermissionError, match="state file locked"):
65+ state.save()
66+ 
67+ assert path.read_bytes() == original
68+ loaded = ClusterState.load(path=path)
69+ assert loaded.node_id == "reg-preserved"
70+ assert loaded.version_clock == 0
71+ assert set(tmp_path.iterdir()) == {path}
72+ 
73+ 
51def test_init_twice_raises(tmp_path):74def test_init_twice_raises(tmp_path):
52 p = tmp_path / "cluster_state.json"75 p = tmp_path / "cluster_state.json"
53 ClusterState.init(node_id="reg-1", path=p)76 ClusterState.init(node_id="reg-1", path=p)
@@ -0,0 +1,164 @@
1+"""Atomic state writes preserve the destination on failure and isolate writers."""
2+ 
3+import errno
4+import json
5+import os
6+import stat
7+from concurrent.futures import ThreadPoolExecutor
8+from pathlib import Path
9+from threading import Barrier, local
10+ 
11+import pytest
12+ 
13+from a2x_registry.common import atomic
14+ 
15+ 
16+@pytest.mark.skipif(os.name != "posix", reason="POSIX permission bits")
17+@pytest.mark.parametrize("mode", [0o600, 0o640, 0o644])
18+def test_existing_permissions_survive_replacement(tmp_path, mode):
19+ path = tmp_path / "state.json"
20+ path.write_text("{}", encoding="utf-8")
21+ path.chmod(mode)
22+ atomic.atomic_write_json(path, {"new": True})
23+ assert stat.S_IMODE(path.stat().st_mode) == mode
24+ 
25+ 
26+@pytest.mark.skipif(os.name != "posix", reason="POSIX permission bits")
27+def test_new_file_has_documented_permissions(tmp_path):
28+ path = tmp_path / "state.json"
29+ atomic.atomic_write_json(path, {})
30+ assert stat.S_IMODE(path.stat().st_mode) == 0o644
31+ 
32+ 
33+def test_write_replaces_complete_json_and_preserves_unrelated_sidecar(tmp_path):
34+ path = tmp_path / "state.json"
35+ path.write_text('{"old": true}', encoding="utf-8")
36+ sidecar = tmp_path / "state.json.tmp"
37+ sidecar.write_text("unrelated file", encoding="utf-8")
38+ 
39+ atomic.atomic_write_json(path, {"node_id": "节点", "version": 2})
40+ 
41+ assert json.loads(path.read_text(encoding="utf-8")) == {"node_id": "节点", "version": 2}
42+ assert path.read_text(encoding="utf-8").endswith("\n")
43+ assert sidecar.read_text(encoding="utf-8") == "unrelated file"
44+ assert set(tmp_path.iterdir()) == {path, sidecar}
45+ 
46+ 
47+@pytest.mark.parametrize("existing", [False, True])
48+def test_denied_replacement_propagates_without_overwriting(tmp_path, monkeypatch, existing):
49+ path = tmp_path / "state.json"
50+ original = b'{"old": true}\n'
51+ if existing:
52+ path.write_bytes(original)
53+ attempts = []
54+ 
55+ def deny_replace(source, target):
56+ attempts.append(source)
57+ raise PermissionError("destination locked")
58+ 
59+ monkeypatch.setattr(atomic.os, "replace", deny_replace)
60+ monkeypatch.setattr(atomic.time, "sleep", lambda _: None)
61+ with pytest.raises(PermissionError, match="destination locked"):
62+ atomic.atomic_write_json(path, {"new": True})
63+ 
64+ assert len(attempts) == 2
65+ assert attempts[0] == attempts[1]
66+ if existing:
67+ assert path.read_bytes() == original
68+ else:
69+ assert not path.exists()
70+ assert set(tmp_path.iterdir()) == ({path} if existing else set())
71+ 
72+ 
73+def test_transient_permission_failure_retries_complete_file(tmp_path, monkeypatch):
74+ path = tmp_path / "state.json"
75+ path.write_text('{"old": true}', encoding="utf-8")
76+ real_replace = os.replace
77+ attempts = []
78+ 
79+ def replace(source, target):
80+ attempts.append(source)
81+ assert json.loads(Path(source).read_text(encoding="utf-8")) == {"new": True}
82+ if len(attempts) == 1:
83+ raise PermissionError("temporarily locked")
84+ assert json.loads(path.read_text(encoding="utf-8")) == {"old": True}
85+ real_replace(source, target)
86+ 
87+ monkeypatch.setattr(atomic.os, "replace", replace)
88+ monkeypatch.setattr(atomic.time, "sleep", lambda _: None)
89+ atomic.atomic_write_json(path, {"new": True})
90+ 
91+ assert len(attempts) == 2
92+ assert json.loads(path.read_text(encoding="utf-8")) == {"new": True}
93+ assert set(tmp_path.iterdir()) == {path}
94+ 
95+ 
96+def test_storage_error_preserves_old_file_without_retry(tmp_path, monkeypatch):
97+ path = tmp_path / "state.json"
98+ path.write_bytes(b"original state")
99+ attempts = []
100+ 
101+ def no_space(source, target):
102+ attempts.append(source)
103+ raise OSError(errno.ENOSPC, "no space")
104+ 
105+ monkeypatch.setattr(atomic.os, "replace", no_space)
106+ with pytest.raises(OSError) as error:
107+ atomic.atomic_write_json(path, {"new": True})
108+ assert error.value.errno == errno.ENOSPC
109+ assert len(attempts) == 1
110+ assert path.read_bytes() == b"original state"
111+ assert set(tmp_path.iterdir()) == {path}
112+ 
113+ 
114+def test_flush_failure_preserves_old_file_and_cleans_temp(tmp_path, monkeypatch):
115+ path = tmp_path / "state.json"
116+ path.write_bytes(b"original state")
117+ 
118+ def fail_flush(_):
119+ raise OSError(errno.EIO, "flush failed")
120+ 
121+ monkeypatch.setattr(atomic.os, "fsync", fail_flush)
122+ with pytest.raises(OSError, match="flush failed"):
123+ atomic.atomic_write_json(path, {"new": True})
124+ assert path.read_bytes() == b"original state"
125+ assert set(tmp_path.iterdir()) == {path}
126+ 
127+ 
128+def test_serialization_failure_leaves_no_partial_files(tmp_path):
129+ path = tmp_path / "state.json"
130+ path.write_bytes(b"original state")
131+ with pytest.raises(TypeError):
132+ atomic.atomic_write_json(path, object())
133+ assert path.read_bytes() == b"original state"
134+ assert set(tmp_path.iterdir()) == {path}
135+ 
136+ 
137+def test_concurrent_writers_use_separate_complete_sibling_files(tmp_path, monkeypatch):
138+ path = tmp_path / "state.json"
139+ real_replace = os.replace
140+ barrier = Barrier(2)
141+ writer_state = local()
142+ sources = []
143+ payloads = []
144+ 
145+ def replace(source, target):
146+ source = Path(source)
147+ assert source.parent == path.parent
148+ # Synchronize each writer once; a Windows permission retry must not
149+ # wait for a second writer that has already completed its replacement.
150+ if not getattr(writer_state, "ready", False):
151+ writer_state.ready = True
152+ sources.append(source)
153+ barrier.wait(timeout=10)
154+ payloads.append(json.loads(source.read_text(encoding="utf-8")))
155+ real_replace(source, target)
156+ 
157+ monkeypatch.setattr(atomic.os, "replace", replace)
158+ with ThreadPoolExecutor(max_workers=2) as pool:
159+ list(pool.map(lambda writer: atomic.atomic_write_json(path, {"writer": writer}), [1, 2]))
160+ 
161+ assert len(set(sources)) == 2
162+ assert sorted(item["writer"] for item in payloads) == [1, 2]
163+ assert json.loads(path.read_text(encoding="utf-8"))["writer"] in (1, 2)
164+ assert set(tmp_path.iterdir()) == {path}
@@ -0,0 +1,33 @@
1+"""All dataset JSON writers preserve the last state on replacement failure."""
2+ 
3+import pytest
4+ 
5+from a2x_registry.common import atomic
6+from a2x_registry.register.store import RegistryStore
7+ 
8+ 
9+@pytest.mark.parametrize("filename,method,args", [
10+ ("api_config.json", "save_api_batch", ([],)),
11+ ("service.json", "write_service_json", ([],)),
12+ ("register_config.json", "write_register_config", ({"a2a": "1.0"},)),
13+ ("vector_config.json", "write_vector_config", ("model", 128)),
14+ ("auth_config.json", "write_auth_config", (True,)),
15+ ("lease_config.json", "write_lease_config", (True, 10, 60, 30)),
16+])
17+def test_dataset_writer_propagates_failure_and_preserves_old_json(
18+ tmp_path, monkeypatch, filename, method, args,
19+):
20+ store = RegistryStore(tmp_path)
21+ path = tmp_path / filename
22+ original = b'{"previous": true}\n'
23+ path.write_bytes(original)
24+ 
25+ def deny_replace(source, target):
26+ raise PermissionError("destination locked")
27+ 
28+ monkeypatch.setattr(atomic.os, "replace", deny_replace)
29+ monkeypatch.setattr(atomic.time, "sleep", lambda _: None)
30+ with pytest.raises(PermissionError, match="destination locked"):
31+ getattr(store, method)(*args)
32+ assert path.read_bytes() == original
33+ assert set(tmp_path.iterdir()) == {path}