| @@ -52,6 +52,17 @@ torch_npu.npu.memory._dump_snapshot("memory_snapshot.pickle") | |||
| 52 | 52 | ||
| 53 | After collection, import <code>memory_snapshot.pickle</code> into MindStudio Insight and analyze it on the memory details (PyTorch Snapshot) interface. | 53 | After collection, import <code>memory_snapshot.pickle</code> into MindStudio Insight and analyze it on the memory details (PyTorch Snapshot) interface. |
| 54 | 54 | ||
| 55 | +### Limits of Parsed Cache Validation | ||
| 56 | + | ||
| 57 | +When importing the same Snapshot again, the parser validates existing artifacts | ||
| 58 | +before reusing them. A logical digest detects changes to parser-owned base data; | ||
| 59 | +consumer-added derived tables or indexes do not change that digest. Legacy artifacts | ||
| 60 | +without a historical digest receive only structural, event-range, dictionary-reference | ||
| 61 | +and SQLite physical-integrity checks. These checks cannot retrospectively prove that | ||
| 62 | +every value is unchanged or that no block rows were deleted. Digests are not security | ||
| 63 | +signatures; import only trusted data. Full validation reads the databases and takes | ||
| 64 | +longer for larger datasets; it does not mean event replay is running again. | ||
| 65 | + | ||
| 55 | ## Case 1: Locating Memory Leaks During the Training Process | 66 | ## Case 1: Locating Memory Leaks During the Training Process |
| 56 | 67 | ||
| 57 | ### Problem Symptom | 68 | ### Problem Symptom |
| @@ -44,6 +44,14 @@ torch_npu.npu.memory._dump_snapshot("memory_snapshot.pickle") | |||
| 44 | 44 | ||
| 45 | 采集完成后,将 `memory_snapshot.pickle` 导入 MindStudio Insight,在内存详情(PyTorch Snapshot)界面进行分析。 | 45 | 采集完成后,将 `memory_snapshot.pickle` 导入 MindStudio Insight,在内存详情(PyTorch Snapshot)界面进行分析。 |
| 46 | 46 | ||
| 47 | +### 解析缓存校验的限制 | ||
| 48 | + | ||
| 49 | +再次导入相同 Snapshot 时,解析脚本会校验已有产物以判断能否复用。带基础数据逻辑摘要的 | ||
| 50 | +产物可以检查基础解析数据是否发生变化;调用方增加派生表或索引不会改变该摘要。 | ||
| 51 | +没有历史摘要的旧产物只能进行结构、事件范围、字典引用和 SQLite 物理完整性检查, | ||
| 52 | +不能追溯证明所有字段未被修改或内存块记录未被删除。摘要不是安全签名,请仅导入可信数据。 | ||
| 53 | +大文件的全量校验需要读取数据库,耗时随数据量增长,不代表正在重新回放解析。 | ||
| 54 | + | ||
| 47 | ## 案例一:定位训练过程中的内存泄漏 | 55 | ## 案例一:定位训练过程中的内存泄漏 |
| 48 | 56 | ||
| 49 | ### 问题现象 | 57 | ### 问题现象 |
| @@ -20,12 +20,15 @@ import os | |||
| 20 | import json | 20 | import json |
| 21 | import sqlite3 | 21 | import sqlite3 |
| 22 | import shutil | 22 | import shutil |
| 23 | +import subprocess | ||
| 24 | +import sys | ||
| 23 | import unittest | 25 | import unittest |
| 24 | from contextlib import closing | 26 | from contextlib import closing |
| 25 | from pathlib import Path | 27 | from pathlib import Path |
| 26 | from unittest import mock | 28 | from unittest import mock |
| 27 | from util.file_util import load_pickle_to_dict | 29 | from util.file_util import load_pickle_to_dict |
| 28 | from util.logger import suppress_logs, restore_logs | 30 | from util.logger import suppress_logs, restore_logs |
| 31 | +from util.snapshot_cache import source_hashes | ||
| 29 | from base import Block, TraceEntry, DeviceSnapshot, BlockState | 32 | from base import Block, TraceEntry, DeviceSnapshot, BlockState |
| 30 | from tools.adaptors import snapshot2db | 33 | from tools.adaptors import snapshot2db |
| 31 | from simulate import SimulateDeviceSnapshot, SimulateHooker | 34 | from simulate import SimulateDeviceSnapshot, SimulateHooker |
| @@ -133,6 +136,7 @@ class Snapshot2DbTest(unittest.TestCase): | |||
| 133 | def testFailedRedumpLeavesBuildingManifest(self): | 136 | def testFailedRedumpLeavesBuildingManifest(self): |
| 134 | dest = self.cache_dir / "failed_redump" | 137 | dest = self.cache_dir / "failed_redump" |
| 135 | self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | 138 | self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) |
| 139 | + (dest / snapshot2db.MANIFEST_FILE_NAME).unlink() | ||
| 136 | original_replay = SimulateDeviceSnapshot.replay_until | 140 | original_replay = SimulateDeviceSnapshot.replay_until |
| 137 | 141 | ||
| 138 | def fail_replay(self, _min_event_idx, _on_event_processed=None): | 142 | def fail_replay(self, _min_event_idx, _on_event_processed=None): |
| @@ -146,6 +150,115 @@ class Snapshot2DbTest(unittest.TestCase): | |||
| 146 | manifest = json.loads((dest / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding="utf-8")) | 150 | manifest = json.loads((dest / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding="utf-8")) |
| 147 | self.assertEqual(manifest["status"], "building") | 151 | self.assertEqual(manifest["status"], "building") |
| 148 | 152 | ||
| 153 | + def testCacheHitSkipsPickleAndPreservesConsumerTables(self): | ||
| 154 | + dest = self.cache_dir / "cache_hit" | ||
| 155 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 156 | + db = self.get_slice_db(dest, 0) | ||
| 157 | + with closing(sqlite3.connect(db)) as conn: | ||
| 158 | + conn.execute("CREATE TABLE consumer_cache(value TEXT)") | ||
| 159 | + conn.execute("INSERT INTO consumer_cache VALUES ('keep')") | ||
| 160 | + conn.commit() | ||
| 161 | + with mock.patch.object(snapshot2db, "load_pickle_to_dict", side_effect=AssertionError("cache miss")): | ||
| 162 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 163 | + with closing(sqlite3.connect(db)) as conn: | ||
| 164 | + self.assertEqual(conn.execute("SELECT value FROM consumer_cache").fetchall(), [('keep',)]) | ||
| 165 | + | ||
| 166 | + def testMissingSliceRebuildsWholeArtifact(self): | ||
| 167 | + dest = self.cache_dir / "missing_slice" | ||
| 168 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 169 | + self.get_slice_db(dest, 0).unlink() | ||
| 170 | + with mock.patch.object(snapshot2db, "load_pickle_to_dict", wraps=snapshot2db.load_pickle_to_dict) as loader: | ||
| 171 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 172 | + loader.assert_called_once() | ||
| 173 | + | ||
| 174 | + def testActiveWriterPreventsRebuildEvenWithMissingManifest(self): | ||
| 175 | + dest = self.cache_dir / 'active_writer' | ||
| 176 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 177 | + db = self.get_slice_db(dest, 0) | ||
| 178 | + script = ( | ||
| 179 | + 'import sqlite3, sys; ' | ||
| 180 | + 'conn=sqlite3.connect(sys.argv[1]); ' | ||
| 181 | + 'conn.execute("PRAGMA journal_mode=DELETE"); ' | ||
| 182 | + 'conn.execute("BEGIN EXCLUSIVE"); ' | ||
| 183 | + 'conn.execute("UPDATE trace_entry_0 SET size=size+1"); ' | ||
| 184 | + 'print("ready", flush=True); ' | ||
| 185 | + 'sys.stdin.readline(); conn.rollback(); conn.close()' | ||
| 186 | + ) | ||
| 187 | + with subprocess.Popen( | ||
| 188 | + [sys.executable, '-c', script, str(db)], | ||
| 189 | + stdin=subprocess.PIPE, | ||
| 190 | + stdout=subprocess.PIPE, | ||
| 191 | + stderr=subprocess.PIPE, | ||
| 192 | + text=True, | ||
| 193 | + ) as writer: | ||
| 194 | + try: | ||
| 195 | + self.assertEqual(writer.stdout.readline().strip(), 'ready') | ||
| 196 | + self.assertTrue(Path(str(db) + '-journal').exists()) | ||
| 197 | + for missing_manifest in (False, True): | ||
| 198 | + if missing_manifest: | ||
| 199 | + (dest / snapshot2db.MANIFEST_FILE_NAME).unlink() | ||
| 200 | + self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 201 | + self.assertTrue(db.exists()) | ||
| 202 | + finally: | ||
| 203 | + writer.communicate('\n', timeout=15) | ||
| 204 | + self.assertEqual(writer.returncode, 0) | ||
| 205 | + | ||
| 206 | + def testDeepManifestRebuilds(self): | ||
| 207 | + dest = self.cache_dir / 'deep_manifest' | ||
| 208 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 209 | + (dest / snapshot2db.MANIFEST_FILE_NAME).write_text('[' * 200000 + '0' + ']' * 200000) | ||
| 210 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 211 | + | ||
| 212 | + def testLegacyMissingDictionaryRebuilds(self): | ||
| 213 | + dest = self.cache_dir / 'legacy_dictionary' | ||
| 214 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 215 | + manifest_path = dest / snapshot2db.MANIFEST_FILE_NAME | ||
| 216 | + manifest = json.loads(manifest_path.read_text(encoding='utf-8')) | ||
| 217 | + del manifest['parserCache'] | ||
| 218 | + manifest_path.write_text(json.dumps(manifest), encoding='utf-8') | ||
| 219 | + with closing(sqlite3.connect(self.get_slice_db(dest, 0))) as conn: | ||
| 220 | + conn.execute('DELETE FROM dictionary') | ||
| 221 | + conn.commit() | ||
| 222 | + with mock.patch.object(snapshot2db, 'load_pickle_to_dict', wraps=snapshot2db.load_pickle_to_dict) as loader: | ||
| 223 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 224 | + loader.assert_called_once() | ||
| 225 | + | ||
| 226 | + def testPrepareErrorReturnsFalseWithoutTraceback(self): | ||
| 227 | + dest = self.cache_dir / 'prepare_error' | ||
| 228 | + with mock.patch.object(snapshot2db, '_prepare_output_directory', side_effect=ValueError('symlink')): | ||
| 229 | + with mock.patch.object(snapshot2db.dump_logger, 'error') as error: | ||
| 230 | + self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 231 | + self.assertIn('Failed to prepare artifact directory', error.call_args.args[0]) | ||
| 232 | + | ||
| 233 | + def testDeviceSymlinkDoesNotRemoveTarget(self): | ||
| 234 | + dest = self.cache_dir / 'symlink_output' | ||
| 235 | + target = self.cache_dir / 'symlink_target' | ||
| 236 | + dest.mkdir() | ||
| 237 | + target.mkdir() | ||
| 238 | + sentinel = target / 'keep.txt' | ||
| 239 | + sentinel.write_text('keep') | ||
| 240 | + link = dest / 'device_0' | ||
| 241 | + try: | ||
| 242 | + link.symlink_to(target, target_is_directory=True) | ||
| 243 | + except OSError as error: | ||
| 244 | + self.skipTest(f'Symlink creation is unavailable: {error}') | ||
| 245 | + self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 246 | + self.assertEqual(sentinel.read_text(), 'keep') | ||
| 247 | + | ||
| 248 | + def testCorruptIndexRebuilds(self): | ||
| 249 | + dest = self.cache_dir / 'corrupt_index' | ||
| 250 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 251 | + db = self.get_slice_db(dest, 0) | ||
| 252 | + with closing(sqlite3.connect(db)) as conn: | ||
| 253 | + page_size = conn.execute('PRAGMA page_size').fetchone()[0] | ||
| 254 | + root = conn.execute("SELECT rootpage FROM sqlite_master WHERE type='index' LIMIT 1").fetchone()[0] | ||
| 255 | + with db.open('r+b') as file: | ||
| 256 | + file.seek((root - 1) * page_size) | ||
| 257 | + file.write(b'\xff' * page_size) | ||
| 258 | + with mock.patch.object(snapshot2db, 'load_pickle_to_dict', wraps=snapshot2db.load_pickle_to_dict) as loader: | ||
| 259 | + self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0)) | ||
| 260 | + loader.assert_called_once() | ||
| 261 | + | ||
| 149 | def testDatabaseInitializationFailureCleansTemporaryFile(self): | 262 | def testDatabaseInitializationFailureCleansTemporaryFile(self): |
| 150 | dest = self.cache_dir / "database_init_failure" | 263 | dest = self.cache_dir / "database_init_failure" |
| 151 | 264 | ||
| @@ -176,7 +289,7 @@ class Snapshot2DbTest(unittest.TestCase): | |||
| 176 | manifest = json.loads((output_dir / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding='utf-8')) | 289 | manifest = json.loads((output_dir / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding='utf-8')) |
| 177 | device = manifest['devices']['0'] | 290 | device = manifest['devices']['0'] |
| 178 | self.assertEqual(manifest['status'], 'complete') | 291 | self.assertEqual(manifest['status'], 'complete') |
| 179 | - self.assertEqual(manifest['cacheHash'], 'hash-v1') | 292 | + self.assertEqual(manifest['cacheHash'], source_hashes(snapshot_path)[1]) |
| 180 | self.assertEqual(device['sliceCount'], 5) | 293 | self.assertEqual(device['sliceCount'], 5) |
| 181 | self.assertEqual(device['readySlices'], [0, 1, 2, 3, 4]) | 294 | self.assertEqual(device['readySlices'], [0, 1, 2, 3, 4]) |
| 182 | self.assertEqual( | 295 | self.assertEqual( |
| @@ -0,0 +1,407 @@ | |||
| 1 | +"""Cache validation must not depend on consumer-added SQLite objects.""" | ||
| 2 | + | ||
| 3 | +import hashlib | ||
| 4 | +import json | ||
| 5 | +import sqlite3 | ||
| 6 | +import tempfile | ||
| 7 | +import unittest | ||
| 8 | +from contextlib import closing | ||
| 9 | +from pathlib import Path | ||
| 10 | + | ||
| 11 | +from util.snapshot_cache import _RowEncoder, seal_manifest, source_hashes, validate_cache | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +class SnapshotCacheTest(unittest.TestCase): | ||
| 15 | + def test_member_log_is_not_duplicated_by_parent_handlers(self): | ||
| 16 | + import io | ||
| 17 | + import logging | ||
| 18 | + from unittest.mock import patch | ||
| 19 | + | ||
| 20 | + from util.snapshot_cache import LOGGER | ||
| 21 | + | ||
| 22 | + output = io.StringIO() | ||
| 23 | + handler = logging.StreamHandler(output) | ||
| 24 | + self.addCleanup(handler.close) | ||
| 25 | + with patch.object(LOGGER, "handlers", [handler]), patch.object(logging.getLogger(), "handlers", [handler]): | ||
| 26 | + validate_cache(self.directory, self.hash) | ||
| 27 | + self.assertEqual(output.getvalue().count("Snapshot cache member checked:"), 1) | ||
| 28 | + | ||
| 29 | + def test_row_encoder_preserves_digest_bytes_and_bounds_cache(self): | ||
| 30 | + encoder = _RowEncoder() | ||
| 31 | + for stack in ('', 'ascii', '\u4e2d\u6587', '"\\\n\t', None, 'x' * (8 * 1024 * 1024)): | ||
| 32 | + row = (1, 2, 3, 4, 0, 5, 6, 7, stack) | ||
| 33 | + expected = json.dumps(row, ensure_ascii=True, separators=(',', ':'), allow_nan=False).encode() | ||
| 34 | + self.assertEqual(encoder.encode(row, True), expected) | ||
| 35 | + self.assertEqual(encoder.encode(row, True), expected) | ||
| 36 | + for index in range(3000): | ||
| 37 | + encoder.encode((index, str(index)), True) | ||
| 38 | + self.assertLessEqual(len(encoder.stacks), 2048) | ||
| 39 | + self.assertLessEqual(encoder.size, 8 * 1024 * 1024) | ||
| 40 | + | ||
| 41 | + def setUp(self): | ||
| 42 | + # unittest owns the resource until addCleanup runs after each test. | ||
| 43 | + self.temp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with | ||
| 44 | + self.addCleanup(self.temp.cleanup) | ||
| 45 | + self.directory = Path(self.temp.name) | ||
| 46 | + self.source = self.directory / "input.pkl" | ||
| 47 | + self.source.write_bytes(b"trusted-test-source") | ||
| 48 | + self.hash = source_hashes(self.source)[1] | ||
| 49 | + self.db = self.directory / "slice.db" | ||
| 50 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 51 | + conn.executescript( | ||
| 52 | + 'CREATE TABLE trace_entry_0(id INTEGER PRIMARY KEY, action INTEGER, address INTEGER, ' | ||
| 53 | + 'size INTEGER, stream INTEGER, allocated INTEGER, active INTEGER, reserved INTEGER, callstack TEXT);' | ||
| 54 | + 'CREATE TABLE block_0(id INTEGER PRIMARY KEY, address INTEGER, size INTEGER, requestedSize INTEGER, ' | ||
| 55 | + 'state INTEGER, allocEventId INTEGER, freeEventId INTEGER);' | ||
| 56 | + 'CREATE TABLE dictionary("table" TEXT, "column" TEXT, "key" TEXT, "value" TEXT);' | ||
| 57 | + "INSERT INTO trace_entry_0 VALUES(0,4,100,10,0,10,10,20,'stack');" | ||
| 58 | + "INSERT INTO block_0 VALUES(0,100,10,10,1,0,-1);" | ||
| 59 | + "INSERT INTO dictionary VALUES('trace_entry_0','action','4','alloc');" | ||
| 60 | + "INSERT INTO dictionary VALUES('block_0','state','1','active_allocated');" | ||
| 61 | + ) | ||
| 62 | + self.manifest = { | ||
| 63 | + "schemaVersion": 1, | ||
| 64 | + "status": "complete", | ||
| 65 | + "sourceFile": str(self.source), | ||
| 66 | + "cacheHash": self.hash, | ||
| 67 | + "eventsPerSlice": 500000, | ||
| 68 | + "devices": { | ||
| 69 | + "0": { | ||
| 70 | + "eventCount": 1, | ||
| 71 | + "sliceCount": 1, | ||
| 72 | + "readySlices": [0], | ||
| 73 | + "slices": [{"index": 0, "startEventId": 0, "endEventId": 0, "file": "slice.db", "ready": True}], | ||
| 74 | + } | ||
| 75 | + }, | ||
| 76 | + } | ||
| 77 | + seal_manifest(self.directory, self.manifest) | ||
| 78 | + self.save() | ||
| 79 | + | ||
| 80 | + def save(self): | ||
| 81 | + (self.directory / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8") | ||
| 82 | + | ||
| 83 | + def test_consumer_write_after_sidecar_check_is_allowed(self): | ||
| 84 | + from unittest.mock import patch | ||
| 85 | + | ||
| 86 | + from util import snapshot_cache | ||
| 87 | + | ||
| 88 | + connect = sqlite3.connect | ||
| 89 | + writers = [] | ||
| 90 | + self.addCleanup(lambda: [writer.close() for writer in writers]) | ||
| 91 | + | ||
| 92 | + def open_during_write(*args, **kwargs): | ||
| 93 | + # Start the consumer after preflight, when no sidecars existed. | ||
| 94 | + writer = connect(self.db) | ||
| 95 | + writers.append(writer) | ||
| 96 | + writer.execute("PRAGMA journal_mode=WAL") | ||
| 97 | + writer.execute("CREATE TABLE consumer_cache(value INTEGER)") | ||
| 98 | + writer.commit() | ||
| 99 | + return connect(*args, **kwargs) | ||
| 100 | + | ||
| 101 | + with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_during_write): | ||
| 102 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 103 | + | ||
| 104 | + def test_base_write_after_sidecar_check_is_detected(self): | ||
| 105 | + from unittest.mock import patch | ||
| 106 | + | ||
| 107 | + from util import snapshot_cache | ||
| 108 | + | ||
| 109 | + connect = sqlite3.connect | ||
| 110 | + writers = [] | ||
| 111 | + self.addCleanup(lambda: [writer.close() for writer in writers]) | ||
| 112 | + | ||
| 113 | + def open_after_base_write(*args, **kwargs): | ||
| 114 | + writer = connect(self.db) | ||
| 115 | + writers.append(writer) | ||
| 116 | + writer.execute("PRAGMA journal_mode=WAL") | ||
| 117 | + writer.execute("UPDATE block_0 SET size=size+1") | ||
| 118 | + writer.commit() | ||
| 119 | + return connect(*args, **kwargs) | ||
| 120 | + | ||
| 121 | + with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_after_base_write): | ||
| 122 | + with self.assertRaisesRegex(ValueError, "parser data"): | ||
| 123 | + validate_cache(self.directory, self.hash) | ||
| 124 | + | ||
| 125 | + def test_consumer_commit_during_row_scan_is_allowed(self): | ||
| 126 | + from unittest.mock import patch | ||
| 127 | + | ||
| 128 | + from util import snapshot_cache | ||
| 129 | + | ||
| 130 | + with closing(sqlite3.connect(self.db)) as writer: | ||
| 131 | + writer.execute("PRAGMA journal_mode=WAL") | ||
| 132 | + encode = snapshot_cache._RowEncoder.encode | ||
| 133 | + committed = False | ||
| 134 | + | ||
| 135 | + def encode_while_writing(encoder, row, inline_stack): | ||
| 136 | + nonlocal committed | ||
| 137 | + if not committed: | ||
| 138 | + writer.execute("CREATE TABLE consumer_cache(value INTEGER)") | ||
| 139 | + writer.commit() | ||
| 140 | + writer.execute("PRAGMA wal_checkpoint(PASSIVE)") | ||
| 141 | + committed = True | ||
| 142 | + return encode(encoder, row, inline_stack) | ||
| 143 | + | ||
| 144 | + with patch.object(snapshot_cache._RowEncoder, "encode", encode_while_writing): | ||
| 145 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 146 | + self.assertTrue(committed) | ||
| 147 | + | ||
| 148 | + def test_parser_seal_does_not_override_producer_seal(self): | ||
| 149 | + self.manifest["ptSnap"] = { | ||
| 150 | + "memberHashKind": "parser-v1", | ||
| 151 | + "members": {"slice.db": "0" * 64}, | ||
| 152 | + } | ||
| 153 | + self.save() | ||
| 154 | + with self.assertRaisesRegex(ValueError, "member hash"): | ||
| 155 | + validate_cache(self.directory, self.hash) | ||
| 156 | + | ||
| 157 | + def test_logical_producer_seal_without_parser_seal_is_checked(self): | ||
| 158 | + self.manifest["ptSnap"] = { | ||
| 159 | + "memberHashKind": "parser-v1", | ||
| 160 | + "members": self.manifest.pop("parserCache")["members"], | ||
| 161 | + } | ||
| 162 | + self.save() | ||
| 163 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 164 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 165 | + conn.execute("UPDATE block_0 SET size=size+1") | ||
| 166 | + conn.commit() | ||
| 167 | + with self.assertRaisesRegex(ValueError, "member hash"): | ||
| 168 | + validate_cache(self.directory, self.hash) | ||
| 169 | + | ||
| 170 | + def test_legacy_producer_seal_is_not_overridden_by_parser_seal(self): | ||
| 171 | + self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} | ||
| 172 | + self.save() | ||
| 173 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 174 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 175 | + conn.execute("CREATE TABLE consumer_extra(value INTEGER)") | ||
| 176 | + with self.assertRaisesRegex(ValueError, "member hash"): | ||
| 177 | + validate_cache(self.directory, self.hash) | ||
| 178 | + | ||
| 179 | + def test_legacy_physical_seal_accepts_closed_wal_database(self): | ||
| 180 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 181 | + conn.execute("PRAGMA journal_mode=WAL") | ||
| 182 | + self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} | ||
| 183 | + self.save() | ||
| 184 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 185 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 186 | + | ||
| 187 | + def test_legacy_physical_seal_rejects_committed_wal(self): | ||
| 188 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 189 | + conn.execute("PRAGMA journal_mode=WAL") | ||
| 190 | + self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} | ||
| 191 | + self.save() | ||
| 192 | + with closing(sqlite3.connect(self.db)) as writer: | ||
| 193 | + writer.execute("CREATE TABLE consumer_cache(value INTEGER)") | ||
| 194 | + writer.commit() | ||
| 195 | + with self.assertRaisesRegex(ValueError, "sidecars"): | ||
| 196 | + validate_cache(self.directory, self.hash) | ||
| 197 | + | ||
| 198 | + def test_source_hash_matches_shared_salt(self): | ||
| 199 | + raw, salted = source_hashes(self.source) | ||
| 200 | + self.assertEqual(raw, hashlib.sha256(self.source.read_bytes()).hexdigest()) | ||
| 201 | + self.assertEqual(salted, hashlib.sha256(b"mem_snapshot_parser_v2" + self.source.read_bytes()).hexdigest()) | ||
| 202 | + | ||
| 203 | + def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self): | ||
| 204 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 205 | + conn.executescript( | ||
| 206 | + 'CREATE TABLE memory_allocation_cache_v1_0(value INTEGER);' | ||
| 207 | + 'CREATE TABLE arbitrary_consumer_data(value TEXT);' | ||
| 208 | + 'CREATE INDEX consumer_index ON block_0(address);' | ||
| 209 | + 'ALTER TABLE trace_entry_0 ADD COLUMN consumer_note TEXT;' | ||
| 210 | + 'CREATE VIEW consumer_view AS SELECT id FROM block_0;' | ||
| 211 | + 'CREATE TRIGGER consumer_trigger AFTER UPDATE ON block_0 BEGIN SELECT 1; END;' | ||
| 212 | + "INSERT INTO arbitrary_consumer_data VALUES('extra');" | ||
| 213 | + "INSERT INTO dictionary VALUES('arbitrary_consumer_data','value','1','extra');" | ||
| 214 | + ) | ||
| 215 | + (self.directory / "parse.log").write_text("consumer log") | ||
| 216 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 217 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 218 | + conn.execute("VACUUM") | ||
| 219 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 220 | + | ||
| 221 | + def test_modified_base_data_is_rejected(self): | ||
| 222 | + for table, assignment in ( | ||
| 223 | + ("block_0", "size=size+1"), | ||
| 224 | + ("trace_entry_0", "allocated=allocated+1"), | ||
| 225 | + ("dictionary", '"value"=\'changed\''), | ||
| 226 | + ): | ||
| 227 | + with self.subTest(table=table): | ||
| 228 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 229 | + conn.execute(f"UPDATE {table} SET {assignment}") | ||
| 230 | + conn.commit() | ||
| 231 | + with self.assertRaises(ValueError): | ||
| 232 | + validate_cache(self.directory, self.hash) | ||
| 233 | + seal_manifest(self.directory, self.manifest) | ||
| 234 | + self.save() | ||
| 235 | + | ||
| 236 | + def test_changed_source_is_rejected(self): | ||
| 237 | + self.source.write_bytes(b"changed") | ||
| 238 | + with self.assertRaises(ValueError): | ||
| 239 | + validate_cache(self.directory, source_hashes(self.source)[1]) | ||
| 240 | + | ||
| 241 | + def test_closed_wal_cache_preserves_database_bytes(self): | ||
| 242 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 243 | + conn.execute("PRAGMA journal_mode=WAL") | ||
| 244 | + before = self.db.read_bytes() | ||
| 245 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 246 | + self.assertEqual(self.db.read_bytes(), before) | ||
| 247 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 248 | + | ||
| 249 | + def test_rollback_journal_is_rejected(self): | ||
| 250 | + Path(str(self.db) + '-journal').write_bytes(b'') | ||
| 251 | + with self.assertRaisesRegex(ValueError, 'rollback journal'): | ||
| 252 | + validate_cache(self.directory, self.hash) | ||
| 253 | + | ||
| 254 | + def test_corrupt_secondary_index_is_rejected(self): | ||
| 255 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 256 | + conn.execute('CREATE INDEX block_address ON block_0(address)') | ||
| 257 | + page_size = conn.execute('PRAGMA page_size').fetchone()[0] | ||
| 258 | + root = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='block_address'").fetchone()[0] | ||
| 259 | + with self.db.open('r+b') as file: | ||
| 260 | + file.seek((root - 1) * page_size) | ||
| 261 | + file.write(b'\xff' * page_size) | ||
| 262 | + with self.assertRaises((ValueError, sqlite3.DatabaseError)): | ||
| 263 | + validate_cache(self.directory, self.hash) | ||
| 264 | + | ||
| 265 | + def test_deeply_nested_manifest_is_a_validation_error(self): | ||
| 266 | + (self.directory / 'manifest.json').write_text('[' * 200000 + '0' + ']' * 200000) | ||
| 267 | + with self.assertRaisesRegex(ValueError, 'nesting'): | ||
| 268 | + validate_cache(self.directory, self.hash) | ||
| 269 | + | ||
| 270 | + def test_legacy_missing_dictionary_reference_is_rejected(self): | ||
| 271 | + del self.manifest['parserCache'] | ||
| 272 | + self.save() | ||
| 273 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 274 | + conn.execute("DELETE FROM dictionary WHERE \"table\"='block_0'") | ||
| 275 | + conn.commit() | ||
| 276 | + with self.assertRaisesRegex(ValueError, 'dictionary reference'): | ||
| 277 | + validate_cache(self.directory, self.hash) | ||
| 278 | + | ||
| 279 | + def test_legacy_producer_physical_hash_is_checked(self): | ||
| 280 | + del self.manifest['parserCache'] | ||
| 281 | + self.manifest['ptSnap'] = {'members': {'slice.db': source_hashes(self.db)[0]}} | ||
| 282 | + self.save() | ||
| 283 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 284 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 285 | + conn.execute('UPDATE block_0 SET size=size+1') | ||
| 286 | + conn.commit() | ||
| 287 | + with self.assertRaisesRegex(ValueError, 'member hash'): | ||
| 288 | + validate_cache(self.directory, self.hash) | ||
| 289 | + | ||
| 290 | + def test_member_validation_reports_completed_members(self): | ||
| 291 | + from unittest.mock import patch | ||
| 292 | + | ||
| 293 | + from util import snapshot_cache | ||
| 294 | + | ||
| 295 | + with patch.object(snapshot_cache.LOGGER, 'info') as info: | ||
| 296 | + validate_cache(self.directory, self.hash) | ||
| 297 | + info.assert_called_once_with('Snapshot cache member checked: %s', 'slice.db') | ||
| 298 | + | ||
| 299 | + def test_active_wal_is_read_instead_of_ignoring_committed_rows(self): | ||
| 300 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 301 | + conn.execute("PRAGMA journal_mode=WAL") | ||
| 302 | + conn.execute("UPDATE block_0 SET size=size+1") | ||
| 303 | + conn.commit() | ||
| 304 | + self.assertTrue(Path(str(self.db) + "-wal").exists()) | ||
| 305 | + with self.assertRaises(ValueError): | ||
| 306 | + validate_cache(self.directory, self.hash) | ||
| 307 | + | ||
| 308 | + def test_missing_or_corrupt_db_is_rejected_without_creating_it(self): | ||
| 309 | + self.db.unlink() | ||
| 310 | + with self.assertRaises(ValueError): | ||
| 311 | + validate_cache(self.directory, self.hash) | ||
| 312 | + self.assertFalse(self.db.exists()) | ||
| 313 | + self.db.write_bytes(b"not a database") | ||
| 314 | + with self.assertRaises(sqlite3.DatabaseError): | ||
| 315 | + validate_cache(self.directory, self.hash) | ||
| 316 | + | ||
| 317 | + def test_incomplete_manifest_or_event_range_is_rejected(self): | ||
| 318 | + for mutation in ( | ||
| 319 | + lambda: self.manifest.update(status="building"), | ||
| 320 | + lambda: self.manifest["devices"]["0"].update(readySlices=[]), | ||
| 321 | + ): | ||
| 322 | + mutation() | ||
| 323 | + self.save() | ||
| 324 | + with self.assertRaises(ValueError): | ||
| 325 | + validate_cache(self.directory, self.hash) | ||
| 326 | + self.manifest["status"] = "complete" | ||
| 327 | + with closing(sqlite3.connect(self.db)) as conn: | ||
| 328 | + conn.execute("DELETE FROM trace_entry_0") | ||
| 329 | + conn.commit() | ||
| 330 | + self.manifest["devices"]["0"]["readySlices"] = [0] | ||
| 331 | + self.save() | ||
| 332 | + with self.assertRaises(ValueError): | ||
| 333 | + validate_cache(self.directory, self.hash) | ||
| 334 | + | ||
| 335 | + def test_missing_json_and_unsafe_member_are_rejected(self): | ||
| 336 | + (self.directory / "manifest.json").unlink() | ||
| 337 | + with self.assertRaises(OSError): | ||
| 338 | + validate_cache(self.directory, self.hash) | ||
| 339 | + self.manifest["devices"]["0"]["slices"][0]["file"] = "../outside.db" | ||
| 340 | + self.save() | ||
| 341 | + with self.assertRaises(ValueError): | ||
| 342 | + validate_cache(self.directory, self.hash) | ||
| 343 | + | ||
| 344 | + def test_legacy_complete_artifact_is_structurally_validated(self): | ||
| 345 | + del self.manifest["parserCache"] | ||
| 346 | + self.save() | ||
| 347 | + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) | ||
| 348 | + | ||
| 349 | + def test_parser_device_selection_does_not_hide_producer_selection(self): | ||
| 350 | + self.manifest["ptSnap"] = {"identity": {"requestedDevice": 0}} | ||
| 351 | + self.save() | ||
| 352 | + with self.assertRaisesRegex(ValueError, "selected device"): | ||
| 353 | + validate_cache(self.directory, self.hash) | ||
| 354 | + self.assertEqual(validate_cache(self.directory, self.hash, device=0), self.manifest) | ||
| 355 | + | ||
| 356 | + def test_declared_device_must_match_inventory_and_request(self): | ||
| 357 | + for selected in (1, True, "0", -1): | ||
| 358 | + with self.subTest(selected=selected): | ||
| 359 | + self.manifest["parserCache"]["requestedDevice"] = selected | ||
| 360 | + self.save() | ||
| 361 | + with self.assertRaises(ValueError): | ||
| 362 | + validate_cache(self.directory, self.hash, device=0) | ||
| 363 | + | ||
| 364 | + def test_device_filtered_cache_cannot_satisfy_all_devices(self): | ||
| 365 | + seal_manifest(self.directory, self.manifest, device=0) | ||
| 366 | + self.save() | ||
| 367 | + self.assertEqual(validate_cache(self.directory, self.hash, device=0), self.manifest) | ||
| 368 | + for selection in (None, 1): | ||
| 369 | + with self.assertRaises(ValueError): | ||
| 370 | + validate_cache(self.directory, self.hash, device=selection) | ||
| 371 | + | ||
| 372 | + def test_invalid_manifest_shape_is_rejected(self): | ||
| 373 | + for data in ([], {**self.manifest, "parserCache": []}, {**self.manifest, "schemaVersion": True}): | ||
| 374 | + (self.directory / "manifest.json").write_text(json.dumps(data)) | ||
| 375 | + with self.assertRaises(ValueError): | ||
| 376 | + validate_cache(self.directory, self.hash) | ||
| 377 | + | ||
| 378 | + def test_duplicate_json_keys_are_rejected(self): | ||
| 379 | + (self.directory / "manifest.json").write_text('{"status":"building","status":"complete"}') | ||
| 380 | + with self.assertRaises(ValueError): | ||
| 381 | + validate_cache(self.directory, self.hash) | ||
| 382 | + | ||
| 383 | + def test_manifest_changed_during_validation_is_rejected(self): | ||
| 384 | + from unittest.mock import patch | ||
| 385 | + | ||
| 386 | + from util import snapshot_cache | ||
| 387 | + | ||
| 388 | + original = snapshot_cache.parser_members | ||
| 389 | + | ||
| 390 | + def change_manifest(directory, manifest): | ||
| 391 | + members = original(directory, manifest) | ||
| 392 | + self.manifest['consumerRevision'] = 1 | ||
| 393 | + self.save() | ||
| 394 | + return members | ||
| 395 | + | ||
| 396 | + with patch.object(snapshot_cache, 'parser_members', side_effect=change_manifest): | ||
| 397 | + with self.assertRaisesRegex(ValueError, 'manifest changed'): | ||
| 398 | + validate_cache(self.directory, self.hash) | ||
| 399 | + | ||
| 400 | + def test_parser_salt_changes_cache_identity(self): | ||
| 401 | + from unittest.mock import patch | ||
| 402 | + | ||
| 403 | + from util import snapshot_cache | ||
| 404 | + | ||
| 405 | + with patch.object(snapshot_cache, "PARSER_HASH_SALT", b"another-parser-version"): | ||
| 406 | + with self.assertRaises(ValueError): | ||
| 407 | + validate_cache(self.directory, source_hashes(self.source)[1]) | ||
| @@ -21,6 +21,7 @@ import sys | |||
| 21 | import argparse | 21 | import argparse |
| 22 | import json | 22 | import json |
| 23 | import math | 23 | import math |
| 24 | +import re | ||
| 24 | import shutil | 25 | import shutil |
| 25 | import sqlite3 | 26 | import sqlite3 |
| 26 | import time | 27 | import time |
| @@ -32,6 +33,13 @@ from tools.adaptors.database import SnapshotDb, block2record, event2record | |||
| 32 | from util.file_util import check_dir_valid, load_pickle_to_dict | 33 | from util.file_util import check_dir_valid, load_pickle_to_dict |
| 33 | from util.logger import get_logger, set_global_log_file | 34 | from util.logger import get_logger, set_global_log_file |
| 34 | from util.timer import timer | 35 | from util.timer import timer |
| 36 | +from util.snapshot_cache import ( | ||
| 37 | + source_hashes, | ||
| 38 | + validate_cache, | ||
| 39 | + seal_manifest, | ||
| 40 | + ensure_cache_not_busy, | ||
| 41 | + SnapshotCacheBusyError, | ||
| 42 | +) | ||
| 35 | 43 | ||
| 36 | dump_logger = get_logger("DatabaseDump") | 44 | dump_logger = get_logger("DatabaseDump") |
| 37 | DEFAULT_EVENTS_PER_SLICE = 500_000 | 45 | DEFAULT_EVENTS_PER_SLICE = 500_000 |
| @@ -385,7 +393,14 @@ class SnapshotSliceManifest: | |||
| 385 | 393 | ||
| 386 | def _prepare_output_directory(output_dir: Path): | 394 | def _prepare_output_directory(output_dir: Path): |
| 387 | output_dir.mkdir(parents=True, exist_ok=True) | 395 | output_dir.mkdir(parents=True, exist_ok=True) |
| 396 | + if output_dir.is_symlink(): | ||
| 397 | + raise ValueError("Artifact directory must not be a symlink") | ||
| 398 | + ensure_cache_not_busy(output_dir) | ||
| 388 | for stale_device_dir in output_dir.glob("device_*"): | 399 | for stale_device_dir in output_dir.glob("device_*"): |
| 400 | + if re.fullmatch(r"device_[0-9]+", stale_device_dir.name) is None: | ||
| 401 | + continue | ||
| 402 | + if stale_device_dir.is_symlink(): | ||
| 403 | + raise ValueError("Snapshot device directory must not be a symlink") | ||
| 389 | if stale_device_dir.is_dir(): | 404 | if stale_device_dir.is_dir(): |
| 390 | shutil.rmtree(stale_device_dir) | 405 | shutil.rmtree(stale_device_dir) |
| 391 | (output_dir / MANIFEST_FILE_NAME).unlink(missing_ok=True) | 406 | (output_dir / MANIFEST_FILE_NAME).unlink(missing_ok=True) |
| @@ -414,6 +429,25 @@ def dump( | |||
| 414 | events_per_slice: int = DEFAULT_EVENTS_PER_SLICE, | 429 | events_per_slice: int = DEFAULT_EVENTS_PER_SLICE, |
| 415 | cache_hash: str = "", | 430 | cache_hash: str = "", |
| 416 | ) -> bool: | 431 | ) -> bool: |
| 432 | + # 兼容旧调用参数,但缓存身份必须由脚本计算,不能信任调用方传入的值。 | ||
| 433 | + try: | ||
| 434 | + dump_logger.info("Snapshot cache validating: %s", pickle_file) | ||
| 435 | + _, cache_hash = source_hashes(Path(pickle_file)) | ||
| 436 | + artifact_dir = Path(output_dir) | ||
| 437 | + try: | ||
| 438 | + cached = validate_cache(artifact_dir, cache_hash, device) | ||
| 439 | + except SnapshotCacheBusyError as error: | ||
| 440 | + dump_logger.error("Failed to validate snapshot cache: %s", error) | ||
| 441 | + return False | ||
| 442 | + except (OSError, ValueError, TypeError, KeyError, sqlite3.Error): | ||
manifest 嵌套过深时抛出 manifest 嵌套 20 万层时,第 228 行( 建议:这里改为 最小用例:写入一个嵌套 20 万层的 ![]() ![]() | |||
| 443 | + cached = None | ||
| 444 | + if cached is not None: | ||
| 445 | + dump_logger.info("Snapshot cache validated: %s", cache_hash) | ||
| 446 | + dump_logger.info("Successfully dump the snapshot to database for devices %s.", list(cached["devices"])) | ||
| 447 | + return True | ||
| 448 | + except (OSError, ValueError) as error: | ||
| 449 | + dump_logger.error("Failed to validate snapshot source: %s", error) | ||
| 450 | + return False | ||
| 417 | try: | 451 | try: |
| 418 | data = load_pickle_to_dict(Path(pickle_file)) | 452 | data = load_pickle_to_dict(Path(pickle_file)) |
| 419 | except Exception as e: | 453 | except Exception as e: |
| @@ -435,8 +469,12 @@ def dump( | |||
| 435 | if events_per_slice <= 0: | 469 | if events_per_slice <= 0: |
| 436 | dump_logger.error("Events per slice must be greater than 0.") | 470 | dump_logger.error("Events per slice must be greater than 0.") |
| 437 | return False | 471 | return False |
| 438 | - artifact_dir = Path(output_dir) | 472 | + try: |
| 439 | - _prepare_output_directory(artifact_dir) | 473 | + _prepare_output_directory(artifact_dir) |
| 474 | + except (OSError, ValueError, sqlite3.Error, SnapshotCacheBusyError) as error: | ||
| 475 | + dump_logger.error("Failed to prepare artifact directory: %s", error) | ||
| 476 | + return False | ||
| 477 | + dump_logger.info("Snapshot cache rebuild: %s", cache_hash) | ||
| 440 | manifest = SnapshotSliceManifest( | 478 | manifest = SnapshotSliceManifest( |
| 441 | artifact_dir, | 479 | artifact_dir, |
| 442 | pickle_file, | 480 | pickle_file, |
| @@ -507,6 +545,11 @@ def dump( | |||
| 507 | ) | 545 | ) |
| 508 | return False | 546 | return False |
| 509 | dump_logger.info("Finished dump the snapshot to database for device %s.", dev) | 547 | dump_logger.info("Finished dump the snapshot to database for device %s.", dev) |
| 548 | + if source_hashes(Path(pickle_file))[1] != cache_hash: | ||
| 549 | + raise ValueError("Snapshot source changed during parsing") | ||
| 550 | + # 基础表逻辑摘要在最终反插完成后生成,调用方新增派生表不会使缓存失效。 | ||
| 551 | + manifest.data["status"] = "complete" | ||
| 552 | + seal_manifest(artifact_dir, manifest.data, device) | ||
| 510 | manifest.mark_complete() | 553 | manifest.mark_complete() |
| 511 | dump_logger.info("Successfully dump the snapshot to database for devices %s.", need_dump_devices) | 554 | dump_logger.info("Successfully dump the snapshot to database for devices %s.", need_dump_devices) |
| 512 | return True | 555 | return True |
| @@ -555,7 +598,7 @@ def get_args(): | |||
| 555 | "--cache_hash", | 598 | "--cache_hash", |
| 556 | type=str, | 599 | type=str, |
| 557 | default="", | 600 | default="", |
| 558 | - help="Salted source hash used by the server to validate parsed artifacts.", | 601 | + help="Deprecated compatibility argument; ignored. The parser calculates the source hash.", |
| 559 | ) | 602 | ) |
| 560 | args = parser.parse_args() | 603 | args = parser.parse_args() |
| 561 | snapshot_path = Path(args.snapshot_file) | 604 | snapshot_path = Path(args.snapshot_file) |
| @@ -0,0 +1,337 @@ | |||
| 1 | +"""Portable msinsight cache contract; consumer-owned tables are not parser data.""" | ||
| 2 | + | ||
| 3 | +from __future__ import annotations | ||
| 4 | + | ||
| 5 | +import hashlib | ||
| 6 | +import json | ||
| 7 | +import os | ||
| 8 | +import re | ||
| 9 | +import sqlite3 | ||
| 10 | +import sys | ||
| 11 | +from collections import OrderedDict | ||
| 12 | +from contextlib import closing | ||
| 13 | +from pathlib import Path | ||
| 14 | +from typing import Any | ||
| 15 | + | ||
| 16 | +from util.logger import get_logger | ||
| 17 | + | ||
| 18 | +PARSER_HASH_SALT = b"mem_snapshot_parser_v2" | ||
| 19 | +CACHE_CONTRACT_VERSION = 1 | ||
| 20 | +TRACE_COLUMNS = ( | ||
| 21 | + "id", | ||
| 22 | + "action", | ||
| 23 | + "address", | ||
| 24 | + "size", | ||
| 25 | + "stream", | ||
| 26 | + "allocated", | ||
| 27 | + "active", | ||
| 28 | + "reserved", | ||
| 29 | + "callstack", | ||
| 30 | +) | ||
| 31 | +BLOCK_COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") | ||
| 32 | +LOGGER = get_logger("SnapshotCache") | ||
| 33 | +# 全局文件 handler 会挂到各级 logger,禁止向父级传播导致同一条日志重复写入。 | ||
| 34 | +LOGGER.propagate = False | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +class SnapshotCacheBusyError(RuntimeError): | ||
| 38 | + """An active SQLite writer must not be treated as an invalid cache.""" | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +def _probe_writer(path: Path) -> None: | ||
| 42 | + # Only used before rebuilding or recovering an invalid rollback-journal cache. | ||
| 43 | + try: | ||
| 44 | + with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=rw", uri=True, timeout=5)) as conn: | ||
| 45 | + conn.execute("BEGIN IMMEDIATE") | ||
| 46 | + conn.rollback() | ||
| 47 | + except sqlite3.OperationalError as error: | ||
| 48 | + if "locked" in str(error).lower() or "busy" in str(error).lower(): | ||
| 49 | + raise SnapshotCacheBusyError("Snapshot member is being written by another process") from error | ||
| 50 | + raise | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +def ensure_cache_not_busy(directory: Path) -> None: | ||
| 54 | + """Preflight all existing members, even when the manifest itself is invalid.""" | ||
| 55 | + for device_dir in directory.glob("device_*"): | ||
| 56 | + if re.fullmatch(r"device_[0-9]+", device_dir.name) is None: | ||
| 57 | + continue | ||
| 58 | + if device_dir.is_symlink(): | ||
| 59 | + raise ValueError("Snapshot device directory must not be a symlink") | ||
| 60 | + for path in device_dir.glob("*.db"): | ||
| 61 | + if path.is_symlink() or any( | ||
| 62 | + Path(str(path) + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal") | ||
| 63 | + ): | ||
| 64 | + raise ValueError("Snapshot has an aliased member or sidecar") | ||
| 65 | + try: | ||
| 66 | + _probe_writer(path) | ||
| 67 | + except sqlite3.DatabaseError as error: | ||
| 68 | + # Corrupt closed databases are rebuildable; lock errors propagate separately. | ||
| 69 | + if "malformed" not in str(error).lower() and "not a database" not in str(error).lower(): | ||
| 70 | + raise | ||
| 71 | + | ||
| 72 | + | ||
| 73 | +class _RowEncoder: | ||
| 74 | + """Bound repeated callstack encoding; preserve the v1 digest bytes.""" | ||
| 75 | + | ||
| 76 | + def __init__(self) -> None: | ||
| 77 | + self.stacks: OrderedDict[str, bytes] = OrderedDict() | ||
| 78 | + self.size = 0 | ||
| 79 | + | ||
| 80 | + def encode(self, row: tuple[Any, ...], inline_stack: bool) -> bytes: | ||
| 81 | + if not inline_stack or not isinstance(row[-1], str): | ||
| 82 | + return json.dumps(row, ensure_ascii=True, separators=(",", ":"), allow_nan=False).encode() | ||
| 83 | + stack = row[-1] | ||
| 84 | + encoded = self.stacks.get(stack) | ||
| 85 | + if encoded is None: | ||
| 86 | + encoded = json.dumps(stack, ensure_ascii=True, allow_nan=False).encode() | ||
| 87 | + cost = sys.getsizeof(stack) + sys.getsizeof(encoded) | ||
| 88 | + if cost <= 8 * 1024 * 1024: | ||
| 89 | + while self.stacks and (self.size + cost > 8 * 1024 * 1024 or len(self.stacks) >= 2048): | ||
| 90 | + key, value = self.stacks.popitem(last=False) | ||
| 91 | + self.size -= sys.getsizeof(key) + sys.getsizeof(value) | ||
| 92 | + self.stacks[stack] = encoded | ||
| 93 | + self.size += cost | ||
| 94 | + else: | ||
| 95 | + self.stacks.move_to_end(stack) | ||
| 96 | + prefix = json.dumps(row[:-1], separators=(",", ":"), allow_nan=False).encode() | ||
| 97 | + return prefix[:-1] + b"," + encoded + b"]" | ||
| 98 | + | ||
| 99 | + | ||
| 100 | +def source_hashes(source: Path) -> tuple[str, str]: | ||
| 101 | + """Return raw and salted SHA-256 in one streaming pass, without unpickling.""" | ||
| 102 | + raw = hashlib.sha256() | ||
| 103 | + salted = hashlib.sha256(PARSER_HASH_SALT) | ||
| 104 | + with source.open("rb") as file: | ||
| 105 | + before = os.fstat(file.fileno()) | ||
| 106 | + for chunk in iter(lambda: file.read(1024 * 1024), b""): | ||
| 107 | + raw.update(chunk) | ||
| 108 | + salted.update(chunk) | ||
| 109 | + after = os.fstat(file.fileno()) | ||
| 110 | + current = source.stat() | ||
| 111 | + | ||
| 112 | + def signature(stat: os.stat_result) -> tuple[int, int, int, int]: | ||
| 113 | + # Windows fstat/stat disagree on ctime for some files; it is not content identity. | ||
| 114 | + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns | ||
| 115 | + | ||
| 116 | + if signature(before) != signature(after) or signature(after) != signature(current): | ||
| 117 | + raise ValueError("Snapshot source changed while hashing") | ||
| 118 | + return raw.hexdigest(), salted.hexdigest() | ||
| 119 | + | ||
| 120 | + | ||
| 121 | +def _integer(value: Any, minimum: int = 0) -> int: | ||
| 122 | + if not isinstance(value, int) or isinstance(value, bool) or value < minimum or value > 2**63 - 1: | ||
| 123 | + raise ValueError("Invalid nonnegative integer in snapshot manifest") | ||
| 124 | + return value | ||
| 125 | + | ||
| 126 | + | ||
| 127 | +def _member(directory: Path, relative: str) -> Path: | ||
| 128 | + if not isinstance(relative, str) or not relative or "\\" in relative: | ||
| 129 | + raise ValueError("Invalid snapshot member path") | ||
| 130 | + path = Path(relative) | ||
| 131 | + if path.is_absolute() or any(part in (".", "..") for part in relative.split("/")): | ||
| 132 | + raise ValueError("Snapshot member escapes artifact directory") | ||
| 133 | + candidate = directory / path | ||
| 134 | + if not candidate.is_file() or candidate.resolve() != directory.resolve() / path: | ||
| 135 | + raise ValueError("Missing or aliased snapshot member") | ||
| 136 | + return candidate | ||
| 137 | + | ||
| 138 | + | ||
| 139 | +def _table_rows( | ||
| 140 | + conn: sqlite3.Connection, | ||
| 141 | + name: str, | ||
| 142 | + columns: tuple[str, ...], | ||
| 143 | + where: str = "", | ||
| 144 | + parameters: tuple[str, ...] = (), | ||
| 145 | +) -> sqlite3.Cursor: | ||
| 146 | + if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() != ("table",): | ||
| 147 | + raise ValueError(f"Missing parser table: {name}") | ||
| 148 | + schema = conn.execute(f'PRAGMA table_info("{name}")').fetchall() | ||
| 149 | + actual = {row[1]: row for row in schema} | ||
| 150 | + for column in columns: | ||
| 151 | + kind = "TEXT" if column in ("callstack", "table", "column", "key", "value") else "INTEGER" | ||
| 152 | + if column not in actual or actual[column][2].upper() != kind: | ||
这里只检查必需列是否存在、类型是否正确,所以 建议:在 docstring 和说明里把允许的范围写完整,即"基础表的额外列、view、trigger 也允许,摘要只覆盖固定列"。可以补一条 ![]() ![]() | |||
| 153 | + raise ValueError(f"Invalid parser column: {name}.{column}") | ||
| 154 | + if "id" in columns and [row[1] for row in schema if row[5]] != ["id"]: | ||
| 155 | + raise ValueError(f"Missing parser primary key: {name}.id") | ||
| 156 | + selected = ",".join(f'"{column}"' for column in columns) | ||
| 157 | + order = '"id"' if "id" in columns else selected | ||
| 158 | + # Identifiers and the optional predicate are internal constants/validated device IDs. | ||
| 159 | + return conn.execute(f'SELECT {selected} FROM "{name}" {where} ORDER BY {order}', parameters) # nosec B608 | ||
| 160 | + | ||
| 161 | + | ||
| 162 | +def member_digest(path: Path, device: str, start: int, end: int) -> str: | ||
索引页物理损坏没有被检出(实测)
建议:在
实测开销:每个约 137 MB 的分片,热缓存约 0.2 s,冷缓存约 1.6 s。 最小用例:建库后用 ![]() ![]() | |||
| 163 | + """Hash fixed base columns; extra columns, tables, indexes, views and triggers are allowed. | ||
| 164 | + | ||
| 165 | + Unrelated dictionary entries are also allowed. Legacy artifacts without a seal | ||
| 166 | + receive structure/reference checks, not proof that all historical values are unchanged. | ||
| 167 | + """ | ||
| 168 | + if re.fullmatch(r"0|[1-9][0-9]*", str(device)) is None: | ||
| 169 | + raise ValueError("Invalid device ID") | ||
| 170 | + digest = hashlib.sha256() | ||
| 171 | + encoder = _RowEncoder() | ||
| 172 | + # Consumers may start writing after the sidecar check. Always use SQLite's | ||
| 173 | + # locking/WAL protocol and one read snapshot, even for a currently closed DB. | ||
| 174 | + sidecars = [Path(str(path) + suffix) for suffix in ("-wal", "-shm", "-journal")] | ||
| 175 | + if any(item.is_symlink() for item in sidecars): | ||
| 176 | + raise ValueError("Snapshot has an aliased sidecar") | ||
| 177 | + if sidecars[2].exists(): | ||
| 178 | + _probe_writer(path) | ||
| 179 | + raise ValueError("Snapshot has a rollback journal") | ||
| 180 | + uri = path.resolve().as_uri() + "?mode=ro" | ||
| 181 | + with closing(sqlite3.connect(uri, uri=True)) as conn: | ||
| 182 | + conn.execute("PRAGMA query_only=ON") | ||
| 183 | + conn.execute("BEGIN") | ||
| 184 | + if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]: | ||
| 185 | + raise ValueError("Snapshot member failed integrity check") | ||
| 186 | + tables = ((f"trace_entry_{device}", TRACE_COLUMNS), (f"block_{device}", BLOCK_COLUMNS)) | ||
| 187 | + for table, columns in tables: | ||
| 188 | + digest.update(json.dumps((table, columns), separators=(",", ":")).encode()) | ||
| 189 | + for row in _table_rows(conn, table, columns): | ||
| 190 | + digest.update(encoder.encode(row, columns == TRACE_COLUMNS)) | ||
| 191 | + digest.update(b"\n") | ||
| 192 | + count, low, high = conn.execute( | ||
| 193 | + f'SELECT COUNT(*), MIN(id), MAX(id) FROM "trace_entry_{device}" WHERE id>=0' # nosec B608 - device validated above | ||
| 194 | + ).fetchone() | ||
| 195 | + if (count, low, high) != (end - start + 1, start, end): | ||
| 196 | + raise ValueError("Snapshot event range is incomplete") | ||
| 197 | + for row in _table_rows( | ||
| 198 | + conn, | ||
| 199 | + "dictionary", | ||
| 200 | + ("table", "column", "key", "value"), | ||
| 201 | + 'WHERE "table" IN (?, ?)', | ||
| 202 | + tuple(table for table, _ in tables), | ||
| 203 | + ): | ||
| 204 | + digest.update(json.dumps(row, ensure_ascii=True, separators=(",", ":")).encode()) | ||
| 205 | + digest.update(b"\n") | ||
| 206 | + for table, column in ((f"trace_entry_{device}", "action"), (f"block_{device}", "state")): | ||
| 207 | + # Both identifiers are internal constants with the device validated above. | ||
| 208 | + missing = conn.execute( | ||
| 209 | + f'SELECT 1 FROM (SELECT DISTINCT "{column}" AS value FROM "{table}") AS base WHERE NOT EXISTS ' # nosec B608 | ||
| 210 | + '(SELECT 1 FROM dictionary AS dict WHERE dict."table"=? AND dict."column"=? ' | ||
| 211 | + 'AND dict."key"=CAST(base.value AS TEXT)) LIMIT 1', | ||
| 212 | + (table, column), | ||
| 213 | + ).fetchone() | ||
| 214 | + if missing: | ||
| 215 | + raise ValueError("Snapshot dictionary reference is missing") | ||
| 216 | + return digest.hexdigest() | ||
| 217 | + | ||
| 218 | + | ||
| 219 | +def parser_members(directory: Path, manifest: dict[str, Any]) -> dict[str, str]: | ||
| 220 | + """Validate the complete manifest and all DBs; never deserialize the source.""" | ||
| 221 | + if _integer(manifest.get("schemaVersion"), 1) != 1: | ||
| 222 | + raise ValueError("Unsupported snapshot manifest version") | ||
| 223 | + if ( | ||
| 224 | + manifest.get("status") != "complete" | ||
| 225 | + or not isinstance(manifest.get("sourceFile"), str) | ||
| 226 | + or not manifest["sourceFile"] | ||
| 227 | + ): | ||
| 228 | + raise ValueError("Snapshot is incomplete") | ||
| 229 | + capacity = _integer(manifest.get("eventsPerSlice"), 1) | ||
| 230 | + devices = manifest.get("devices") | ||
| 231 | + if not isinstance(devices, dict) or not devices: | ||
| 232 | + raise ValueError("Missing snapshot devices") | ||
| 233 | + members = {} | ||
| 234 | + for device, data in devices.items(): | ||
| 235 | + if re.fullmatch(r"0|[1-9][0-9]*", device) is None: | ||
| 236 | + raise ValueError("Invalid device ID") | ||
| 237 | + if not isinstance(data, dict): | ||
| 238 | + raise ValueError("Invalid device inventory") | ||
| 239 | + count = _integer(data["eventCount"], 1) | ||
| 240 | + size = _integer(data["sliceCount"], 1) | ||
| 241 | + if size != (count + capacity - 1) // capacity or len(data["slices"]) != size: | ||
| 242 | + raise ValueError("Incomplete snapshot slice inventory") | ||
| 243 | + if sorted(_integer(index) for index in data["readySlices"]) != list(range(size)): | ||
| 244 | + raise ValueError("Snapshot slices are not ready") | ||
| 245 | + for index, item in enumerate(data["slices"]): | ||
| 246 | + start, end = index * capacity, min((index + 1) * capacity, count) - 1 | ||
| 247 | + if ( | ||
| 248 | + _integer(item["index"]), | ||
| 249 | + _integer(item["startEventId"]), | ||
| 250 | + _integer(item["endEventId"]), | ||
| 251 | + ) != (index, start, end) or item["ready"] is not True: | ||
| 252 | + raise ValueError("Invalid snapshot slice range") | ||
| 253 | + relative = item["file"] | ||
| 254 | + if relative in members: | ||
| 255 | + raise ValueError("Duplicate snapshot member") | ||
| 256 | + members[relative] = member_digest(_member(directory, relative), device, start, end) | ||
| 257 | + LOGGER.info("Snapshot cache member checked: %s", relative) | ||
| 258 | + return members | ||
| 259 | + | ||
| 260 | + | ||
| 261 | +def seal_manifest(directory: Path, manifest: dict[str, Any], device: int | None = None) -> None: | ||
| 262 | + manifest["parserCache"] = { | ||
| 263 | + "version": CACHE_CONTRACT_VERSION, | ||
| 264 | + "requestedDevice": device, | ||
| 265 | + "members": parser_members(directory, manifest), | ||
| 266 | + } | ||
| 267 | + | ||
| 268 | + | ||
| 269 | +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: | ||
| 270 | + result = {} | ||
| 271 | + for key, value in pairs: | ||
| 272 | + if key in result: | ||
| 273 | + raise ValueError("Duplicate snapshot manifest key") | ||
| 274 | + result[key] = value | ||
| 275 | + return result | ||
| 276 | + | ||
| 277 | + | ||
| 278 | +def validate_cache(directory: Path, cache_hash: str, device: int | None = None) -> dict[str, Any]: | ||
| 279 | + path = directory / "manifest.json" | ||
| 280 | + original = path.read_bytes() | ||
| 281 | + try: | ||
| 282 | + manifest = json.loads(original, object_pairs_hook=_unique_object) | ||
| 283 | + except RecursionError as error: | ||
| 284 | + raise ValueError("Snapshot manifest nesting is too deep") from error | ||
| 285 | + if not isinstance(manifest, dict): | ||
| 286 | + raise ValueError("Invalid snapshot manifest") | ||
| 287 | + if not cache_hash or manifest.get("cacheHash") != cache_hash: | ||
| 288 | + raise ValueError("Snapshot source hash or parser salt changed") | ||
| 289 | + if device is not None and set(manifest["devices"]) != {str(device)}: | ||
| 290 | + raise ValueError("Snapshot device selection changed") | ||
| 291 | + for extension in ("parserCache", "ptSnap"): | ||
| 292 | + if extension in manifest and not isinstance(manifest[extension], dict): | ||
| 293 | + raise ValueError("Invalid snapshot extension") | ||
| 294 | + if not isinstance(manifest.get("ptSnap", {}).get("identity", {}), dict): | ||
| 295 | + raise ValueError("Invalid producer identity") | ||
| 296 | + # Validate each declaration independently; an absent selection in one seal | ||
| 297 | + # must not mask a producer's device-filtered export. | ||
| 298 | + for selection in ( | ||
| 299 | + manifest.get("parserCache", {}), | ||
| 300 | + manifest.get("ptSnap", {}).get("identity", {}), | ||
| 301 | + ): | ||
| 302 | + selected_device = selection.get("requestedDevice") | ||
| 303 | + if selected_device is None: | ||
| 304 | + continue | ||
| 305 | + _integer(selected_device) | ||
| 306 | + if device is None: | ||
| 307 | + raise ValueError("Snapshot cache contains only a selected device") | ||
| 308 | + if selected_device != device or set(manifest["devices"]) != {str(selected_device)}: | ||
| 309 | + raise ValueError("Snapshot device selection changed") | ||
| 310 | + members = parser_members(directory, manifest) | ||
| 311 | + attestation = manifest.get("parserCache") | ||
| 312 | + # Legacy producers have no logical attestation; still validate every base table and range. | ||
旧版无 seal 产物:行级篡改无法检出(实测,中) manifest 里没有
另外,pt-snap main 导出的 建议(都符合宽松契约,只要求"异常就重建"):
最小用例:用 master 生成旧产物,删除一个 ![]() ![]() | |||
| 313 | + if attestation is not None and ( | ||
| 314 | + _integer(attestation.get("version"), 1) != CACHE_CONTRACT_VERSION or attestation.get("members") != members | ||
| 315 | + ): | ||
| 316 | + raise ValueError("Snapshot parser data changed") | ||
| 317 | + producer = manifest.get("ptSnap", {}) | ||
| 318 | + if "memberHashKind" in producer and producer["memberHashKind"] != "parser-v1": | ||
| 319 | + raise ValueError("Unsupported member hash contract") | ||
| 320 | + if producer.get("memberHashKind") == "parser-v1" and producer.get("members") != members: | ||
| 321 | + raise ValueError("Snapshot producer member hash changed") | ||
| 322 | + if "members" in producer and producer.get("memberHashKind") != "parser-v1": | ||
| 323 | + physical = producer["members"] | ||
| 324 | + if not isinstance(physical, dict) or set(physical) != set(members): | ||
| 325 | + raise ValueError("Invalid legacy snapshot member inventory") | ||
| 326 | + for relative, expected in physical.items(): | ||
| 327 | + member = _member(directory, relative) | ||
| 328 | + wal = Path(str(member) + "-wal") | ||
| 329 | + # A read-only open can create an empty WAL and SHM. They contain no | ||
| 330 | + # committed rows; only the DB bytes need the legacy physical digest. | ||
| 331 | + if Path(str(member) + "-journal").exists() or (wal.exists() and wal.stat().st_size): | ||
| 332 | + raise ValueError("Legacy physical snapshot hash cannot validate live sidecars") | ||
| 333 | + if source_hashes(member)[0] != expected: | ||
| 334 | + raise ValueError("Legacy snapshot member hash changed") | ||
| 335 | + if path.read_bytes() != original: | ||
| 336 | + raise ValueError("Snapshot manifest changed during validation") | ||
| 337 | + return manifest | ||
| @@ -20,7 +20,6 @@ | |||
| 20 | #include "PythonUtil.h" | 20 | #include "PythonUtil.h" |
| 21 | #include "SafeFile.h" | 21 | #include "SafeFile.h" |
| 22 | #include "DataBaseManager.h" | 22 | #include "DataBaseManager.h" |
| 23 | -#include "HashUtil.h" | ||
| 24 | #include "MemSnapshotParser.h" | 23 | #include "MemSnapshotParser.h" |
| 25 | #include "MemSnapshotDatabase.h" | 24 | #include "MemSnapshotDatabase.h" |
| 26 | #include "MemSnapshotSliceService.h" | 25 | #include "MemSnapshotSliceService.h" |
| @@ -40,7 +39,6 @@ | |||
| 40 | namespace Dic::Module { | 39 | namespace Dic::Module { |
| 41 | using namespace Dic::Module::Timeline; | 40 | using namespace Dic::Module::Timeline; |
| 42 | using Dic::OpenReadFileSafely; | 41 | using Dic::OpenReadFileSafely; |
| 43 | -constexpr std::string_view MEM_SNAPSHOT_PARSER_HASH_SALT = "mem_snapshot_parser_v2"; | ||
| 44 | constexpr int MEM_SNAPSHOT_EVENTS_PER_SLICE = 500000; | 42 | constexpr int MEM_SNAPSHOT_EVENTS_PER_SLICE = 500000; |
| 45 | 43 | ||
| 46 | void MemSnapshotParserContext::Reset( | 44 | void MemSnapshotParserContext::Reset( |
| @@ -74,7 +72,15 @@ std::string MemSnapshotParserContext::GetLogPath() const { return logPath; } | |||
| 74 | 72 | ||
| 75 | std::string MemSnapshotParserContext::GetOutputDbPath() const { return outputDbPath; } | 73 | std::string MemSnapshotParserContext::GetOutputDbPath() const { return outputDbPath; } |
| 76 | 74 | ||
| 77 | -std::string MemSnapshotParserContext::GetFileHash() const { return fileHash; } | 75 | +std::string MemSnapshotParserContext::GetFileHash() const { |
| 76 | + std::shared_lock<std::shared_mutex> lock(_mutex); | ||
| 77 | + return fileHash; | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +void MemSnapshotParserContext::SetFileHash(std::string hash) { | ||
| 81 | + std::unique_lock<std::shared_mutex> lock(_mutex); | ||
| 82 | + fileHash = std::move(hash); | ||
| 83 | +} | ||
| 78 | 84 | ||
| 79 | bool MemSnapshotParserContext::IsInitialSuccessSent() const { | 85 | bool MemSnapshotParserContext::IsInitialSuccessSent() const { |
| 80 | std::shared_lock<std::shared_mutex> lock(_mutex); | 86 | std::shared_lock<std::shared_mutex> lock(_mutex); |
| @@ -154,20 +160,12 @@ void MemSnapshotParser::Reset() { | |||
| 154 | MemSnapshotDatabase::Reset(); | 160 | MemSnapshotDatabase::Reset(); |
| 155 | } | 161 | } |
| 156 | 162 | ||
| 157 | -std::string MemSnapshotParser::CalculateFileHash(const std::string &filePath) { | ||
| 158 | - const std::string hash = HashUtil::CalculateFileSha256(filePath, MEM_SNAPSHOT_PARSER_HASH_SALT); | ||
| 159 | - if (hash.empty()) { | ||
| 160 | - Server::ServerLog::Error("[Snapshot] Failed to open pickle file for hashing: %.", filePath); | ||
| 161 | - } | ||
| 162 | - return hash; | ||
| 163 | -} | ||
| 164 | - | ||
| 165 | void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFilePath) { | 163 | void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFilePath) { |
| 166 | const std::string realPicklePath = FileUtil::GetRealPath(pickleFilePath); | 164 | const std::string realPicklePath = FileUtil::GetRealPath(pickleFilePath); |
| 167 | const std::string &snapshotPath = realPicklePath.empty() ? pickleFilePath : realPicklePath; | 165 | const std::string &snapshotPath = realPicklePath.empty() ? pickleFilePath : realPicklePath; |
| 168 | const std::string outputPath = MemSnapshotSliceService::GetArtifactDirectory(snapshotPath); | 166 | const std::string outputPath = MemSnapshotSliceService::GetArtifactDirectory(snapshotPath); |
| 169 | const std::string logPath = MemSnapshotSliceService::GetLogPath(snapshotPath); | 167 | const std::string logPath = MemSnapshotSliceService::GetLogPath(snapshotPath); |
| 170 | - parseContext.Reset(snapshotPath, logPath, outputPath, CalculateFileHash(snapshotPath)); | 168 | + parseContext.Reset(snapshotPath, logPath, outputPath); |
| 171 | auto traceId = TraceIdManager::GenerateTraceId(); | 169 | auto traceId = TraceIdManager::GenerateTraceId(); |
| 172 | Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, log file: %, output db file: %.", | 170 | Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, log file: %, output db file: %.", |
| 173 | parseContext.GetPicklePath(), parseContext.GetLogPath(), parseContext.GetOutputDbPath()); | 171 | parseContext.GetPicklePath(), parseContext.GetLogPath(), parseContext.GetOutputDbPath()); |
| @@ -177,62 +175,6 @@ void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFil | |||
| 177 | 175 | ||
| 178 | MemSnapshotParserContext &MemSnapshotParser::GetParseContext() { return parseContext; } | 176 | MemSnapshotParserContext &MemSnapshotParser::GetParseContext() { return parseContext; } |
| 179 | 177 | ||
| 180 | -/*** | ||
| 181 | - * 在满足以下任意条件之一(按顺序检查)时,需要重新解析pickle文件: | ||
| 182 | - * 1. 数据库连接打开失败或db初始化失败 | ||
| 183 | - * 2. 通过2的校验,但数据库版本与当前版本不一致 | ||
| 184 | - * | ||
| 185 | - * @brief 检查是否需要解析或重新解析pickle文件 | ||
| 186 | - * @return true 需要解析或重新解析,此时会关闭已打开的连接并清空DatabaseManager纳管实例。 | ||
| 187 | - * @return false 不需要解析或重新解析。此时将不会清空DatabaseManager及纳管实例,可以不需要重复打开。 | ||
| 188 | - */ | ||
| 189 | -bool MemSnapshotParser::CheckIfParsingNeed(const MemSnapshotParserContext &context) { | ||
| 190 | - const auto manifest = MemSnapshotSliceService::LoadManifest(context.GetPicklePath()); | ||
| 191 | - if (!manifest.has_value() || !manifest->IsComplete()) { | ||
| 192 | - Server::ServerLog::Info("[Snapshot] Slice manifest is missing or incomplete; the file needs to re-parse."); | ||
| 193 | - MemSnapshotDatabase::Reset(); | ||
| 194 | - return true; | ||
| 195 | - } | ||
| 196 | - const std::string snapshotCacheHash = context.GetFileHash(); | ||
| 197 | - const std::string &cachedSnapshotHash = manifest->cacheHash; | ||
| 198 | - if (snapshotCacheHash.empty() || cachedSnapshotHash.empty() || snapshotCacheHash != cachedSnapshotHash) { | ||
| 199 | - Server::ServerLog::Info( | ||
| 200 | - "[Snapshot] Snapshot cache hash changed or is unavailable. The file needs to re-parse."); | ||
| 201 | - MemSnapshotDatabase::Reset(); | ||
| 202 | - return true; | ||
| 203 | - } | ||
| 204 | - for (const auto &[deviceId, device] : manifest->devices) { | ||
| 205 | - for (const auto &slice : device.slices) { | ||
| 206 | - const auto dbPath = MemSnapshotSliceService::ResolveSliceDbPath(context.GetPicklePath(), slice); | ||
| 207 | - if (!slice.ready || dbPath.empty() || !FileUtil::IsFilePathExist(dbPath)) { | ||
| 208 | - Server::ServerLog::Info( | ||
| 209 | - "[Snapshot] Slice artifact is missing for device %, slice %; the file needs to re-parse.", deviceId, | ||
| 210 | - slice.index); | ||
| 211 | - MemSnapshotDatabase::Reset(); | ||
| 212 | - return true; | ||
| 213 | - } | ||
| 214 | - std::recursive_mutex validationMutex; | ||
| 215 | - FullDb::MemSnapshotDatabase validationDatabase(validationMutex); | ||
| 216 | - if (!validationDatabase.OpenDbReadOnly(dbPath) || !validationDatabase.IsDeviceIdValid(deviceId)) { | ||
| 217 | - Server::ServerLog::Info( | ||
| 218 | - "[Snapshot] Slice database is incomplete for device %, slice %; the file needs to re-parse.", | ||
| 219 | - deviceId, slice.index); | ||
| 220 | - MemSnapshotDatabase::Reset(); | ||
| 221 | - return true; | ||
| 222 | - } | ||
| 223 | - if (!FullDb::MemSnapshotDatabase::HasMemoryAllocationCache(dbPath, deviceId) && | ||
| 224 | - !FullDb::MemSnapshotDatabase::BuildMemoryAllocationCache(dbPath, deviceId)) { | ||
| 225 | - Server::ServerLog::Info( | ||
| 226 | - "[Snapshot] Allocation cache is missing for device %, slice %; the file needs to re-parse.", | ||
| 227 | - deviceId, slice.index); | ||
| 228 | - MemSnapshotDatabase::Reset(); | ||
| 229 | - return true; | ||
| 230 | - } | ||
| 231 | - } | ||
| 232 | - } | ||
| 233 | - return false; | ||
| 234 | -} | ||
| 235 | - | ||
| 236 | MemSnapshotParser::MemSnapshotParser() { | 178 | MemSnapshotParser::MemSnapshotParser() { |
| 237 | // MemSnapshot解析snapshot解析至少需要双线程 | 179 | // MemSnapshot解析snapshot解析至少需要双线程 |
| 238 | _threadPool = std::make_unique<ThreadPool>(2); | 180 | _threadPool = std::make_unique<ThreadPool>(2); |
| @@ -242,28 +184,19 @@ MemSnapshotParser::~MemSnapshotParser() { _threadPool->ShutDown(); } | |||
| 242 | 184 | ||
| 243 | void MemSnapshotParser::ParseMemSnapshotTask() { | 185 | void MemSnapshotParser::ParseMemSnapshotTask() { |
| 244 | Server::ServerLog::Info("[Snapshot] Parse snapshot thread started."); | 186 | Server::ServerLog::Info("[Snapshot] Parse snapshot thread started."); |
| 245 | - if (!CheckIfParsingNeed(Instance().parseContext)) { | 187 | + // 缓存校验、旧解析产物清理由脚本统一负责;这里只清理本次进度日志。 |
| 246 | - Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, artifact directory: % is up-to-date.", | 188 | + MemSnapshotDatabase::Reset(); |
| 247 | - Instance().parseContext.GetPicklePath(), Instance().parseContext.GetOutputDbPath()); | 189 | + const auto logPath = Instance().parseContext.GetLogPath(); |
| 248 | - Instance().parseContext.SetProgress(100); | 190 | + if (FileUtil::IsFilePathExist(logPath) && !FileUtil::RemoveFile(logPath)) { |
| 249 | - Instance().parseContext.SetState(ParserState::UP_TO_DATE); | 191 | + Server::ServerLog::Error("[Snapshot] Failed to remove stale parsing log: %.", logPath); |
| 192 | + Instance().parseContext.SetState(ParserState::FINISH_FAILURE); | ||
| 250 | return; | 193 | return; |
| 251 | } | 194 | } |
| 252 | - // 需要首次解析或重新解析的场景 | ||
| 253 | - const std::vector<std::string> staleStateFiles = { | ||
| 254 | - Instance().parseContext.GetLogPath(), | ||
| 255 | - MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath()), | ||
| 256 | - }; | ||
| 257 | - for (const auto &path : staleStateFiles) { | ||
| 258 | - if (FileUtil::IsFilePathExist(path) && !FileUtil::RemoveFile(path)) { | ||
| 259 | - Server::ServerLog::Warn("[Snapshot] Failed to remove stale parsing state file: %.", path); | ||
| 260 | - } | ||
| 261 | - } | ||
| 262 | const std::string memSnapDumpScriptsPath = FileUtil::SplicePath("mem_snap_dump", "tools", "dump2db.py"); | 195 | const std::string memSnapDumpScriptsPath = FileUtil::SplicePath("mem_snap_dump", "tools", "dump2db.py"); |
| 263 | Server::ServerLog::Info("[Snapshot] Start parsing."); | 196 | Server::ServerLog::Info("[Snapshot] Start parsing."); |
| 264 | std::vector<std::string> arguments{Instance().parseContext.GetPicklePath(), "--dump_dir", | 197 | std::vector<std::string> arguments{Instance().parseContext.GetPicklePath(), "--dump_dir", |
| 265 | - Instance().parseContext.GetOutputDbPath(), "--log", Instance().parseContext.GetLogPath(), "--cache_hash", | 198 | + Instance().parseContext.GetOutputDbPath(), "--log", Instance().parseContext.GetLogPath(), "--events_per_slice", |
| 266 | - Instance().parseContext.GetFileHash(), "--events_per_slice", std::to_string(MEM_SNAPSHOT_EVENTS_PER_SLICE)}; | 199 | + std::to_string(MEM_SNAPSHOT_EVENTS_PER_SLICE)}; |
| 267 | Instance().parseContext.SetState(ParserState::Processing); | 200 | Instance().parseContext.SetState(ParserState::Processing); |
| 268 | try { | 201 | try { |
| 269 | Server::ServerLog::Info( | 202 | Server::ServerLog::Info( |
| @@ -281,6 +214,7 @@ int ReadProgressInLogFile(std::ifstream &file, std::string &err) { | |||
| 281 | // 读取最新进展 | 214 | // 读取最新进展 |
| 282 | const std::regex progressReg(R"((\d+(?:\.\d+)?)% of entries have been processed)"); | 215 | const std::regex progressReg(R"((\d+(?:\.\d+)?)% of entries have been processed)"); |
| 283 | const std::string parseFailedKeyWord = "Failed to dump the snapshot to database."; | 216 | const std::string parseFailedKeyWord = "Failed to dump the snapshot to database."; |
| 217 | + const std::regex cacheReadyReg(R"(Snapshot cache (?:validated|rebuild): ([0-9a-f]{64}))"); | ||
| 284 | std::string line; | 218 | std::string line; |
| 285 | std::string lastProgressLine; | 219 | std::string lastProgressLine; |
| 286 | while (std::getline(file, line)) { | 220 | while (std::getline(file, line)) { |
| @@ -291,6 +225,10 @@ int ReadProgressInLogFile(std::ifstream &file, std::string &err) { | |||
| 291 | err = line; | 225 | err = line; |
| 292 | return -1; | 226 | return -1; |
| 293 | } | 227 | } |
| 228 | + std::smatch cacheMatch; | ||
| 229 | + if (std::regex_search(line, cacheMatch, cacheReadyReg)) { | ||
| 230 | + MemSnapshotParser::Instance().GetParseContext().SetFileHash(cacheMatch[1].str()); | ||
| 231 | + } | ||
| 294 | if (std::regex_search(line, progressReg)) { | 232 | if (std::regex_search(line, progressReg)) { |
| 295 | lastProgressLine = line; | 233 | lastProgressLine = line; |
| 296 | } | 234 | } |
| @@ -346,7 +284,8 @@ void MemSnapshotParser::ParseDaemonTask() { | |||
| 346 | const auto manifestPath = MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath()); | 284 | const auto manifestPath = MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath()); |
| 347 | std::uintmax_t manifestSize = 0; | 285 | std::uintmax_t manifestSize = 0; |
| 348 | uint64_t currentManifestWriteTime = 0; | 286 | uint64_t currentManifestWriteTime = 0; |
| 349 | - if (MemSnapshotSliceService::GetFileFingerprint(manifestPath, manifestSize, currentManifestWriteTime)) { | 287 | + if (!Instance().parseContext.GetFileHash().empty() && |
| 288 | + MemSnapshotSliceService::GetFileFingerprint(manifestPath, manifestSize, currentManifestWriteTime)) { | ||
| 350 | const bool manifestChanged = | 289 | const bool manifestChanged = |
| 351 | !manifestWriteTime.has_value() || manifestWriteTime.value() != currentManifestWriteTime; | 290 | !manifestWriteTime.has_value() || manifestWriteTime.value() != currentManifestWriteTime; |
| 352 | if (manifestChanged) { | 291 | if (manifestChanged) { |
| @@ -379,12 +318,22 @@ void MemSnapshotParser::ParseDaemonTask() { | |||
| 379 | Instance().parseContext.SetProgress(std::min(newProgress, 99)); | 318 | Instance().parseContext.SetProgress(std::min(newProgress, 99)); |
| 380 | SLEEP(100); | 319 | SLEEP(100); |
| 381 | } | 320 | } |
| 382 | - SendReadySliceEvents(notifiedSlices); | ||
| 383 | if (Instance().parseContext.GetState() == ParserState::FINISH_SUCCESS) { | 321 | if (Instance().parseContext.GetState() == ParserState::FINISH_SUCCESS) { |
| 384 | std::ifstream file = OpenReadFileSafely(Instance().parseContext.GetLogPath()); | 322 | std::ifstream file = OpenReadFileSafely(Instance().parseContext.GetLogPath()); |
| 385 | - if (DoubleCheckSuccessInLogFile(file) && Instance().TryOpenParsingResultDbAndSetVersion()) { | 323 | + const auto manifest = MemSnapshotSliceService::LoadManifest(Instance().parseContext.GetPicklePath()); |
| 386 | - Instance().parseContext.SetProgress(100); | 324 | + if (DoubleCheckSuccessInLogFile(file) && manifest.has_value() && manifest->IsComplete()) { |
| 387 | - Server::ServerLog::Info("Parse thread has successfully finished with double check."); | 325 | + Instance().parseContext.SetFileHash(manifest->cacheHash); |
| 326 | + SendReadySliceEvents(notifiedSlices); | ||
| 327 | + size_t sliceCount = 0; | ||
| 328 | + for (const auto &[deviceId, device] : manifest->devices) { | ||
| 329 | + sliceCount += device.slices.size(); | ||
| 330 | + } | ||
| 331 | + if (notifiedSlices.size() != sliceCount) { | ||
| 332 | + Instance().parseContext.SetState(ParserState::FINISH_FAILURE); | ||
| 333 | + } else { | ||
| 334 | + Instance().parseContext.SetProgress(100); | ||
| 335 | + Server::ServerLog::Info("Parse thread has successfully finished with double check."); | ||
| 336 | + } | ||
| 388 | } else { | 337 | } else { |
| 389 | Server::ServerLog::Warn("The parsing thread returned a success response, but we did not find a " | 338 | Server::ServerLog::Warn("The parsing thread returned a success response, but we did not find a " |
| 390 | "corresponding success event confirmed in the logs."); | 339 | "corresponding success event confirmed in the logs."); |
| @@ -470,8 +419,6 @@ void MemSnapshotParser::ParseCallBack() { | |||
| 470 | SendEvent(std::move(event)); | 419 | SendEvent(std::move(event)); |
| 471 | } | 420 | } |
| 472 | 421 | ||
| 473 | -bool MemSnapshotParser::TryOpenParsingResultDbAndSetVersion() const { return !CheckIfParsingNeed(parseContext); } | ||
| 474 | - | ||
| 475 | std::unique_ptr<MemScopeParseSuccessEvent> MemSnapshotParser::BuildParseSuccessEventFromContext( | 422 | std::unique_ptr<MemScopeParseSuccessEvent> MemSnapshotParser::BuildParseSuccessEventFromContext( |
| 476 | const std::unordered_set<std::string> *preparedSlices) const { | 423 | const std::unordered_set<std::string> *preparedSlices) const { |
| 477 | const auto manifest = MemSnapshotSliceService::LoadManifest(parseContext.GetPicklePath()); | 424 | const auto manifest = MemSnapshotSliceService::LoadManifest(parseContext.GetPicklePath()); |
| @@ -45,6 +45,7 @@ struct MemSnapshotParserContext { | |||
| 45 | std::string GetLogPath() const; | 45 | std::string GetLogPath() const; |
| 46 | std::string GetOutputDbPath() const; | 46 | std::string GetOutputDbPath() const; |
| 47 | std::string GetFileHash() const; | 47 | std::string GetFileHash() const; |
| 48 | + void SetFileHash(std::string hash); | ||
| 48 | bool IsInitialSuccessSent() const; | 49 | bool IsInitialSuccessSent() const; |
| 49 | bool WasInitialSuccessSentWhileBuilding() const; | 50 | bool WasInitialSuccessSentWhileBuilding() const; |
| 50 | void MarkInitialSuccessSent(bool parsingComplete); | 51 | void MarkInitialSuccessSent(bool parsingComplete); |
| @@ -78,9 +79,6 @@ class MemSnapshotParser { | |||
| 78 | // 异步解析Pickle文件API | 79 | // 异步解析Pickle文件API |
| 79 | void AsyncParseMemSnapshotPickle(const std::string &pickleFilePath); | 80 | void AsyncParseMemSnapshotPickle(const std::string &pickleFilePath); |
| 80 | MemSnapshotParserContext &GetParseContext(); | 81 | MemSnapshotParserContext &GetParseContext(); |
| 81 | - static std::string CalculateFileHash(const std::string &filePath); | ||
| 82 | - // 用于检查是否需要解析或重新解析pickle文件 | ||
| 83 | - static bool CheckIfParsingNeed(const MemSnapshotParserContext &context); | ||
| 84 | 82 | ||
| 85 | private: | 83 | private: |
| 86 | MemSnapshotParser(); | 84 | MemSnapshotParser(); |
| @@ -90,8 +88,6 @@ class MemSnapshotParser { | |||
| 90 | // 守护线程,实施读取解析进展 | 88 | // 守护线程,实施读取解析进展 |
| 91 | static void ParseDaemonTask(); | 89 | static void ParseDaemonTask(); |
| 92 | static void ParseCallBack(); | 90 | static void ParseCallBack(); |
| 93 | - // 解析完成回调方法之一,用于二次检查db结果并设置解析结果db版本 | ||
| 94 | - bool TryOpenParsingResultDbAndSetVersion() const; | ||
| 95 | // 构造解析响应事件 | 91 | // 构造解析响应事件 |
| 96 | std::unique_ptr<MemScopeParseSuccessEvent> BuildParseSuccessEventFromContext( | 92 | std::unique_ptr<MemScopeParseSuccessEvent> BuildParseSuccessEventFromContext( |
| 97 | const std::unordered_set<std::string> *preparedSlices = nullptr) const; | 93 | const std::unordered_set<std::string> *preparedSlices = nullptr) const; |
| @@ -15,14 +15,10 @@ | |||
| 15 | * ------------------------------------------------------------------------- */ | 15 | * ------------------------------------------------------------------------- */ |
| 16 | 16 | ||
| 17 | #include <fstream> | 17 | #include <fstream> |
| 18 | -#include <chrono> | ||
| 19 | -#include <thread> | ||
| 20 | #include <gtest/gtest.h> | 18 | #include <gtest/gtest.h> |
| 21 | #include "FileUtil.h" | 19 | #include "FileUtil.h" |
| 22 | #include "MemSnapshotParser.h" | 20 | #include "MemSnapshotParser.h" |
| 23 | -#include "MemSnapshotDatabase.h" | ||
| 24 | #include "MemSnapshotSliceService.h" | 21 | #include "MemSnapshotSliceService.h" |
C++ 单测编译失败(实测) 本 PR 删除了 建议:在本文件加 ![]() ![]() | |||
| 25 | -#include "sqlite3.h" | ||
| 26 | 22 | ||
| 27 | using namespace Dic::Module; | 23 | using namespace Dic::Module; |
| 28 | using namespace Dic; | 24 | using namespace Dic; |
| @@ -37,7 +33,7 @@ class MemSnapshotParserTest : public ::testing::Test { | |||
| 37 | static void SetUpTestSuite() { | 33 | static void SetUpTestSuite() { |
| 38 | testPicklePath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.pkl"); | 34 | testPicklePath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.pkl"); |
| 39 | testLogPath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.log"); | 35 | testLogPath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.log"); |
| 40 | - testOutputDbPath = MemSnapshotSliceService::GetArtifactDirectory(testPicklePath); | 36 | + testOutputDbPath = Dic::Module::MemSnapshot::MemSnapshotSliceService::GetArtifactDirectory(testPicklePath); |
| 41 | 37 | ||
| 42 | std::ofstream pickleFile(testPicklePath); | 38 | std::ofstream pickleFile(testPicklePath); |
| 43 | pickleFile.close(); | 39 | pickleFile.close(); |
| @@ -105,6 +101,23 @@ TEST_F(MemSnapshotParserTest, ConfirmsSuccessKeywordInLog) { | |||
| 105 | FileUtil::RemoveFile(testLogPath); | 101 | FileUtil::RemoveFile(testLogPath); |
| 106 | } | 102 | } |
| 107 | 103 | ||
| 104 | +TEST_F(MemSnapshotParserTest, ReadsScriptCacheIdentityBeforePublishingSlices) { | ||
| 105 | + const std::string hash(64, 'a'); | ||
| 106 | + for (const auto &action : {"validated", "rebuild"}) { | ||
| 107 | + parser->GetParseContext().SetFileHash(""); | ||
| 108 | + { | ||
| 109 | + std::ofstream output(testLogPath, std::ios::trunc); | ||
| 110 | + output << "Snapshot cache " << action << ": " << hash << "\n"; | ||
| 111 | + output << "1% of entries have been processed\n"; | ||
| 112 | + } | ||
| 113 | + std::ifstream log(testLogPath); | ||
| 114 | + std::string error; | ||
| 115 | + EXPECT_EQ(ReadProgressInLogFile(log, error), 1); | ||
| 116 | + EXPECT_EQ(parser->GetParseContext().GetFileHash(), hash); | ||
| 117 | + } | ||
| 118 | + FileUtil::RemoveFile(testLogPath); | ||
| 119 | +} | ||
| 120 | + | ||
| 108 | // 测试解析器重置功能 | 121 | // 测试解析器重置功能 |
| 109 | TEST_F(MemSnapshotParserTest, Reset) { | 122 | TEST_F(MemSnapshotParserTest, Reset) { |
| 110 | // 先设置一些状态 | 123 | // 先设置一些状态 |
| @@ -168,112 +181,11 @@ TEST_F(MemSnapshotParserTest, ParseContextManagement) { | |||
| 168 | EXPECT_TRUE(parser->GetParseContext().IsFinished()); | 181 | EXPECT_TRUE(parser->GetParseContext().IsFinished()); |
| 169 | } | 182 | } |
| 170 | 183 | ||
| 171 | -// 测试是否需要解析的检查逻辑 | 184 | +TEST_F(MemSnapshotParserTest, FileHashIsPublishedByParserScript) { |
| 172 | -TEST_F(MemSnapshotParserTest, CheckIfParsingNeed) { | 185 | + parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath); |
| 173 | - // 重置解析器 | 186 | + EXPECT_TRUE(parser->GetParseContext().GetFileHash().empty()); |
| 174 | - parser->Reset(); | 187 | + parser->GetParseContext().SetFileHash("script-generated-hash"); |
| 175 | - | 188 | + EXPECT_EQ(parser->GetParseContext().GetFileHash(), "script-generated-hash"); |
| 176 | - // 设置测试上下文 | 189 | + parser->GetParseContext().Reset(); |
| 177 | - const std::string fileHash = parser->CalculateFileHash(testPicklePath); | 190 | + EXPECT_TRUE(parser->GetParseContext().GetFileHash().empty()); |
| 178 | - parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash); | ||
| 179 | - | ||
| 180 | - // 由于输出数据库文件不存在,应该需要解析 | ||
| 181 | - bool needParse = parser->CheckIfParsingNeed(parser->GetParseContext()); | ||
| 182 | - EXPECT_TRUE(needParse); | ||
| 183 | - | ||
| 184 | - const auto deviceDir = fs::path(testOutputDbPath) / "device_0"; | ||
| 185 | - fs::create_directories(deviceDir); | ||
| 186 | - const auto sliceDbPath = deviceDir / "slice_00000.db"; | ||
| 187 | - sqlite3 *sliceDb = nullptr; | ||
| 188 | - ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK); | ||
| 189 | - ASSERT_EQ(sqlite3_exec(sliceDb, | ||
| 190 | - "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT);" | ||
| 191 | - "CREATE TABLE block_0 (`id` INTEGER PRIMARY KEY, `address` INTEGER, `size` INTEGER, " | ||
| 192 | - "`requestedSize` INTEGER, `state` INTEGER DEFAULT 99, `allocEventId` INTEGER, `freeEventId` INTEGER);" | ||
| 193 | - "CREATE TABLE trace_entry_0 (`id` INTEGER PRIMARY KEY, `action` INTEGER, `address` INTEGER, `size` " | ||
| 194 | - "INTEGER, `stream` INTEGER, `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, `callstack` " | ||
| 195 | - "TEXT);" | ||
| 196 | - "INSERT INTO trace_entry_0 VALUES (0, 0, 0, 0, 0, 10, 10, 20, '');", | ||
| 197 | - nullptr, nullptr, nullptr), | ||
| 198 | - SQLITE_OK); | ||
| 199 | - sqlite3_close(sliceDb); | ||
| 200 | - ASSERT_TRUE(Dic::Module::FullDb::MemSnapshotDatabase::BuildMemoryAllocationCache(sliceDbPath.string(), "0")); | ||
| 201 | - std::ofstream manifest(MemSnapshotSliceService::GetManifestPath(testPicklePath), std::ios::trunc); | ||
| 202 | - manifest << R"({"schemaVersion":1,"status":"complete","sourceFile":"snapshot_invalid.pkl","cacheHash":")" | ||
| 203 | - << fileHash | ||
| 204 | - << R"(","eventsPerSlice":100,"devices":{"0":{"eventCount":1,"sliceCount":1,)" | ||
| 205 | - R"("readySlices":[0],"slices":[{"index":0,"startEventId":0,"endEventId":0,)" | ||
| 206 | - R"("file":"device_0/slice_00000.db","ready":true}]}}})"; | ||
| 207 | - manifest.close(); | ||
| 208 | - | ||
| 209 | - EXPECT_FALSE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 210 | - | ||
| 211 | - // Exercise the asynchronous cache-hit path without launching a pickle subprocess. | ||
| 212 | - parser->AsyncParseMemSnapshotPickle(testPicklePath); | ||
| 213 | - for (int attempt = 0; attempt < 100 && parser->GetParseContext().GetState() != ParserState::UP_TO_DATE; ++attempt) { | ||
| 214 | - std::this_thread::sleep_for(std::chrono::milliseconds(50)); | ||
| 215 | - } | ||
| 216 | - EXPECT_EQ(parser->GetParseContext().GetState(), ParserState::UP_TO_DATE); | ||
| 217 | - EXPECT_EQ(parser->GetParseContext().GetProgress(), 100); | ||
| 218 | - parser->Reset(); | ||
| 219 | - parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash); | ||
| 220 | - | ||
| 221 | - ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK); | ||
| 222 | - ASSERT_EQ(sqlite3_exec(sliceDb, "DROP TABLE memory_allocation_cache_v1_0", nullptr, nullptr, nullptr), SQLITE_OK); | ||
| 223 | - sqlite3_close(sliceDb); | ||
| 224 | - // 缺 cache 时可从源表现场重建,无需重新解析 pickle | ||
| 225 | - EXPECT_FALSE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 226 | - ASSERT_TRUE(Dic::Module::FullDb::MemSnapshotDatabase::HasMemoryAllocationCache(sliceDbPath.string(), "0")); | ||
| 227 | - | ||
| 228 | - ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK); | ||
| 229 | - ASSERT_EQ(sqlite3_exec(sliceDb, "DROP TABLE block_0", nullptr, nullptr, nullptr), SQLITE_OK); | ||
| 230 | - sqlite3_close(sliceDb); | ||
| 231 | - EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 232 | - ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK); | ||
| 233 | - ASSERT_EQ( | ||
| 234 | - sqlite3_exec(sliceDb, "CREATE TABLE block_0 (id INTEGER PRIMARY KEY)", nullptr, nullptr, nullptr), SQLITE_OK); | ||
| 235 | - sqlite3_close(sliceDb); | ||
| 236 | - | ||
| 237 | - parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, "changed-hash"); | ||
| 238 | - EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 239 | - | ||
| 240 | - parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash); | ||
| 241 | - fs::remove(sliceDbPath); | ||
| 242 | - EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 243 | - | ||
| 244 | - ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK); | ||
| 245 | - sqlite3_close(sliceDb); | ||
| 246 | - std::ofstream invalidManifest(MemSnapshotSliceService::GetManifestPath(testPicklePath), std::ios::trunc); | ||
| 247 | - invalidManifest << R"({"schemaVersion":1,"status":"complete"})"; | ||
| 248 | - invalidManifest.close(); | ||
| 249 | - EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext())); | ||
| 250 | - | ||
| 251 | - fs::remove_all(testOutputDbPath); | ||
| 252 | -} | ||
| 253 | - | ||
| 254 | -TEST_F(MemSnapshotParserTest, CalculateFileHashTracksFileContent) { | ||
| 255 | - { | ||
| 256 | - std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::trunc); | ||
| 257 | - pickleFile << "snapshot-content"; | ||
| 258 | - } | ||
| 259 | - const std::string firstHash = parser->CalculateFileHash(testPicklePath); | ||
| 260 | - const std::string secondHash = parser->CalculateFileHash(testPicklePath); | ||
| 261 | - EXPECT_FALSE(firstHash.empty()); | ||
| 262 | - EXPECT_EQ(firstHash, secondHash); | ||
| 263 | - | ||
| 264 | - { | ||
| 265 | - std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::app); | ||
| 266 | - pickleFile << "-changed"; | ||
| 267 | - } | ||
| 268 | - EXPECT_NE(firstHash, parser->CalculateFileHash(testPicklePath)); | ||
| 269 | -} | ||
| 270 | - | ||
| 271 | -TEST_F(MemSnapshotParserTest, CalculateFileHashIncludesParserSalt) { | ||
| 272 | - { | ||
| 273 | - std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::trunc); | ||
| 274 | - pickleFile << "snapshot-content"; | ||
| 275 | - } | ||
| 276 | - // This value changes when MEM_SNAPSHOT_PARSER_HASH_SALT changes. | ||
| 277 | - EXPECT_EQ( | ||
| 278 | - "adaad4452d193309891d52c7803603f11ef9a73aae9d8fce35df9b22891cb1be", parser->CalculateFileHash(testPicklePath)); | ||
| 279 | } | 191 | } |


校验阶段没有进度输出(实测)
复用时要对全部成员做逻辑摘要。实测 200 万事件用时 9.2–9.5 s;作者数据中 62.8 万事件是 31–41 s。这段时间里没有任何日志或进度输出,C++ 侧拿不到进度,GUI 看起来像卡住了。另外,首次解析从 28.7 s 增加到 37.7 s(实测),多出来的部分主要是 seal 计算。
建议:在
parser_members中每完成一个成员输出一次进度,例如Validating snapshot cache: 3/12,格式沿用现有的report_progress,由 C++ 转发给前端。或者至少在开始校验时先输出Snapshot cache validating,让用户知道正在做什么。