Merged
[MemScope] Unify snapshot cache validation in parser scripts #1305
[MemScope] Unify snapshot cache validation in parser scripts #1305
Merged
heiheiheiAdd on Oct 15
Total: 9 files changed+988-214
@@ -52,6 +52,17 @@ torch_npu.npu.memory._dump_snapshot("memory_snapshot.pickle")
52 52 
53After collection, import <code>memory_snapshot.pickle</code> into MindStudio Insight and analyze it on the memory details (PyTorch Snapshot) interface.53After collection, import <code>memory_snapshot.pickle</code> into MindStudio Insight and analyze it on the memory details (PyTorch Snapshot) interface.
54 54 
55+### Limits of Parsed Cache Validation
56+ 
57+When importing the same Snapshot again, the parser validates existing artifacts
58+before reusing them. A logical digest detects changes to parser-owned base data;
59+consumer-added derived tables or indexes do not change that digest. Legacy artifacts
60+without a historical digest receive only structural, event-range, dictionary-reference
61+and SQLite physical-integrity checks. These checks cannot retrospectively prove that
62+every value is unchanged or that no block rows were deleted. Digests are not security
63+signatures; import only trusted data. Full validation reads the databases and takes
64+longer for larger datasets; it does not mean event replay is running again.
65+ 
55## Case 1: Locating Memory Leaks During the Training Process66## Case 1: Locating Memory Leaks During the Training Process
56 67 
57### Problem Symptom68### Problem Symptom
@@ -44,6 +44,14 @@ torch_npu.npu.memory._dump_snapshot("memory_snapshot.pickle")
44 44 
45采集完成后,将 `memory_snapshot.pickle` 导入 MindStudio Insight,在内存详情(PyTorch Snapshot)界面进行分析。45采集完成后,将 `memory_snapshot.pickle` 导入 MindStudio Insight,在内存详情(PyTorch Snapshot)界面进行分析。
46 46 
47+### 解析缓存校验的限制
48+ 
49+再次导入相同 Snapshot 时,解析脚本会校验已有产物以判断能否复用。带基础数据逻辑摘要的
50+产物可以检查基础解析数据是否发生变化;调用方增加派生表或索引不会改变该摘要。
51+没有历史摘要的旧产物只能进行结构、事件范围、字典引用和 SQLite 物理完整性检查,
52+不能追溯证明所有字段未被修改或内存块记录未被删除。摘要不是安全签名,请仅导入可信数据。
53+大文件的全量校验需要读取数据库,耗时随数据量增长,不代表正在重新回放解析。
54+ 
47## 案例一:定位训练过程中的内存泄漏55## 案例一:定位训练过程中的内存泄漏
48 56 
49### 问题现象57### 问题现象
@@ -20,12 +20,15 @@ import os
20import json20import json
21import sqlite321import sqlite3
22import shutil22import shutil
23+import subprocess
24+import sys
23import unittest25import unittest
24from contextlib import closing26from contextlib import closing
25from pathlib import Path27from pathlib import Path
26from unittest import mock28from unittest import mock
27from util.file_util import load_pickle_to_dict29from util.file_util import load_pickle_to_dict
28from util.logger import suppress_logs, restore_logs30from util.logger import suppress_logs, restore_logs
31+from util.snapshot_cache import source_hashes
29from base import Block, TraceEntry, DeviceSnapshot, BlockState32from base import Block, TraceEntry, DeviceSnapshot, BlockState
30from tools.adaptors import snapshot2db33from tools.adaptors import snapshot2db
31from simulate import SimulateDeviceSnapshot, SimulateHooker34from simulate import SimulateDeviceSnapshot, SimulateHooker
@@ -133,6 +136,7 @@ class Snapshot2DbTest(unittest.TestCase):
133 def testFailedRedumpLeavesBuildingManifest(self):136 def testFailedRedumpLeavesBuildingManifest(self):
134 dest = self.cache_dir / "failed_redump"137 dest = self.cache_dir / "failed_redump"
135 self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))138 self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
139+ (dest / snapshot2db.MANIFEST_FILE_NAME).unlink()
136 original_replay = SimulateDeviceSnapshot.replay_until140 original_replay = SimulateDeviceSnapshot.replay_until
137 141 
138 def fail_replay(self, _min_event_idx, _on_event_processed=None):142 def fail_replay(self, _min_event_idx, _on_event_processed=None):
@@ -146,6 +150,115 @@ class Snapshot2DbTest(unittest.TestCase):
146 manifest = json.loads((dest / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding="utf-8"))150 manifest = json.loads((dest / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding="utf-8"))
147 self.assertEqual(manifest["status"], "building")151 self.assertEqual(manifest["status"], "building")
148 152 
153+ def testCacheHitSkipsPickleAndPreservesConsumerTables(self):
154+ dest = self.cache_dir / "cache_hit"
155+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
156+ db = self.get_slice_db(dest, 0)
157+ with closing(sqlite3.connect(db)) as conn:
158+ conn.execute("CREATE TABLE consumer_cache(value TEXT)")
159+ conn.execute("INSERT INTO consumer_cache VALUES ('keep')")
160+ conn.commit()
161+ with mock.patch.object(snapshot2db, "load_pickle_to_dict", side_effect=AssertionError("cache miss")):
162+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
163+ with closing(sqlite3.connect(db)) as conn:
164+ self.assertEqual(conn.execute("SELECT value FROM consumer_cache").fetchall(), [('keep',)])
165+ 
166+ def testMissingSliceRebuildsWholeArtifact(self):
167+ dest = self.cache_dir / "missing_slice"
168+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
169+ self.get_slice_db(dest, 0).unlink()
170+ with mock.patch.object(snapshot2db, "load_pickle_to_dict", wraps=snapshot2db.load_pickle_to_dict) as loader:
171+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
172+ loader.assert_called_once()
173+ 
174+ def testActiveWriterPreventsRebuildEvenWithMissingManifest(self):
175+ dest = self.cache_dir / 'active_writer'
176+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
177+ db = self.get_slice_db(dest, 0)
178+ script = (
179+ 'import sqlite3, sys; '
180+ 'conn=sqlite3.connect(sys.argv[1]); '
181+ 'conn.execute("PRAGMA journal_mode=DELETE"); '
182+ 'conn.execute("BEGIN EXCLUSIVE"); '
183+ 'conn.execute("UPDATE trace_entry_0 SET size=size+1"); '
184+ 'print("ready", flush=True); '
185+ 'sys.stdin.readline(); conn.rollback(); conn.close()'
186+ )
187+ with subprocess.Popen(
188+ [sys.executable, '-c', script, str(db)],
189+ stdin=subprocess.PIPE,
190+ stdout=subprocess.PIPE,
191+ stderr=subprocess.PIPE,
192+ text=True,
193+ ) as writer:
194+ try:
195+ self.assertEqual(writer.stdout.readline().strip(), 'ready')
196+ self.assertTrue(Path(str(db) + '-journal').exists())
197+ for missing_manifest in (False, True):
198+ if missing_manifest:
199+ (dest / snapshot2db.MANIFEST_FILE_NAME).unlink()
200+ self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0))
201+ self.assertTrue(db.exists())
202+ finally:
203+ writer.communicate('\n', timeout=15)
204+ self.assertEqual(writer.returncode, 0)
205+ 
206+ def testDeepManifestRebuilds(self):
207+ dest = self.cache_dir / 'deep_manifest'
208+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
209+ (dest / snapshot2db.MANIFEST_FILE_NAME).write_text('[' * 200000 + '0' + ']' * 200000)
210+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
211+ 
212+ def testLegacyMissingDictionaryRebuilds(self):
213+ dest = self.cache_dir / 'legacy_dictionary'
214+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
215+ manifest_path = dest / snapshot2db.MANIFEST_FILE_NAME
216+ manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
217+ del manifest['parserCache']
218+ manifest_path.write_text(json.dumps(manifest), encoding='utf-8')
219+ with closing(sqlite3.connect(self.get_slice_db(dest, 0))) as conn:
220+ conn.execute('DELETE FROM dictionary')
221+ conn.commit()
222+ with mock.patch.object(snapshot2db, 'load_pickle_to_dict', wraps=snapshot2db.load_pickle_to_dict) as loader:
223+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
224+ loader.assert_called_once()
225+ 
226+ def testPrepareErrorReturnsFalseWithoutTraceback(self):
227+ dest = self.cache_dir / 'prepare_error'
228+ with mock.patch.object(snapshot2db, '_prepare_output_directory', side_effect=ValueError('symlink')):
229+ with mock.patch.object(snapshot2db.dump_logger, 'error') as error:
230+ self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0))
231+ self.assertIn('Failed to prepare artifact directory', error.call_args.args[0])
232+ 
233+ def testDeviceSymlinkDoesNotRemoveTarget(self):
234+ dest = self.cache_dir / 'symlink_output'
235+ target = self.cache_dir / 'symlink_target'
236+ dest.mkdir()
237+ target.mkdir()
238+ sentinel = target / 'keep.txt'
239+ sentinel.write_text('keep')
240+ link = dest / 'device_0'
241+ try:
242+ link.symlink_to(target, target_is_directory=True)
243+ except OSError as error:
244+ self.skipTest(f'Symlink creation is unavailable: {error}')
245+ self.assertFalse(snapshot2db.dump(self.snapshot_path, dest, 0))
246+ self.assertEqual(sentinel.read_text(), 'keep')
247+ 
248+ def testCorruptIndexRebuilds(self):
249+ dest = self.cache_dir / 'corrupt_index'
250+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
251+ db = self.get_slice_db(dest, 0)
252+ with closing(sqlite3.connect(db)) as conn:
253+ page_size = conn.execute('PRAGMA page_size').fetchone()[0]
254+ root = conn.execute("SELECT rootpage FROM sqlite_master WHERE type='index' LIMIT 1").fetchone()[0]
255+ with db.open('r+b') as file:
256+ file.seek((root - 1) * page_size)
257+ file.write(b'\xff' * page_size)
258+ with mock.patch.object(snapshot2db, 'load_pickle_to_dict', wraps=snapshot2db.load_pickle_to_dict) as loader:
259+ self.assertTrue(snapshot2db.dump(self.snapshot_path, dest, 0))
260+ loader.assert_called_once()
261+ 
149 def testDatabaseInitializationFailureCleansTemporaryFile(self):262 def testDatabaseInitializationFailureCleansTemporaryFile(self):
150 dest = self.cache_dir / "database_init_failure"263 dest = self.cache_dir / "database_init_failure"
151 264 
@@ -176,7 +289,7 @@ class Snapshot2DbTest(unittest.TestCase):
176 manifest = json.loads((output_dir / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding='utf-8'))289 manifest = json.loads((output_dir / snapshot2db.MANIFEST_FILE_NAME).read_text(encoding='utf-8'))
177 device = manifest['devices']['0']290 device = manifest['devices']['0']
178 self.assertEqual(manifest['status'], 'complete')291 self.assertEqual(manifest['status'], 'complete')
179- self.assertEqual(manifest['cacheHash'], 'hash-v1')292+ self.assertEqual(manifest['cacheHash'], source_hashes(snapshot_path)[1])
180 self.assertEqual(device['sliceCount'], 5)293 self.assertEqual(device['sliceCount'], 5)
181 self.assertEqual(device['readySlices'], [0, 1, 2, 3, 4])294 self.assertEqual(device['readySlices'], [0, 1, 2, 3, 4])
182 self.assertEqual(295 self.assertEqual(
@@ -0,0 +1,407 @@
1+"""Cache validation must not depend on consumer-added SQLite objects."""
2+ 
3+import hashlib
4+import json
5+import sqlite3
6+import tempfile
7+import unittest
8+from contextlib import closing
9+from pathlib import Path
10+ 
11+from util.snapshot_cache import _RowEncoder, seal_manifest, source_hashes, validate_cache
12+ 
13+ 
14+class SnapshotCacheTest(unittest.TestCase):
15+ def test_member_log_is_not_duplicated_by_parent_handlers(self):
16+ import io
17+ import logging
18+ from unittest.mock import patch
19+ 
20+ from util.snapshot_cache import LOGGER
21+ 
22+ output = io.StringIO()
23+ handler = logging.StreamHandler(output)
24+ self.addCleanup(handler.close)
25+ with patch.object(LOGGER, "handlers", [handler]), patch.object(logging.getLogger(), "handlers", [handler]):
26+ validate_cache(self.directory, self.hash)
27+ self.assertEqual(output.getvalue().count("Snapshot cache member checked:"), 1)
28+ 
29+ def test_row_encoder_preserves_digest_bytes_and_bounds_cache(self):
30+ encoder = _RowEncoder()
31+ for stack in ('', 'ascii', '\u4e2d\u6587', '"\\\n\t', None, 'x' * (8 * 1024 * 1024)):
32+ row = (1, 2, 3, 4, 0, 5, 6, 7, stack)
33+ expected = json.dumps(row, ensure_ascii=True, separators=(',', ':'), allow_nan=False).encode()
34+ self.assertEqual(encoder.encode(row, True), expected)
35+ self.assertEqual(encoder.encode(row, True), expected)
36+ for index in range(3000):
37+ encoder.encode((index, str(index)), True)
38+ self.assertLessEqual(len(encoder.stacks), 2048)
39+ self.assertLessEqual(encoder.size, 8 * 1024 * 1024)
40+ 
41+ def setUp(self):
42+ # unittest owns the resource until addCleanup runs after each test.
43+ self.temp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with
44+ self.addCleanup(self.temp.cleanup)
45+ self.directory = Path(self.temp.name)
46+ self.source = self.directory / "input.pkl"
47+ self.source.write_bytes(b"trusted-test-source")
48+ self.hash = source_hashes(self.source)[1]
49+ self.db = self.directory / "slice.db"
50+ with closing(sqlite3.connect(self.db)) as conn:
51+ conn.executescript(
52+ 'CREATE TABLE trace_entry_0(id INTEGER PRIMARY KEY, action INTEGER, address INTEGER, '
53+ 'size INTEGER, stream INTEGER, allocated INTEGER, active INTEGER, reserved INTEGER, callstack TEXT);'
54+ 'CREATE TABLE block_0(id INTEGER PRIMARY KEY, address INTEGER, size INTEGER, requestedSize INTEGER, '
55+ 'state INTEGER, allocEventId INTEGER, freeEventId INTEGER);'
56+ 'CREATE TABLE dictionary("table" TEXT, "column" TEXT, "key" TEXT, "value" TEXT);'
57+ "INSERT INTO trace_entry_0 VALUES(0,4,100,10,0,10,10,20,'stack');"
58+ "INSERT INTO block_0 VALUES(0,100,10,10,1,0,-1);"
59+ "INSERT INTO dictionary VALUES('trace_entry_0','action','4','alloc');"
60+ "INSERT INTO dictionary VALUES('block_0','state','1','active_allocated');"
61+ )
62+ self.manifest = {
63+ "schemaVersion": 1,
64+ "status": "complete",
65+ "sourceFile": str(self.source),
66+ "cacheHash": self.hash,
67+ "eventsPerSlice": 500000,
68+ "devices": {
69+ "0": {
70+ "eventCount": 1,
71+ "sliceCount": 1,
72+ "readySlices": [0],
73+ "slices": [{"index": 0, "startEventId": 0, "endEventId": 0, "file": "slice.db", "ready": True}],
74+ }
75+ },
76+ }
77+ seal_manifest(self.directory, self.manifest)
78+ self.save()
79+ 
80+ def save(self):
81+ (self.directory / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8")
82+ 
83+ def test_consumer_write_after_sidecar_check_is_allowed(self):
84+ from unittest.mock import patch
85+ 
86+ from util import snapshot_cache
87+ 
88+ connect = sqlite3.connect
89+ writers = []
90+ self.addCleanup(lambda: [writer.close() for writer in writers])
91+ 
92+ def open_during_write(*args, **kwargs):
93+ # Start the consumer after preflight, when no sidecars existed.
94+ writer = connect(self.db)
95+ writers.append(writer)
96+ writer.execute("PRAGMA journal_mode=WAL")
97+ writer.execute("CREATE TABLE consumer_cache(value INTEGER)")
98+ writer.commit()
99+ return connect(*args, **kwargs)
100+ 
101+ with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_during_write):
102+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
103+ 
104+ def test_base_write_after_sidecar_check_is_detected(self):
105+ from unittest.mock import patch
106+ 
107+ from util import snapshot_cache
108+ 
109+ connect = sqlite3.connect
110+ writers = []
111+ self.addCleanup(lambda: [writer.close() for writer in writers])
112+ 
113+ def open_after_base_write(*args, **kwargs):
114+ writer = connect(self.db)
115+ writers.append(writer)
116+ writer.execute("PRAGMA journal_mode=WAL")
117+ writer.execute("UPDATE block_0 SET size=size+1")
118+ writer.commit()
119+ return connect(*args, **kwargs)
120+ 
121+ with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_after_base_write):
122+ with self.assertRaisesRegex(ValueError, "parser data"):
123+ validate_cache(self.directory, self.hash)
124+ 
125+ def test_consumer_commit_during_row_scan_is_allowed(self):
126+ from unittest.mock import patch
127+ 
128+ from util import snapshot_cache
129+ 
130+ with closing(sqlite3.connect(self.db)) as writer:
131+ writer.execute("PRAGMA journal_mode=WAL")
132+ encode = snapshot_cache._RowEncoder.encode
133+ committed = False
134+ 
135+ def encode_while_writing(encoder, row, inline_stack):
136+ nonlocal committed
137+ if not committed:
138+ writer.execute("CREATE TABLE consumer_cache(value INTEGER)")
139+ writer.commit()
140+ writer.execute("PRAGMA wal_checkpoint(PASSIVE)")
141+ committed = True
142+ return encode(encoder, row, inline_stack)
143+ 
144+ with patch.object(snapshot_cache._RowEncoder, "encode", encode_while_writing):
145+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
146+ self.assertTrue(committed)
147+ 
148+ def test_parser_seal_does_not_override_producer_seal(self):
149+ self.manifest["ptSnap"] = {
150+ "memberHashKind": "parser-v1",
151+ "members": {"slice.db": "0" * 64},
152+ }
153+ self.save()
154+ with self.assertRaisesRegex(ValueError, "member hash"):
155+ validate_cache(self.directory, self.hash)
156+ 
157+ def test_logical_producer_seal_without_parser_seal_is_checked(self):
158+ self.manifest["ptSnap"] = {
159+ "memberHashKind": "parser-v1",
160+ "members": self.manifest.pop("parserCache")["members"],
161+ }
162+ self.save()
163+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
164+ with closing(sqlite3.connect(self.db)) as conn:
165+ conn.execute("UPDATE block_0 SET size=size+1")
166+ conn.commit()
167+ with self.assertRaisesRegex(ValueError, "member hash"):
168+ validate_cache(self.directory, self.hash)
169+ 
170+ def test_legacy_producer_seal_is_not_overridden_by_parser_seal(self):
171+ self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}}
172+ self.save()
173+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
174+ with closing(sqlite3.connect(self.db)) as conn:
175+ conn.execute("CREATE TABLE consumer_extra(value INTEGER)")
176+ with self.assertRaisesRegex(ValueError, "member hash"):
177+ validate_cache(self.directory, self.hash)
178+ 
179+ def test_legacy_physical_seal_accepts_closed_wal_database(self):
180+ with closing(sqlite3.connect(self.db)) as conn:
181+ conn.execute("PRAGMA journal_mode=WAL")
182+ self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}}
183+ self.save()
184+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
185+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
186+ 
187+ def test_legacy_physical_seal_rejects_committed_wal(self):
188+ with closing(sqlite3.connect(self.db)) as conn:
189+ conn.execute("PRAGMA journal_mode=WAL")
190+ self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}}
191+ self.save()
192+ with closing(sqlite3.connect(self.db)) as writer:
193+ writer.execute("CREATE TABLE consumer_cache(value INTEGER)")
194+ writer.commit()
195+ with self.assertRaisesRegex(ValueError, "sidecars"):
196+ validate_cache(self.directory, self.hash)
197+ 
198+ def test_source_hash_matches_shared_salt(self):
199+ raw, salted = source_hashes(self.source)
200+ self.assertEqual(raw, hashlib.sha256(self.source.read_bytes()).hexdigest())
201+ self.assertEqual(salted, hashlib.sha256(b"mem_snapshot_parser_v2" + self.source.read_bytes()).hexdigest())
202+ 
203+ def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self):
204+ with closing(sqlite3.connect(self.db)) as conn:
205+ conn.executescript(
206+ 'CREATE TABLE memory_allocation_cache_v1_0(value INTEGER);'
207+ 'CREATE TABLE arbitrary_consumer_data(value TEXT);'
208+ 'CREATE INDEX consumer_index ON block_0(address);'
209+ 'ALTER TABLE trace_entry_0 ADD COLUMN consumer_note TEXT;'
210+ 'CREATE VIEW consumer_view AS SELECT id FROM block_0;'
211+ 'CREATE TRIGGER consumer_trigger AFTER UPDATE ON block_0 BEGIN SELECT 1; END;'
212+ "INSERT INTO arbitrary_consumer_data VALUES('extra');"
213+ "INSERT INTO dictionary VALUES('arbitrary_consumer_data','value','1','extra');"
214+ )
215+ (self.directory / "parse.log").write_text("consumer log")
216+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
217+ with closing(sqlite3.connect(self.db)) as conn:
218+ conn.execute("VACUUM")
219+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
220+ 
221+ def test_modified_base_data_is_rejected(self):
222+ for table, assignment in (
223+ ("block_0", "size=size+1"),
224+ ("trace_entry_0", "allocated=allocated+1"),
225+ ("dictionary", '"value"=\'changed\''),
226+ ):
227+ with self.subTest(table=table):
228+ with closing(sqlite3.connect(self.db)) as conn:
229+ conn.execute(f"UPDATE {table} SET {assignment}")
230+ conn.commit()
231+ with self.assertRaises(ValueError):
232+ validate_cache(self.directory, self.hash)
233+ seal_manifest(self.directory, self.manifest)
234+ self.save()
235+ 
236+ def test_changed_source_is_rejected(self):
237+ self.source.write_bytes(b"changed")
238+ with self.assertRaises(ValueError):
239+ validate_cache(self.directory, source_hashes(self.source)[1])
240+ 
241+ def test_closed_wal_cache_preserves_database_bytes(self):
242+ with closing(sqlite3.connect(self.db)) as conn:
243+ conn.execute("PRAGMA journal_mode=WAL")
244+ before = self.db.read_bytes()
245+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
246+ self.assertEqual(self.db.read_bytes(), before)
247+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
248+ 
249+ def test_rollback_journal_is_rejected(self):
250+ Path(str(self.db) + '-journal').write_bytes(b'')
251+ with self.assertRaisesRegex(ValueError, 'rollback journal'):
252+ validate_cache(self.directory, self.hash)
253+ 
254+ def test_corrupt_secondary_index_is_rejected(self):
255+ with closing(sqlite3.connect(self.db)) as conn:
256+ conn.execute('CREATE INDEX block_address ON block_0(address)')
257+ page_size = conn.execute('PRAGMA page_size').fetchone()[0]
258+ root = conn.execute("SELECT rootpage FROM sqlite_master WHERE name='block_address'").fetchone()[0]
259+ with self.db.open('r+b') as file:
260+ file.seek((root - 1) * page_size)
261+ file.write(b'\xff' * page_size)
262+ with self.assertRaises((ValueError, sqlite3.DatabaseError)):
263+ validate_cache(self.directory, self.hash)
264+ 
265+ def test_deeply_nested_manifest_is_a_validation_error(self):
266+ (self.directory / 'manifest.json').write_text('[' * 200000 + '0' + ']' * 200000)
267+ with self.assertRaisesRegex(ValueError, 'nesting'):
268+ validate_cache(self.directory, self.hash)
269+ 
270+ def test_legacy_missing_dictionary_reference_is_rejected(self):
271+ del self.manifest['parserCache']
272+ self.save()
273+ with closing(sqlite3.connect(self.db)) as conn:
274+ conn.execute("DELETE FROM dictionary WHERE \"table\"='block_0'")
275+ conn.commit()
276+ with self.assertRaisesRegex(ValueError, 'dictionary reference'):
277+ validate_cache(self.directory, self.hash)
278+ 
279+ def test_legacy_producer_physical_hash_is_checked(self):
280+ del self.manifest['parserCache']
281+ self.manifest['ptSnap'] = {'members': {'slice.db': source_hashes(self.db)[0]}}
282+ self.save()
283+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
284+ with closing(sqlite3.connect(self.db)) as conn:
285+ conn.execute('UPDATE block_0 SET size=size+1')
286+ conn.commit()
287+ with self.assertRaisesRegex(ValueError, 'member hash'):
288+ validate_cache(self.directory, self.hash)
289+ 
290+ def test_member_validation_reports_completed_members(self):
291+ from unittest.mock import patch
292+ 
293+ from util import snapshot_cache
294+ 
295+ with patch.object(snapshot_cache.LOGGER, 'info') as info:
296+ validate_cache(self.directory, self.hash)
297+ info.assert_called_once_with('Snapshot cache member checked: %s', 'slice.db')
298+ 
299+ def test_active_wal_is_read_instead_of_ignoring_committed_rows(self):
300+ with closing(sqlite3.connect(self.db)) as conn:
301+ conn.execute("PRAGMA journal_mode=WAL")
302+ conn.execute("UPDATE block_0 SET size=size+1")
303+ conn.commit()
304+ self.assertTrue(Path(str(self.db) + "-wal").exists())
305+ with self.assertRaises(ValueError):
306+ validate_cache(self.directory, self.hash)
307+ 
308+ def test_missing_or_corrupt_db_is_rejected_without_creating_it(self):
309+ self.db.unlink()
310+ with self.assertRaises(ValueError):
311+ validate_cache(self.directory, self.hash)
312+ self.assertFalse(self.db.exists())
313+ self.db.write_bytes(b"not a database")
314+ with self.assertRaises(sqlite3.DatabaseError):
315+ validate_cache(self.directory, self.hash)
316+ 
317+ def test_incomplete_manifest_or_event_range_is_rejected(self):
318+ for mutation in (
319+ lambda: self.manifest.update(status="building"),
320+ lambda: self.manifest["devices"]["0"].update(readySlices=[]),
321+ ):
322+ mutation()
323+ self.save()
324+ with self.assertRaises(ValueError):
325+ validate_cache(self.directory, self.hash)
326+ self.manifest["status"] = "complete"
327+ with closing(sqlite3.connect(self.db)) as conn:
328+ conn.execute("DELETE FROM trace_entry_0")
329+ conn.commit()
330+ self.manifest["devices"]["0"]["readySlices"] = [0]
331+ self.save()
332+ with self.assertRaises(ValueError):
333+ validate_cache(self.directory, self.hash)
334+ 
335+ def test_missing_json_and_unsafe_member_are_rejected(self):
336+ (self.directory / "manifest.json").unlink()
337+ with self.assertRaises(OSError):
338+ validate_cache(self.directory, self.hash)
339+ self.manifest["devices"]["0"]["slices"][0]["file"] = "../outside.db"
340+ self.save()
341+ with self.assertRaises(ValueError):
342+ validate_cache(self.directory, self.hash)
343+ 
344+ def test_legacy_complete_artifact_is_structurally_validated(self):
345+ del self.manifest["parserCache"]
346+ self.save()
347+ self.assertEqual(validate_cache(self.directory, self.hash), self.manifest)
348+ 
349+ def test_parser_device_selection_does_not_hide_producer_selection(self):
350+ self.manifest["ptSnap"] = {"identity": {"requestedDevice": 0}}
351+ self.save()
352+ with self.assertRaisesRegex(ValueError, "selected device"):
353+ validate_cache(self.directory, self.hash)
354+ self.assertEqual(validate_cache(self.directory, self.hash, device=0), self.manifest)
355+ 
356+ def test_declared_device_must_match_inventory_and_request(self):
357+ for selected in (1, True, "0", -1):
358+ with self.subTest(selected=selected):
359+ self.manifest["parserCache"]["requestedDevice"] = selected
360+ self.save()
361+ with self.assertRaises(ValueError):
362+ validate_cache(self.directory, self.hash, device=0)
363+ 
364+ def test_device_filtered_cache_cannot_satisfy_all_devices(self):
365+ seal_manifest(self.directory, self.manifest, device=0)
366+ self.save()
367+ self.assertEqual(validate_cache(self.directory, self.hash, device=0), self.manifest)
368+ for selection in (None, 1):
369+ with self.assertRaises(ValueError):
370+ validate_cache(self.directory, self.hash, device=selection)
371+ 
372+ def test_invalid_manifest_shape_is_rejected(self):
373+ for data in ([], {**self.manifest, "parserCache": []}, {**self.manifest, "schemaVersion": True}):
374+ (self.directory / "manifest.json").write_text(json.dumps(data))
375+ with self.assertRaises(ValueError):
376+ validate_cache(self.directory, self.hash)
377+ 
378+ def test_duplicate_json_keys_are_rejected(self):
379+ (self.directory / "manifest.json").write_text('{"status":"building","status":"complete"}')
380+ with self.assertRaises(ValueError):
381+ validate_cache(self.directory, self.hash)
382+ 
383+ def test_manifest_changed_during_validation_is_rejected(self):
384+ from unittest.mock import patch
385+ 
386+ from util import snapshot_cache
387+ 
388+ original = snapshot_cache.parser_members
389+ 
390+ def change_manifest(directory, manifest):
391+ members = original(directory, manifest)
392+ self.manifest['consumerRevision'] = 1
393+ self.save()
394+ return members
395+ 
396+ with patch.object(snapshot_cache, 'parser_members', side_effect=change_manifest):
397+ with self.assertRaisesRegex(ValueError, 'manifest changed'):
398+ validate_cache(self.directory, self.hash)
399+ 
400+ def test_parser_salt_changes_cache_identity(self):
401+ from unittest.mock import patch
402+ 
403+ from util import snapshot_cache
404+ 
405+ with patch.object(snapshot_cache, "PARSER_HASH_SALT", b"another-parser-version"):
406+ with self.assertRaises(ValueError):
407+ validate_cache(self.directory, source_hashes(self.source)[1])
@@ -21,6 +21,7 @@ import sys
21import argparse21import argparse
22import json22import json
23import math23import math
24+import re
24import shutil25import shutil
25import sqlite326import sqlite3
26import time27import time
@@ -32,6 +33,13 @@ from tools.adaptors.database import SnapshotDb, block2record, event2record
32from util.file_util import check_dir_valid, load_pickle_to_dict33from util.file_util import check_dir_valid, load_pickle_to_dict
33from util.logger import get_logger, set_global_log_file34from util.logger import get_logger, set_global_log_file
34from util.timer import timer35from util.timer import timer
36+from util.snapshot_cache import (
37+ source_hashes,
38+ validate_cache,
39+ seal_manifest,
40+ ensure_cache_not_busy,
41+ SnapshotCacheBusyError,
42+)
35 43 
36dump_logger = get_logger("DatabaseDump")44dump_logger = get_logger("DatabaseDump")
37DEFAULT_EVENTS_PER_SLICE = 500_00045DEFAULT_EVENTS_PER_SLICE = 500_000
@@ -385,7 +393,14 @@ class SnapshotSliceManifest:
385 393 
386def _prepare_output_directory(output_dir: Path):394def _prepare_output_directory(output_dir: Path):
387 output_dir.mkdir(parents=True, exist_ok=True)395 output_dir.mkdir(parents=True, exist_ok=True)
396+ if output_dir.is_symlink():
397+ raise ValueError("Artifact directory must not be a symlink")
398+ ensure_cache_not_busy(output_dir)
388 for stale_device_dir in output_dir.glob("device_*"):399 for stale_device_dir in output_dir.glob("device_*"):
400+ if re.fullmatch(r"device_[0-9]+", stale_device_dir.name) is None:
401+ continue
402+ if stale_device_dir.is_symlink():
403+ raise ValueError("Snapshot device directory must not be a symlink")
389 if stale_device_dir.is_dir():404 if stale_device_dir.is_dir():
390 shutil.rmtree(stale_device_dir)405 shutil.rmtree(stale_device_dir)
391 (output_dir / MANIFEST_FILE_NAME).unlink(missing_ok=True)406 (output_dir / MANIFEST_FILE_NAME).unlink(missing_ok=True)
@@ -414,6 +429,25 @@ def dump(
414 events_per_slice: int = DEFAULT_EVENTS_PER_SLICE,429 events_per_slice: int = DEFAULT_EVENTS_PER_SLICE,
415 cache_hash: str = "",430 cache_hash: str = "",
416) -> bool:431) -> bool:
432+ # 兼容旧调用参数,但缓存身份必须由脚本计算,不能信任调用方传入的值。
433+ try:
434+ dump_logger.info("Snapshot cache validating: %s", pickle_file)
435+ _, cache_hash = source_hashes(Path(pickle_file))
436+ artifact_dir = Path(output_dir)
437+ try:
438+ cached = validate_cache(artifact_dir, cache_hash, device)
codeasier
codeasiercodeasierOct 15

校验阶段没有进度输出(实测)

复用时要对全部成员做逻辑摘要。实测 200 万事件用时 9.2–9.5 s;作者数据中 62.8 万事件是 31–41 s。这段时间里没有任何日志或进度输出,C++ 侧拿不到进度,GUI 看起来像卡住了。另外,首次解析从 28.7 s 增加到 37.7 s(实测),多出来的部分主要是 seal 计算。

建议:在 parser_members 中每完成一个成员输出一次进度,例如 Validating snapshot cache: 3/12,格式沿用现有的 report_progress,由 C++ 转发给前端。或者至少在开始校验时先输出 Snapshot cache validating,让用户知道正在做什么。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
439+ except SnapshotCacheBusyError as error:
440+ dump_logger.error("Failed to validate snapshot cache: %s", error)
441+ return False
442+ except (OSError, ValueError, TypeError, KeyError, sqlite3.Error):
codeasier
codeasiercodeasierOct 15

manifest 嵌套过深时抛出 RecursionError,没有走重建(实测)

manifest 嵌套 20 万层时,第 228 行(snapshot_cache.py)的 json.loads 抛出 RecursionError。这里的 except 列表没有覆盖它,dump() 直接以 traceback 退出(rc=1),不会重建。同一套矩阵里,ptSnap=[] 等类型错误会被 snapshot_cache.py:236 正确拦截并重建(实测)。

建议:这里改为 except (OSError, ValueError, TypeError, KeyError, AttributeError, RecursionError, sqlite3.Error)。其中 AttributeError 属于防御性补充(推断,本轮没有构造出触发路径)。也可以在 validate_cache 内部把这些异常统一转成 ValueError。

最小用例:写入一个嵌套 20 万层的 manifest.json,期望 dump() 输出 Snapshot cache rebuild 并返回 True。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
443+ cached = None
444+ if cached is not None:
445+ dump_logger.info("Snapshot cache validated: %s", cache_hash)
446+ dump_logger.info("Successfully dump the snapshot to database for devices %s.", list(cached["devices"]))
447+ return True
448+ except (OSError, ValueError) as error:
449+ dump_logger.error("Failed to validate snapshot source: %s", error)
450+ return False
417 try:451 try:
418 data = load_pickle_to_dict(Path(pickle_file))452 data = load_pickle_to_dict(Path(pickle_file))
419 except Exception as e:453 except Exception as e:
@@ -435,8 +469,12 @@ def dump(
435 if events_per_slice <= 0:469 if events_per_slice <= 0:
436 dump_logger.error("Events per slice must be greater than 0.")470 dump_logger.error("Events per slice must be greater than 0.")
437 return False471 return False
438- artifact_dir = Path(output_dir)472+ try:
439- _prepare_output_directory(artifact_dir)473+ _prepare_output_directory(artifact_dir)
474+ except (OSError, ValueError, sqlite3.Error, SnapshotCacheBusyError) as error:
475+ dump_logger.error("Failed to prepare artifact directory: %s", error)
476+ return False
477+ dump_logger.info("Snapshot cache rebuild: %s", cache_hash)
440 manifest = SnapshotSliceManifest(478 manifest = SnapshotSliceManifest(
441 artifact_dir,479 artifact_dir,
442 pickle_file,480 pickle_file,
@@ -507,6 +545,11 @@ def dump(
507 )545 )
508 return False546 return False
509 dump_logger.info("Finished dump the snapshot to database for device %s.", dev)547 dump_logger.info("Finished dump the snapshot to database for device %s.", dev)
548+ if source_hashes(Path(pickle_file))[1] != cache_hash:
549+ raise ValueError("Snapshot source changed during parsing")
550+ # 基础表逻辑摘要在最终反插完成后生成,调用方新增派生表不会使缓存失效。
551+ manifest.data["status"] = "complete"
552+ seal_manifest(artifact_dir, manifest.data, device)
510 manifest.mark_complete()553 manifest.mark_complete()
511 dump_logger.info("Successfully dump the snapshot to database for devices %s.", need_dump_devices)554 dump_logger.info("Successfully dump the snapshot to database for devices %s.", need_dump_devices)
512 return True555 return True
@@ -555,7 +598,7 @@ def get_args():
555 "--cache_hash",598 "--cache_hash",
556 type=str,599 type=str,
557 default="",600 default="",
558- help="Salted source hash used by the server to validate parsed artifacts.",601+ help="Deprecated compatibility argument; ignored. The parser calculates the source hash.",
559 )602 )
560 args = parser.parse_args()603 args = parser.parse_args()
561 snapshot_path = Path(args.snapshot_file)604 snapshot_path = Path(args.snapshot_file)
@@ -0,0 +1,337 @@
1+"""Portable msinsight cache contract; consumer-owned tables are not parser data."""
2+ 
3+from __future__ import annotations
4+ 
5+import hashlib
6+import json
7+import os
8+import re
9+import sqlite3
10+import sys
11+from collections import OrderedDict
12+from contextlib import closing
13+from pathlib import Path
14+from typing import Any
15+ 
16+from util.logger import get_logger
17+ 
18+PARSER_HASH_SALT = b"mem_snapshot_parser_v2"
19+CACHE_CONTRACT_VERSION = 1
20+TRACE_COLUMNS = (
21+ "id",
22+ "action",
23+ "address",
24+ "size",
25+ "stream",
26+ "allocated",
27+ "active",
28+ "reserved",
29+ "callstack",
30+)
31+BLOCK_COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId")
32+LOGGER = get_logger("SnapshotCache")
33+# 全局文件 handler 会挂到各级 logger,禁止向父级传播导致同一条日志重复写入。
34+LOGGER.propagate = False
35+ 
36+ 
37+class SnapshotCacheBusyError(RuntimeError):
38+ """An active SQLite writer must not be treated as an invalid cache."""
39+ 
40+ 
41+def _probe_writer(path: Path) -> None:
42+ # Only used before rebuilding or recovering an invalid rollback-journal cache.
43+ try:
44+ with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=rw", uri=True, timeout=5)) as conn:
45+ conn.execute("BEGIN IMMEDIATE")
46+ conn.rollback()
47+ except sqlite3.OperationalError as error:
48+ if "locked" in str(error).lower() or "busy" in str(error).lower():
49+ raise SnapshotCacheBusyError("Snapshot member is being written by another process") from error
50+ raise
51+ 
52+ 
53+def ensure_cache_not_busy(directory: Path) -> None:
54+ """Preflight all existing members, even when the manifest itself is invalid."""
55+ for device_dir in directory.glob("device_*"):
56+ if re.fullmatch(r"device_[0-9]+", device_dir.name) is None:
57+ continue
58+ if device_dir.is_symlink():
59+ raise ValueError("Snapshot device directory must not be a symlink")
60+ for path in device_dir.glob("*.db"):
61+ if path.is_symlink() or any(
62+ Path(str(path) + suffix).is_symlink() for suffix in ("-wal", "-shm", "-journal")
63+ ):
64+ raise ValueError("Snapshot has an aliased member or sidecar")
65+ try:
66+ _probe_writer(path)
67+ except sqlite3.DatabaseError as error:
68+ # Corrupt closed databases are rebuildable; lock errors propagate separately.
69+ if "malformed" not in str(error).lower() and "not a database" not in str(error).lower():
70+ raise
71+ 
72+ 
73+class _RowEncoder:
74+ """Bound repeated callstack encoding; preserve the v1 digest bytes."""
75+ 
76+ def __init__(self) -> None:
77+ self.stacks: OrderedDict[str, bytes] = OrderedDict()
78+ self.size = 0
79+ 
80+ def encode(self, row: tuple[Any, ...], inline_stack: bool) -> bytes:
81+ if not inline_stack or not isinstance(row[-1], str):
82+ return json.dumps(row, ensure_ascii=True, separators=(",", ":"), allow_nan=False).encode()
83+ stack = row[-1]
84+ encoded = self.stacks.get(stack)
85+ if encoded is None:
86+ encoded = json.dumps(stack, ensure_ascii=True, allow_nan=False).encode()
87+ cost = sys.getsizeof(stack) + sys.getsizeof(encoded)
88+ if cost <= 8 * 1024 * 1024:
89+ while self.stacks and (self.size + cost > 8 * 1024 * 1024 or len(self.stacks) >= 2048):
90+ key, value = self.stacks.popitem(last=False)
91+ self.size -= sys.getsizeof(key) + sys.getsizeof(value)
92+ self.stacks[stack] = encoded
93+ self.size += cost
94+ else:
95+ self.stacks.move_to_end(stack)
96+ prefix = json.dumps(row[:-1], separators=(",", ":"), allow_nan=False).encode()
97+ return prefix[:-1] + b"," + encoded + b"]"
98+ 
99+ 
100+def source_hashes(source: Path) -> tuple[str, str]:
101+ """Return raw and salted SHA-256 in one streaming pass, without unpickling."""
102+ raw = hashlib.sha256()
103+ salted = hashlib.sha256(PARSER_HASH_SALT)
104+ with source.open("rb") as file:
105+ before = os.fstat(file.fileno())
106+ for chunk in iter(lambda: file.read(1024 * 1024), b""):
107+ raw.update(chunk)
108+ salted.update(chunk)
109+ after = os.fstat(file.fileno())
110+ current = source.stat()
111+ 
112+ def signature(stat: os.stat_result) -> tuple[int, int, int, int]:
113+ # Windows fstat/stat disagree on ctime for some files; it is not content identity.
114+ return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns
115+ 
116+ if signature(before) != signature(after) or signature(after) != signature(current):
117+ raise ValueError("Snapshot source changed while hashing")
118+ return raw.hexdigest(), salted.hexdigest()
119+ 
120+ 
121+def _integer(value: Any, minimum: int = 0) -> int:
122+ if not isinstance(value, int) or isinstance(value, bool) or value < minimum or value > 2**63 - 1:
123+ raise ValueError("Invalid nonnegative integer in snapshot manifest")
124+ return value
125+ 
126+ 
127+def _member(directory: Path, relative: str) -> Path:
128+ if not isinstance(relative, str) or not relative or "\\" in relative:
129+ raise ValueError("Invalid snapshot member path")
130+ path = Path(relative)
131+ if path.is_absolute() or any(part in (".", "..") for part in relative.split("/")):
132+ raise ValueError("Snapshot member escapes artifact directory")
133+ candidate = directory / path
134+ if not candidate.is_file() or candidate.resolve() != directory.resolve() / path:
135+ raise ValueError("Missing or aliased snapshot member")
136+ return candidate
137+ 
138+ 
139+def _table_rows(
140+ conn: sqlite3.Connection,
141+ name: str,
142+ columns: tuple[str, ...],
143+ where: str = "",
144+ parameters: tuple[str, ...] = (),
145+) -> sqlite3.Cursor:
146+ if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() != ("table",):
147+ raise ValueError(f"Missing parser table: {name}")
148+ schema = conn.execute(f'PRAGMA table_info("{name}")').fetchall()
149+ actual = {row[1]: row for row in schema}
150+ for column in columns:
151+ kind = "TEXT" if column in ("callstack", "table", "column", "key", "value") else "INTEGER"
152+ if column not in actual or actual[column][2].upper() != kind:
codeasier
codeasiercodeasierOct 15

ADD COLUMN、view、trigger 也会被接受(实测,低)

这里只检查必需列是否存在、类型是否正确,所以 ALTER TABLE trace_entry_0 ADD COLUMN x、CREATE VIEW、CREATE TRIGGER、VACUUM 之后都仍然判定 validated。在宽松契约下这可以接受,不要求改成严格校验。但 PR 描述和这里的 docstring 只提到了"extra tables, indexes and dictionary entries"。

建议:在 docstring 和说明里把允许的范围写完整,即"基础表的额外列、view、trigger 也允许,摘要只覆盖固定列"。可以补一条 ADD COLUMN 后仍然 validated 的用例,把这个行为固定下来。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
153+ raise ValueError(f"Invalid parser column: {name}.{column}")
154+ if "id" in columns and [row[1] for row in schema if row[5]] != ["id"]:
155+ raise ValueError(f"Missing parser primary key: {name}.id")
156+ selected = ",".join(f'"{column}"' for column in columns)
157+ order = '"id"' if "id" in columns else selected
158+ # Identifiers and the optional predicate are internal constants/validated device IDs.
159+ return conn.execute(f'SELECT {selected} FROM "{name}" {where} ORDER BY {order}', parameters) # nosec B608
160+ 
161+ 
162+def member_digest(path: Path, device: str, start: int, end: int) -> str:
codeasier
codeasiercodeasierOct 15

索引页物理损坏没有被检出(实测)

member_digest 只按 ORDER BY id 读主表 b-tree,不访问二级索引。在 slice_00001.db 中把某个索引的页填满垃圾数据后,validate_cache 仍判定通过,输出 Snapshot cache validated,C++ 直接复用;后续凡是走这个索引的查询,结果都会出错或报 database disk image is malformed。pt-snap #268 中同一份代码也存在这个问题。

建议:在 BEGIN 之后加

if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]:
    raise ValueError("Snapshot member failed integrity check")

实测开销:每个约 137 MB 的分片,热缓存约 0.2 s,冷缓存约 1.6 s。

最小用例:建库后用 PRAGMA page_size 结合 sqlite_master.rootpage 定位某个索引的根页,写入垃圾数据,期望 validate_cache 抛出 ValueError,dump() 走重建流程。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
163+ """Hash fixed base columns; extra columns, tables, indexes, views and triggers are allowed.
164+ 
165+ Unrelated dictionary entries are also allowed. Legacy artifacts without a seal
166+ receive structure/reference checks, not proof that all historical values are unchanged.
167+ """
168+ if re.fullmatch(r"0|[1-9][0-9]*", str(device)) is None:
169+ raise ValueError("Invalid device ID")
170+ digest = hashlib.sha256()
171+ encoder = _RowEncoder()
172+ # Consumers may start writing after the sidecar check. Always use SQLite's
173+ # locking/WAL protocol and one read snapshot, even for a currently closed DB.
174+ sidecars = [Path(str(path) + suffix) for suffix in ("-wal", "-shm", "-journal")]
175+ if any(item.is_symlink() for item in sidecars):
176+ raise ValueError("Snapshot has an aliased sidecar")
177+ if sidecars[2].exists():
178+ _probe_writer(path)
179+ raise ValueError("Snapshot has a rollback journal")
180+ uri = path.resolve().as_uri() + "?mode=ro"
181+ with closing(sqlite3.connect(uri, uri=True)) as conn:
182+ conn.execute("PRAGMA query_only=ON")
183+ conn.execute("BEGIN")
184+ if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]:
185+ raise ValueError("Snapshot member failed integrity check")
186+ tables = ((f"trace_entry_{device}", TRACE_COLUMNS), (f"block_{device}", BLOCK_COLUMNS))
187+ for table, columns in tables:
188+ digest.update(json.dumps((table, columns), separators=(",", ":")).encode())
189+ for row in _table_rows(conn, table, columns):
190+ digest.update(encoder.encode(row, columns == TRACE_COLUMNS))
191+ digest.update(b"\n")
192+ count, low, high = conn.execute(
193+ f'SELECT COUNT(*), MIN(id), MAX(id) FROM "trace_entry_{device}" WHERE id>=0' # nosec B608 - device validated above
194+ ).fetchone()
195+ if (count, low, high) != (end - start + 1, start, end):
196+ raise ValueError("Snapshot event range is incomplete")
197+ for row in _table_rows(
198+ conn,
199+ "dictionary",
200+ ("table", "column", "key", "value"),
201+ 'WHERE "table" IN (?, ?)',
202+ tuple(table for table, _ in tables),
203+ ):
204+ digest.update(json.dumps(row, ensure_ascii=True, separators=(",", ":")).encode())
205+ digest.update(b"\n")
206+ for table, column in ((f"trace_entry_{device}", "action"), (f"block_{device}", "state")):
207+ # Both identifiers are internal constants with the device validated above.
208+ missing = conn.execute(
209+ f'SELECT 1 FROM (SELECT DISTINCT "{column}" AS value FROM "{table}") AS base WHERE NOT EXISTS ' # nosec B608
210+ '(SELECT 1 FROM dictionary AS dict WHERE dict."table"=? AND dict."column"=? '
211+ 'AND dict."key"=CAST(base.value AS TEXT)) LIMIT 1',
212+ (table, column),
213+ ).fetchone()
214+ if missing:
215+ raise ValueError("Snapshot dictionary reference is missing")
216+ return digest.hexdigest()
217+ 
218+ 
219+def parser_members(directory: Path, manifest: dict[str, Any]) -> dict[str, str]:
220+ """Validate the complete manifest and all DBs; never deserialize the source."""
221+ if _integer(manifest.get("schemaVersion"), 1) != 1:
222+ raise ValueError("Unsupported snapshot manifest version")
223+ if (
224+ manifest.get("status") != "complete"
225+ or not isinstance(manifest.get("sourceFile"), str)
226+ or not manifest["sourceFile"]
227+ ):
228+ raise ValueError("Snapshot is incomplete")
229+ capacity = _integer(manifest.get("eventsPerSlice"), 1)
230+ devices = manifest.get("devices")
231+ if not isinstance(devices, dict) or not devices:
232+ raise ValueError("Missing snapshot devices")
233+ members = {}
234+ for device, data in devices.items():
235+ if re.fullmatch(r"0|[1-9][0-9]*", device) is None:
236+ raise ValueError("Invalid device ID")
237+ if not isinstance(data, dict):
238+ raise ValueError("Invalid device inventory")
239+ count = _integer(data["eventCount"], 1)
240+ size = _integer(data["sliceCount"], 1)
241+ if size != (count + capacity - 1) // capacity or len(data["slices"]) != size:
242+ raise ValueError("Incomplete snapshot slice inventory")
243+ if sorted(_integer(index) for index in data["readySlices"]) != list(range(size)):
244+ raise ValueError("Snapshot slices are not ready")
245+ for index, item in enumerate(data["slices"]):
246+ start, end = index * capacity, min((index + 1) * capacity, count) - 1
247+ if (
248+ _integer(item["index"]),
249+ _integer(item["startEventId"]),
250+ _integer(item["endEventId"]),
251+ ) != (index, start, end) or item["ready"] is not True:
252+ raise ValueError("Invalid snapshot slice range")
253+ relative = item["file"]
254+ if relative in members:
255+ raise ValueError("Duplicate snapshot member")
256+ members[relative] = member_digest(_member(directory, relative), device, start, end)
257+ LOGGER.info("Snapshot cache member checked: %s", relative)
258+ return members
259+ 
260+ 
261+def seal_manifest(directory: Path, manifest: dict[str, Any], device: int | None = None) -> None:
262+ manifest["parserCache"] = {
263+ "version": CACHE_CONTRACT_VERSION,
264+ "requestedDevice": device,
265+ "members": parser_members(directory, manifest),
266+ }
267+ 
268+ 
269+def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
270+ result = {}
271+ for key, value in pairs:
272+ if key in result:
273+ raise ValueError("Duplicate snapshot manifest key")
274+ result[key] = value
275+ return result
276+ 
277+ 
278+def validate_cache(directory: Path, cache_hash: str, device: int | None = None) -> dict[str, Any]:
279+ path = directory / "manifest.json"
280+ original = path.read_bytes()
281+ try:
282+ manifest = json.loads(original, object_pairs_hook=_unique_object)
283+ except RecursionError as error:
284+ raise ValueError("Snapshot manifest nesting is too deep") from error
285+ if not isinstance(manifest, dict):
286+ raise ValueError("Invalid snapshot manifest")
287+ if not cache_hash or manifest.get("cacheHash") != cache_hash:
288+ raise ValueError("Snapshot source hash or parser salt changed")
289+ if device is not None and set(manifest["devices"]) != {str(device)}:
290+ raise ValueError("Snapshot device selection changed")
291+ for extension in ("parserCache", "ptSnap"):
292+ if extension in manifest and not isinstance(manifest[extension], dict):
293+ raise ValueError("Invalid snapshot extension")
294+ if not isinstance(manifest.get("ptSnap", {}).get("identity", {}), dict):
295+ raise ValueError("Invalid producer identity")
296+ # Validate each declaration independently; an absent selection in one seal
297+ # must not mask a producer's device-filtered export.
298+ for selection in (
299+ manifest.get("parserCache", {}),
300+ manifest.get("ptSnap", {}).get("identity", {}),
301+ ):
302+ selected_device = selection.get("requestedDevice")
303+ if selected_device is None:
304+ continue
305+ _integer(selected_device)
306+ if device is None:
307+ raise ValueError("Snapshot cache contains only a selected device")
308+ if selected_device != device or set(manifest["devices"]) != {str(selected_device)}:
309+ raise ValueError("Snapshot device selection changed")
310+ members = parser_members(directory, manifest)
311+ attestation = manifest.get("parserCache")
312+ # Legacy producers have no logical attestation; still validate every base table and range.
codeasier
codeasiercodeasierOct 15

旧版无 seal 产物:行级篡改无法检出(实测,中)

manifest 里没有 parserCache 时,只校验表、列、主键和 id 区间。以下改动都会被判为 validated 并复用:

  • 删除 block 行;
  • 修改 trace 或 block 的值;
  • 删除 trace/block 用到的 dictionary 条目。

另外,pt-snap main 导出的 ptSnap.members 里本来就带有每个成员的字节级 sha256,这里完全没有使用。

建议(都符合宽松契约,只要求"异常就重建"):

  1. 有 ptSnap.members(并且 memberHashKind 不是 parser-v1)时,校验文件的 sha256。如果担心 GUI 流程追加的派生表导致误判,可以在不匹配时直接重建,因为旧产物只会重建一次。
  2. 增加轻量一致性检查:trace 和 block 表中引用 dictionary 的列,所有取值都能在 dictionary 里找到;block.allocEventId/freeEventId 为 -1,或落在本 slice 的事件区间内。
  3. 如果以上都不做,至少在说明里写明"旧产物只做结构校验"。

最小用例:用 master 生成旧产物,删除一个 dictionary 条目,期望 dump() 输出 Snapshot cache rebuild。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
313+ if attestation is not None and (
314+ _integer(attestation.get("version"), 1) != CACHE_CONTRACT_VERSION or attestation.get("members") != members
315+ ):
316+ raise ValueError("Snapshot parser data changed")
317+ producer = manifest.get("ptSnap", {})
318+ if "memberHashKind" in producer and producer["memberHashKind"] != "parser-v1":
319+ raise ValueError("Unsupported member hash contract")
320+ if producer.get("memberHashKind") == "parser-v1" and producer.get("members") != members:
321+ raise ValueError("Snapshot producer member hash changed")
322+ if "members" in producer and producer.get("memberHashKind") != "parser-v1":
323+ physical = producer["members"]
324+ if not isinstance(physical, dict) or set(physical) != set(members):
325+ raise ValueError("Invalid legacy snapshot member inventory")
326+ for relative, expected in physical.items():
327+ member = _member(directory, relative)
328+ wal = Path(str(member) + "-wal")
329+ # A read-only open can create an empty WAL and SHM. They contain no
330+ # committed rows; only the DB bytes need the legacy physical digest.
331+ if Path(str(member) + "-journal").exists() or (wal.exists() and wal.stat().st_size):
332+ raise ValueError("Legacy physical snapshot hash cannot validate live sidecars")
333+ if source_hashes(member)[0] != expected:
334+ raise ValueError("Legacy snapshot member hash changed")
335+ if path.read_bytes() != original:
336+ raise ValueError("Snapshot manifest changed during validation")
337+ return manifest
@@ -20,7 +20,6 @@
20#include "PythonUtil.h"20#include "PythonUtil.h"
21#include "SafeFile.h"21#include "SafeFile.h"
22#include "DataBaseManager.h"22#include "DataBaseManager.h"
23-#include "HashUtil.h"
24#include "MemSnapshotParser.h"23#include "MemSnapshotParser.h"
25#include "MemSnapshotDatabase.h"24#include "MemSnapshotDatabase.h"
26#include "MemSnapshotSliceService.h"25#include "MemSnapshotSliceService.h"
@@ -40,7 +39,6 @@
40namespace Dic::Module {39namespace Dic::Module {
41using namespace Dic::Module::Timeline;40using namespace Dic::Module::Timeline;
42using Dic::OpenReadFileSafely;41using Dic::OpenReadFileSafely;
43-constexpr std::string_view MEM_SNAPSHOT_PARSER_HASH_SALT = "mem_snapshot_parser_v2";
44constexpr int MEM_SNAPSHOT_EVENTS_PER_SLICE = 500000;42constexpr int MEM_SNAPSHOT_EVENTS_PER_SLICE = 500000;
45 43 
46void MemSnapshotParserContext::Reset(44void MemSnapshotParserContext::Reset(
@@ -74,7 +72,15 @@ std::string MemSnapshotParserContext::GetLogPath() const { return logPath; }
74 72 
75std::string MemSnapshotParserContext::GetOutputDbPath() const { return outputDbPath; }73std::string MemSnapshotParserContext::GetOutputDbPath() const { return outputDbPath; }
76 74 
77-std::string MemSnapshotParserContext::GetFileHash() const { return fileHash; }75+std::string MemSnapshotParserContext::GetFileHash() const {
76+ std::shared_lock<std::shared_mutex> lock(_mutex);
77+ return fileHash;
78+}
79+ 
80+void MemSnapshotParserContext::SetFileHash(std::string hash) {
81+ std::unique_lock<std::shared_mutex> lock(_mutex);
82+ fileHash = std::move(hash);
83+}
78 84 
79bool MemSnapshotParserContext::IsInitialSuccessSent() const {85bool MemSnapshotParserContext::IsInitialSuccessSent() const {
80 std::shared_lock<std::shared_mutex> lock(_mutex);86 std::shared_lock<std::shared_mutex> lock(_mutex);
@@ -154,20 +160,12 @@ void MemSnapshotParser::Reset() {
154 MemSnapshotDatabase::Reset();160 MemSnapshotDatabase::Reset();
155}161}
156 162 
157-std::string MemSnapshotParser::CalculateFileHash(const std::string &filePath) {
158- const std::string hash = HashUtil::CalculateFileSha256(filePath, MEM_SNAPSHOT_PARSER_HASH_SALT);
159- if (hash.empty()) {
160- Server::ServerLog::Error("[Snapshot] Failed to open pickle file for hashing: %.", filePath);
161- }
162- return hash;
163-}
164- 
165void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFilePath) {163void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFilePath) {
166 const std::string realPicklePath = FileUtil::GetRealPath(pickleFilePath);164 const std::string realPicklePath = FileUtil::GetRealPath(pickleFilePath);
167 const std::string &snapshotPath = realPicklePath.empty() ? pickleFilePath : realPicklePath;165 const std::string &snapshotPath = realPicklePath.empty() ? pickleFilePath : realPicklePath;
168 const std::string outputPath = MemSnapshotSliceService::GetArtifactDirectory(snapshotPath);166 const std::string outputPath = MemSnapshotSliceService::GetArtifactDirectory(snapshotPath);
169 const std::string logPath = MemSnapshotSliceService::GetLogPath(snapshotPath);167 const std::string logPath = MemSnapshotSliceService::GetLogPath(snapshotPath);
170- parseContext.Reset(snapshotPath, logPath, outputPath, CalculateFileHash(snapshotPath));168+ parseContext.Reset(snapshotPath, logPath, outputPath);
171 auto traceId = TraceIdManager::GenerateTraceId();169 auto traceId = TraceIdManager::GenerateTraceId();
172 Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, log file: %, output db file: %.",170 Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, log file: %, output db file: %.",
173 parseContext.GetPicklePath(), parseContext.GetLogPath(), parseContext.GetOutputDbPath());171 parseContext.GetPicklePath(), parseContext.GetLogPath(), parseContext.GetOutputDbPath());
@@ -177,62 +175,6 @@ void MemSnapshotParser::AsyncParseMemSnapshotPickle(const std::string &pickleFil
177 175 
178MemSnapshotParserContext &MemSnapshotParser::GetParseContext() { return parseContext; }176MemSnapshotParserContext &MemSnapshotParser::GetParseContext() { return parseContext; }
179 177 
180-/***
181- * 在满足以下任意条件之一(按顺序检查)时,需要重新解析pickle文件:
182- * 1. 数据库连接打开失败或db初始化失败
183- * 2. 通过2的校验,但数据库版本与当前版本不一致
184- *
185- * @brief 检查是否需要解析或重新解析pickle文件
186- * @return true 需要解析或重新解析,此时会关闭已打开的连接并清空DatabaseManager纳管实例。
187- * @return false 不需要解析或重新解析。此时将不会清空DatabaseManager及纳管实例,可以不需要重复打开。
188- */
189-bool MemSnapshotParser::CheckIfParsingNeed(const MemSnapshotParserContext &context) {
190- const auto manifest = MemSnapshotSliceService::LoadManifest(context.GetPicklePath());
191- if (!manifest.has_value() || !manifest->IsComplete()) {
192- Server::ServerLog::Info("[Snapshot] Slice manifest is missing or incomplete; the file needs to re-parse.");
193- MemSnapshotDatabase::Reset();
194- return true;
195- }
196- const std::string snapshotCacheHash = context.GetFileHash();
197- const std::string &cachedSnapshotHash = manifest->cacheHash;
198- if (snapshotCacheHash.empty() || cachedSnapshotHash.empty() || snapshotCacheHash != cachedSnapshotHash) {
199- Server::ServerLog::Info(
200- "[Snapshot] Snapshot cache hash changed or is unavailable. The file needs to re-parse.");
201- MemSnapshotDatabase::Reset();
202- return true;
203- }
204- for (const auto &[deviceId, device] : manifest->devices) {
205- for (const auto &slice : device.slices) {
206- const auto dbPath = MemSnapshotSliceService::ResolveSliceDbPath(context.GetPicklePath(), slice);
207- if (!slice.ready || dbPath.empty() || !FileUtil::IsFilePathExist(dbPath)) {
208- Server::ServerLog::Info(
209- "[Snapshot] Slice artifact is missing for device %, slice %; the file needs to re-parse.", deviceId,
210- slice.index);
211- MemSnapshotDatabase::Reset();
212- return true;
213- }
214- std::recursive_mutex validationMutex;
215- FullDb::MemSnapshotDatabase validationDatabase(validationMutex);
216- if (!validationDatabase.OpenDbReadOnly(dbPath) || !validationDatabase.IsDeviceIdValid(deviceId)) {
217- Server::ServerLog::Info(
218- "[Snapshot] Slice database is incomplete for device %, slice %; the file needs to re-parse.",
219- deviceId, slice.index);
220- MemSnapshotDatabase::Reset();
221- return true;
222- }
223- if (!FullDb::MemSnapshotDatabase::HasMemoryAllocationCache(dbPath, deviceId) &&
224- !FullDb::MemSnapshotDatabase::BuildMemoryAllocationCache(dbPath, deviceId)) {
225- Server::ServerLog::Info(
226- "[Snapshot] Allocation cache is missing for device %, slice %; the file needs to re-parse.",
227- deviceId, slice.index);
228- MemSnapshotDatabase::Reset();
229- return true;
230- }
231- }
232- }
233- return false;
234-}
235- 
236MemSnapshotParser::MemSnapshotParser() {178MemSnapshotParser::MemSnapshotParser() {
237 // MemSnapshot解析snapshot解析至少需要双线程179 // MemSnapshot解析snapshot解析至少需要双线程
238 _threadPool = std::make_unique<ThreadPool>(2);180 _threadPool = std::make_unique<ThreadPool>(2);
@@ -242,28 +184,19 @@ MemSnapshotParser::~MemSnapshotParser() { _threadPool->ShutDown(); }
242 184 
243void MemSnapshotParser::ParseMemSnapshotTask() {185void MemSnapshotParser::ParseMemSnapshotTask() {
244 Server::ServerLog::Info("[Snapshot] Parse snapshot thread started.");186 Server::ServerLog::Info("[Snapshot] Parse snapshot thread started.");
245- if (!CheckIfParsingNeed(Instance().parseContext)) {187+ // 缓存校验、旧解析产物清理由脚本统一负责;这里只清理本次进度日志。
246- Server::ServerLog::Info("[Snapshot] Parsing pickle file: %, artifact directory: % is up-to-date.",188+ MemSnapshotDatabase::Reset();
247- Instance().parseContext.GetPicklePath(), Instance().parseContext.GetOutputDbPath());189+ const auto logPath = Instance().parseContext.GetLogPath();
248- Instance().parseContext.SetProgress(100);190+ if (FileUtil::IsFilePathExist(logPath) && !FileUtil::RemoveFile(logPath)) {
249- Instance().parseContext.SetState(ParserState::UP_TO_DATE);191+ Server::ServerLog::Error("[Snapshot] Failed to remove stale parsing log: %.", logPath);
192+ Instance().parseContext.SetState(ParserState::FINISH_FAILURE);
250 return;193 return;
251 }194 }
252- // 需要首次解析或重新解析的场景
253- const std::vector<std::string> staleStateFiles = {
254- Instance().parseContext.GetLogPath(),
255- MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath()),
256- };
257- for (const auto &path : staleStateFiles) {
258- if (FileUtil::IsFilePathExist(path) && !FileUtil::RemoveFile(path)) {
259- Server::ServerLog::Warn("[Snapshot] Failed to remove stale parsing state file: %.", path);
260- }
261- }
262 const std::string memSnapDumpScriptsPath = FileUtil::SplicePath("mem_snap_dump", "tools", "dump2db.py");195 const std::string memSnapDumpScriptsPath = FileUtil::SplicePath("mem_snap_dump", "tools", "dump2db.py");
263 Server::ServerLog::Info("[Snapshot] Start parsing.");196 Server::ServerLog::Info("[Snapshot] Start parsing.");
264 std::vector<std::string> arguments{Instance().parseContext.GetPicklePath(), "--dump_dir",197 std::vector<std::string> arguments{Instance().parseContext.GetPicklePath(), "--dump_dir",
265- Instance().parseContext.GetOutputDbPath(), "--log", Instance().parseContext.GetLogPath(), "--cache_hash",198+ Instance().parseContext.GetOutputDbPath(), "--log", Instance().parseContext.GetLogPath(), "--events_per_slice",
266- Instance().parseContext.GetFileHash(), "--events_per_slice", std::to_string(MEM_SNAPSHOT_EVENTS_PER_SLICE)};199+ std::to_string(MEM_SNAPSHOT_EVENTS_PER_SLICE)};
267 Instance().parseContext.SetState(ParserState::Processing);200 Instance().parseContext.SetState(ParserState::Processing);
268 try {201 try {
269 Server::ServerLog::Info(202 Server::ServerLog::Info(
@@ -281,6 +214,7 @@ int ReadProgressInLogFile(std::ifstream &file, std::string &err) {
281 // 读取最新进展214 // 读取最新进展
282 const std::regex progressReg(R"((\d+(?:\.\d+)?)% of entries have been processed)");215 const std::regex progressReg(R"((\d+(?:\.\d+)?)% of entries have been processed)");
283 const std::string parseFailedKeyWord = "Failed to dump the snapshot to database.";216 const std::string parseFailedKeyWord = "Failed to dump the snapshot to database.";
217+ const std::regex cacheReadyReg(R"(Snapshot cache (?:validated|rebuild): ([0-9a-f]{64}))");
284 std::string line;218 std::string line;
285 std::string lastProgressLine;219 std::string lastProgressLine;
286 while (std::getline(file, line)) {220 while (std::getline(file, line)) {
@@ -291,6 +225,10 @@ int ReadProgressInLogFile(std::ifstream &file, std::string &err) {
291 err = line;225 err = line;
292 return -1;226 return -1;
293 }227 }
228+ std::smatch cacheMatch;
229+ if (std::regex_search(line, cacheMatch, cacheReadyReg)) {
230+ MemSnapshotParser::Instance().GetParseContext().SetFileHash(cacheMatch[1].str());
231+ }
294 if (std::regex_search(line, progressReg)) {232 if (std::regex_search(line, progressReg)) {
295 lastProgressLine = line;233 lastProgressLine = line;
296 }234 }
@@ -346,7 +284,8 @@ void MemSnapshotParser::ParseDaemonTask() {
346 const auto manifestPath = MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath());284 const auto manifestPath = MemSnapshotSliceService::GetManifestPath(Instance().parseContext.GetPicklePath());
347 std::uintmax_t manifestSize = 0;285 std::uintmax_t manifestSize = 0;
348 uint64_t currentManifestWriteTime = 0;286 uint64_t currentManifestWriteTime = 0;
349- if (MemSnapshotSliceService::GetFileFingerprint(manifestPath, manifestSize, currentManifestWriteTime)) {287+ if (!Instance().parseContext.GetFileHash().empty() &&
288+ MemSnapshotSliceService::GetFileFingerprint(manifestPath, manifestSize, currentManifestWriteTime)) {
350 const bool manifestChanged =289 const bool manifestChanged =
351 !manifestWriteTime.has_value() || manifestWriteTime.value() != currentManifestWriteTime;290 !manifestWriteTime.has_value() || manifestWriteTime.value() != currentManifestWriteTime;
352 if (manifestChanged) {291 if (manifestChanged) {
@@ -379,12 +318,22 @@ void MemSnapshotParser::ParseDaemonTask() {
379 Instance().parseContext.SetProgress(std::min(newProgress, 99));318 Instance().parseContext.SetProgress(std::min(newProgress, 99));
380 SLEEP(100);319 SLEEP(100);
381 }320 }
382- SendReadySliceEvents(notifiedSlices);
383 if (Instance().parseContext.GetState() == ParserState::FINISH_SUCCESS) {321 if (Instance().parseContext.GetState() == ParserState::FINISH_SUCCESS) {
384 std::ifstream file = OpenReadFileSafely(Instance().parseContext.GetLogPath());322 std::ifstream file = OpenReadFileSafely(Instance().parseContext.GetLogPath());
385- if (DoubleCheckSuccessInLogFile(file) && Instance().TryOpenParsingResultDbAndSetVersion()) {323+ const auto manifest = MemSnapshotSliceService::LoadManifest(Instance().parseContext.GetPicklePath());
386- Instance().parseContext.SetProgress(100);324+ if (DoubleCheckSuccessInLogFile(file) && manifest.has_value() && manifest->IsComplete()) {
387- Server::ServerLog::Info("Parse thread has successfully finished with double check.");325+ Instance().parseContext.SetFileHash(manifest->cacheHash);
326+ SendReadySliceEvents(notifiedSlices);
327+ size_t sliceCount = 0;
328+ for (const auto &[deviceId, device] : manifest->devices) {
329+ sliceCount += device.slices.size();
330+ }
331+ if (notifiedSlices.size() != sliceCount) {
332+ Instance().parseContext.SetState(ParserState::FINISH_FAILURE);
333+ } else {
334+ Instance().parseContext.SetProgress(100);
335+ Server::ServerLog::Info("Parse thread has successfully finished with double check.");
336+ }
388 } else {337 } else {
389 Server::ServerLog::Warn("The parsing thread returned a success response, but we did not find a "338 Server::ServerLog::Warn("The parsing thread returned a success response, but we did not find a "
390 "corresponding success event confirmed in the logs.");339 "corresponding success event confirmed in the logs.");
@@ -470,8 +419,6 @@ void MemSnapshotParser::ParseCallBack() {
470 SendEvent(std::move(event));419 SendEvent(std::move(event));
471}420}
472 421 
473-bool MemSnapshotParser::TryOpenParsingResultDbAndSetVersion() const { return !CheckIfParsingNeed(parseContext); }
474- 
475std::unique_ptr<MemScopeParseSuccessEvent> MemSnapshotParser::BuildParseSuccessEventFromContext(422std::unique_ptr<MemScopeParseSuccessEvent> MemSnapshotParser::BuildParseSuccessEventFromContext(
476 const std::unordered_set<std::string> *preparedSlices) const {423 const std::unordered_set<std::string> *preparedSlices) const {
477 const auto manifest = MemSnapshotSliceService::LoadManifest(parseContext.GetPicklePath());424 const auto manifest = MemSnapshotSliceService::LoadManifest(parseContext.GetPicklePath());
@@ -45,6 +45,7 @@ struct MemSnapshotParserContext {
45 std::string GetLogPath() const;45 std::string GetLogPath() const;
46 std::string GetOutputDbPath() const;46 std::string GetOutputDbPath() const;
47 std::string GetFileHash() const;47 std::string GetFileHash() const;
48+ void SetFileHash(std::string hash);
48 bool IsInitialSuccessSent() const;49 bool IsInitialSuccessSent() const;
49 bool WasInitialSuccessSentWhileBuilding() const;50 bool WasInitialSuccessSentWhileBuilding() const;
50 void MarkInitialSuccessSent(bool parsingComplete);51 void MarkInitialSuccessSent(bool parsingComplete);
@@ -78,9 +79,6 @@ class MemSnapshotParser {
78 // 异步解析Pickle文件API79 // 异步解析Pickle文件API
79 void AsyncParseMemSnapshotPickle(const std::string &pickleFilePath);80 void AsyncParseMemSnapshotPickle(const std::string &pickleFilePath);
80 MemSnapshotParserContext &GetParseContext();81 MemSnapshotParserContext &GetParseContext();
81- static std::string CalculateFileHash(const std::string &filePath);
82- // 用于检查是否需要解析或重新解析pickle文件
83- static bool CheckIfParsingNeed(const MemSnapshotParserContext &context);
84 82 
85 private:83 private:
86 MemSnapshotParser();84 MemSnapshotParser();
@@ -90,8 +88,6 @@ class MemSnapshotParser {
90 // 守护线程,实施读取解析进展88 // 守护线程,实施读取解析进展
91 static void ParseDaemonTask();89 static void ParseDaemonTask();
92 static void ParseCallBack();90 static void ParseCallBack();
93- // 解析完成回调方法之一,用于二次检查db结果并设置解析结果db版本
94- bool TryOpenParsingResultDbAndSetVersion() const;
95 // 构造解析响应事件91 // 构造解析响应事件
96 std::unique_ptr<MemScopeParseSuccessEvent> BuildParseSuccessEventFromContext(92 std::unique_ptr<MemScopeParseSuccessEvent> BuildParseSuccessEventFromContext(
97 const std::unordered_set<std::string> *preparedSlices = nullptr) const;93 const std::unordered_set<std::string> *preparedSlices = nullptr) const;
@@ -15,14 +15,10 @@
15 * ------------------------------------------------------------------------- */15 * ------------------------------------------------------------------------- */
16 16 
17#include <fstream>17#include <fstream>
18-#include <chrono>
19-#include <thread>
20#include <gtest/gtest.h>18#include <gtest/gtest.h>
21#include "FileUtil.h"19#include "FileUtil.h"
22#include "MemSnapshotParser.h"20#include "MemSnapshotParser.h"
23-#include "MemSnapshotDatabase.h"
24#include "MemSnapshotSliceService.h"21#include "MemSnapshotSliceService.h"
codeasier
codeasiercodeasierOct 15

C++ 单测编译失败(实测)

本 PR 删除了 #include "MemSnapshotDatabase.h",而 MemSnapshotParserTest.cpp:36 的 MemSnapshotSliceService::GetArtifactDirectory(...) 原先是靠这个头文件间接引入 Dic::Module::MemSnapshot 命名空间的。删掉以后,本地按 master 的构建方式编译 insight_test 报 'MemSnapshotSliceService' has not been declared。在临时副本里加上 using namespace Dic::Module::MemSnapshot; 后可以编译通过,用例也能跑。 流水线 #2347(commit 97cb9726)的 UT_msinsight 显示 FAILED,与此一致(推断,没有看到 UT 日志)。

建议:在本文件加 using namespace Dic::Module::MemSnapshot;,或者把调用写成全限定名 MemSnapshot::MemSnapshotSliceService::...。

likedislike
heiheihei
heiheihei
Oct 15 Comments:
likedislike
25-#include "sqlite3.h"
26 22 
27using namespace Dic::Module;23using namespace Dic::Module;
28using namespace Dic;24using namespace Dic;
@@ -37,7 +33,7 @@ class MemSnapshotParserTest : public ::testing::Test {
37 static void SetUpTestSuite() {33 static void SetUpTestSuite() {
38 testPicklePath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.pkl");34 testPicklePath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.pkl");
39 testLogPath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.log");35 testLogPath = FileUtil::SplicePath(::testing::TempDir(), "mem_snapshot_parser_test.log");
40- testOutputDbPath = MemSnapshotSliceService::GetArtifactDirectory(testPicklePath);36+ testOutputDbPath = Dic::Module::MemSnapshot::MemSnapshotSliceService::GetArtifactDirectory(testPicklePath);
41 37 
42 std::ofstream pickleFile(testPicklePath);38 std::ofstream pickleFile(testPicklePath);
43 pickleFile.close();39 pickleFile.close();
@@ -105,6 +101,23 @@ TEST_F(MemSnapshotParserTest, ConfirmsSuccessKeywordInLog) {
105 FileUtil::RemoveFile(testLogPath);101 FileUtil::RemoveFile(testLogPath);
106}102}
107 103 
104+TEST_F(MemSnapshotParserTest, ReadsScriptCacheIdentityBeforePublishingSlices) {
105+ const std::string hash(64, 'a');
106+ for (const auto &action : {"validated", "rebuild"}) {
107+ parser->GetParseContext().SetFileHash("");
108+ {
109+ std::ofstream output(testLogPath, std::ios::trunc);
110+ output << "Snapshot cache " << action << ": " << hash << "\n";
111+ output << "1% of entries have been processed\n";
112+ }
113+ std::ifstream log(testLogPath);
114+ std::string error;
115+ EXPECT_EQ(ReadProgressInLogFile(log, error), 1);
116+ EXPECT_EQ(parser->GetParseContext().GetFileHash(), hash);
117+ }
118+ FileUtil::RemoveFile(testLogPath);
119+}
120+ 
108// 测试解析器重置功能121// 测试解析器重置功能
109TEST_F(MemSnapshotParserTest, Reset) {122TEST_F(MemSnapshotParserTest, Reset) {
110 // 先设置一些状态123 // 先设置一些状态
@@ -168,112 +181,11 @@ TEST_F(MemSnapshotParserTest, ParseContextManagement) {
168 EXPECT_TRUE(parser->GetParseContext().IsFinished());181 EXPECT_TRUE(parser->GetParseContext().IsFinished());
169}182}
170 183 
171-// 测试是否需要解析的检查逻辑184+TEST_F(MemSnapshotParserTest, FileHashIsPublishedByParserScript) {
172-TEST_F(MemSnapshotParserTest, CheckIfParsingNeed) {185+ parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath);
173- // 重置解析器186+ EXPECT_TRUE(parser->GetParseContext().GetFileHash().empty());
174- parser->Reset();187+ parser->GetParseContext().SetFileHash("script-generated-hash");
175- 188+ EXPECT_EQ(parser->GetParseContext().GetFileHash(), "script-generated-hash");
176- // 设置测试上下文189+ parser->GetParseContext().Reset();
177- const std::string fileHash = parser->CalculateFileHash(testPicklePath);190+ EXPECT_TRUE(parser->GetParseContext().GetFileHash().empty());
178- parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash);
179- 
180- // 由于输出数据库文件不存在,应该需要解析
181- bool needParse = parser->CheckIfParsingNeed(parser->GetParseContext());
182- EXPECT_TRUE(needParse);
183- 
184- const auto deviceDir = fs::path(testOutputDbPath) / "device_0";
185- fs::create_directories(deviceDir);
186- const auto sliceDbPath = deviceDir / "slice_00000.db";
187- sqlite3 *sliceDb = nullptr;
188- ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK);
189- ASSERT_EQ(sqlite3_exec(sliceDb,
190- "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT);"
191- "CREATE TABLE block_0 (`id` INTEGER PRIMARY KEY, `address` INTEGER, `size` INTEGER, "
192- "`requestedSize` INTEGER, `state` INTEGER DEFAULT 99, `allocEventId` INTEGER, `freeEventId` INTEGER);"
193- "CREATE TABLE trace_entry_0 (`id` INTEGER PRIMARY KEY, `action` INTEGER, `address` INTEGER, `size` "
194- "INTEGER, `stream` INTEGER, `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, `callstack` "
195- "TEXT);"
196- "INSERT INTO trace_entry_0 VALUES (0, 0, 0, 0, 0, 10, 10, 20, '');",
197- nullptr, nullptr, nullptr),
198- SQLITE_OK);
199- sqlite3_close(sliceDb);
200- ASSERT_TRUE(Dic::Module::FullDb::MemSnapshotDatabase::BuildMemoryAllocationCache(sliceDbPath.string(), "0"));
201- std::ofstream manifest(MemSnapshotSliceService::GetManifestPath(testPicklePath), std::ios::trunc);
202- manifest << R"({"schemaVersion":1,"status":"complete","sourceFile":"snapshot_invalid.pkl","cacheHash":")"
203- << fileHash
204- << R"(","eventsPerSlice":100,"devices":{"0":{"eventCount":1,"sliceCount":1,)"
205- R"("readySlices":[0],"slices":[{"index":0,"startEventId":0,"endEventId":0,)"
206- R"("file":"device_0/slice_00000.db","ready":true}]}}})";
207- manifest.close();
208- 
209- EXPECT_FALSE(parser->CheckIfParsingNeed(parser->GetParseContext()));
210- 
211- // Exercise the asynchronous cache-hit path without launching a pickle subprocess.
212- parser->AsyncParseMemSnapshotPickle(testPicklePath);
213- for (int attempt = 0; attempt < 100 && parser->GetParseContext().GetState() != ParserState::UP_TO_DATE; ++attempt) {
214- std::this_thread::sleep_for(std::chrono::milliseconds(50));
215- }
216- EXPECT_EQ(parser->GetParseContext().GetState(), ParserState::UP_TO_DATE);
217- EXPECT_EQ(parser->GetParseContext().GetProgress(), 100);
218- parser->Reset();
219- parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash);
220- 
221- ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK);
222- ASSERT_EQ(sqlite3_exec(sliceDb, "DROP TABLE memory_allocation_cache_v1_0", nullptr, nullptr, nullptr), SQLITE_OK);
223- sqlite3_close(sliceDb);
224- // 缺 cache 时可从源表现场重建,无需重新解析 pickle
225- EXPECT_FALSE(parser->CheckIfParsingNeed(parser->GetParseContext()));
226- ASSERT_TRUE(Dic::Module::FullDb::MemSnapshotDatabase::HasMemoryAllocationCache(sliceDbPath.string(), "0"));
227- 
228- ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK);
229- ASSERT_EQ(sqlite3_exec(sliceDb, "DROP TABLE block_0", nullptr, nullptr, nullptr), SQLITE_OK);
230- sqlite3_close(sliceDb);
231- EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext()));
232- ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK);
233- ASSERT_EQ(
234- sqlite3_exec(sliceDb, "CREATE TABLE block_0 (id INTEGER PRIMARY KEY)", nullptr, nullptr, nullptr), SQLITE_OK);
235- sqlite3_close(sliceDb);
236- 
237- parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, "changed-hash");
238- EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext()));
239- 
240- parser->GetParseContext().Reset(testPicklePath, testLogPath, testOutputDbPath, fileHash);
241- fs::remove(sliceDbPath);
242- EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext()));
243- 
244- ASSERT_EQ(sqlite3_open(sliceDbPath.string().c_str(), &sliceDb), SQLITE_OK);
245- sqlite3_close(sliceDb);
246- std::ofstream invalidManifest(MemSnapshotSliceService::GetManifestPath(testPicklePath), std::ios::trunc);
247- invalidManifest << R"({"schemaVersion":1,"status":"complete"})";
248- invalidManifest.close();
249- EXPECT_TRUE(parser->CheckIfParsingNeed(parser->GetParseContext()));
250- 
251- fs::remove_all(testOutputDbPath);
252-}
253- 
254-TEST_F(MemSnapshotParserTest, CalculateFileHashTracksFileContent) {
255- {
256- std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::trunc);
257- pickleFile << "snapshot-content";
258- }
259- const std::string firstHash = parser->CalculateFileHash(testPicklePath);
260- const std::string secondHash = parser->CalculateFileHash(testPicklePath);
261- EXPECT_FALSE(firstHash.empty());
262- EXPECT_EQ(firstHash, secondHash);
263- 
264- {
265- std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::app);
266- pickleFile << "-changed";
267- }
268- EXPECT_NE(firstHash, parser->CalculateFileHash(testPicklePath));
269-}
270- 
271-TEST_F(MemSnapshotParserTest, CalculateFileHashIncludesParserSalt) {
272- {
273- std::ofstream pickleFile(testPicklePath, std::ios::binary | std::ios::trunc);
274- pickleFile << "snapshot-content";
275- }
276- // This value changes when MEM_SNAPSHOT_PARSER_HASH_SALT changes.
277- EXPECT_EQ(
278- "adaad4452d193309891d52c7803603f11ef9a73aae9d8fce35df9b22891cb1be", parser->CalculateFileHash(testPicklePath));
279}191}