已合并
基于HMDFS的批量小文件共享优化 #415
基于HMDFS的批量小文件共享优化 #415
已合并
zengjy创建于 13 天前
18 个文件变更+785-97
@@ -938,6 +938,37 @@ static struct sbi_attribute sbi_write_cache_timeout_sec_attr =
938 __ATTR(write_cache_timeout_sec, 0664, sbi_write_cache_timeout_sec_show,938 __ATTR(write_cache_timeout_sec, 0664, sbi_write_cache_timeout_sec_show,
939 sbi_write_cache_timeout_sec_store);939 sbi_write_cache_timeout_sec_store);
940 940 
941+static ssize_t sbi_small_file_optimization_show(struct kobject *kobj,
942+ struct sbi_attribute *attr, char *buf)
943+{
944+ const struct hmdfs_sb_info *sbi = to_sbi(kobj);
945+ 
946+ return snprintf(buf, PAGE_SIZE, "%d\n",
947+ READ_ONCE(sbi->small_file_optimization));
948+}
949+ 
950+static ssize_t sbi_small_file_optimization_store(struct kobject *kobj,
951+ struct sbi_attribute *attr,
952+ const char *buf, size_t len)
953+{
954+ struct hmdfs_sb_info *sbi = to_sbi(kobj);
955+ bool small_file_optimization;
956+ int err;
957+ 
958+ err = kstrtobool(buf, &small_file_optimization);
959+ if (err)
960+ return err;
961+ if (small_file_optimization && !HMDFS_SMALL_FILE_OPT_SUPPORTED)
962+ return -EOPNOTSUPP;
963+ 
964+ WRITE_ONCE(sbi->small_file_optimization, small_file_optimization);
965+ return len;
966+}
967+ 
968+static struct sbi_attribute sbi_small_file_optimization_attr =
969+ __ATTR(small_file_optimization, 0644, sbi_small_file_optimization_show,
lijiawei
lijiaweilijiawei12 天前

[次要] [fs/hmdfs/comm/device_node.c:967] sbi_small_file_optimization_attr 使用 0644 权限,允许所有用户读取但仅 root 写入。而同文件中 sbi_write_cache_timeout_sec_attr 使用 0664 权限。建议确认此权限差异是否符合预期,保持一致性。

likedislike
zengjy
4 天前 评论:
970+ sbi_small_file_optimization_store);
971+ 
941static ssize_t sbi_node_evt_cb_delay_show(struct kobject *kobj,972static ssize_t sbi_node_evt_cb_delay_show(struct kobject *kobj,
942 struct sbi_attribute *attr,973 struct sbi_attribute *attr,
943 char *buf)974 char *buf)
@@ -1213,6 +1244,7 @@ static struct attribute *sbi_attrs[] = {
1213 &sbi_dcache_threshold_attr.attr,1244 &sbi_dcache_threshold_attr.attr,
1214 &sbi_dcache_timeout_attr.attr,1245 &sbi_dcache_timeout_attr.attr,
1215 &sbi_write_cache_timeout_sec_attr.attr,1246 &sbi_write_cache_timeout_sec_attr.attr,
1247+ &sbi_small_file_optimization_attr.attr,
1216 &sbi_local_op_attr.attr,1248 &sbi_local_op_attr.attr,
1217 &sbi_delay_resp_attr.attr,1249 &sbi_delay_resp_attr.attr,
1218 &sbi_wb_timeout_ms_attr.attr,1250 &sbi_wb_timeout_ms_attr.attr,
@@ -1335,6 +1367,7 @@ HMDFS_CMD_ATTR(rename, F_RENAME);
1335HMDFS_CMD_ATTR(setattr, F_SETATTR);1367HMDFS_CMD_ATTR(setattr, F_SETATTR);
1336HMDFS_CMD_ATTR(statfs, F_STATFS);1368HMDFS_CMD_ATTR(statfs, F_STATFS);
1337HMDFS_CMD_ATTR(drop_push, F_DROP_PUSH);1369HMDFS_CMD_ATTR(drop_push, F_DROP_PUSH);
1370+HMDFS_CMD_ATTR(drop_page_push, F_DROP_PAGE_PUSH);
1338HMDFS_CMD_ATTR(getattr, F_GETATTR);1371HMDFS_CMD_ATTR(getattr, F_GETATTR);
1339HMDFS_CMD_ATTR(fsync, F_FSYNC);1372HMDFS_CMD_ATTR(fsync, F_FSYNC);
1340HMDFS_CMD_ATTR(syncfs, F_SYNCFS);1373HMDFS_CMD_ATTR(syncfs, F_SYNCFS);
@@ -1354,6 +1387,7 @@ static struct attribute *sbi_timeout_attrs[] = {
1354 ATTR_LIST(getattr), ATTR_LIST(fsync),1387 ATTR_LIST(getattr), ATTR_LIST(fsync),
1355 ATTR_LIST(syncfs), ATTR_LIST(getxattr),1388 ATTR_LIST(syncfs), ATTR_LIST(getxattr),
1356 ATTR_LIST(setxattr), ATTR_LIST(listxattr),1389 ATTR_LIST(setxattr), ATTR_LIST(listxattr),
1390+ ATTR_LIST(drop_page_push),
1357 NULL1391 NULL
1358};1392};
1359ATTRIBUTE_GROUPS(sbi_timeout);1393ATTRIBUTE_GROUPS(sbi_timeout);
@@ -27,6 +27,7 @@ void hmdfs_message_verify_init(void)
27 need_response[F_RELEASE] = false;27 need_response[F_RELEASE] = false;
28 need_response[F_CONNECT_REKEY] = false;28 need_response[F_CONNECT_REKEY] = false;
29 need_response[F_DROP_PUSH] = false;29 need_response[F_DROP_PUSH] = false;
30+ need_response[F_DROP_PAGE_PUSH] = false;
30 31 
31 for (flag = 0; flag < C_FLAG_SIZE; flag++) {32 for (flag = 0; flag < C_FLAG_SIZE; flag++) {
32 for (cmd = 0; cmd < F_SIZE; cmd++) {33 for (cmd = 0; cmd < F_SIZE; cmd++) {
@@ -43,11 +44,12 @@ void hmdfs_message_verify_init(void)
43 sizeof(struct open_request) + PATH_MAX + 1;44 sizeof(struct open_request) + PATH_MAX + 1;
44 message_length[C_REQUEST][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =45 message_length[C_REQUEST][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
45 MESSAGE_LEN_JUDGE_RANGE;46 MESSAGE_LEN_JUDGE_RANGE;
46- message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MIN_INDEX] = 0;47+ message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MIN_INDEX] =
47- message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MAX_INDEX] =
48 sizeof(struct open_response);48 sizeof(struct open_response);
49+ message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MAX_INDEX] =
50+ HMDFS_OPEN_PREFETCH_RESPONSE_SIZE;
49 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =51 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
50- MESSAGE_LEN_JUDGE_BIN;52+ MESSAGE_LEN_JUDGE_RANGE;
51 53 
52 message_length[C_REQUEST][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] =54 message_length[C_REQUEST][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] =
53 sizeof(struct atomic_open_request);55 sizeof(struct atomic_open_request);
@@ -57,9 +59,9 @@ void hmdfs_message_verify_init(void)
57 = MESSAGE_LEN_JUDGE_RANGE;59 = MESSAGE_LEN_JUDGE_RANGE;
58 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] = 0;60 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] = 0;
59 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_MAX_INDEX] =61 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_MAX_INDEX] =
60- sizeof(struct atomic_open_response);62+ HMDFS_ATOMIC_OPEN_PREFETCH_RESPONSE_SIZE;
61 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX]63 message_length[C_RESPONSE][F_ATOMIC_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX]
62- = MESSAGE_LEN_JUDGE_BIN;64+ = MESSAGE_LEN_JUDGE_RANGE;
63 65 
64 message_length[C_REQUEST][F_RELEASE][HMDFS_MESSAGE_MIN_INDEX] =66 message_length[C_REQUEST][F_RELEASE][HMDFS_MESSAGE_MIN_INDEX] =
65 sizeof(struct release_request);67 sizeof(struct release_request);
@@ -269,6 +271,13 @@ void hmdfs_message_verify_init(void)
269 sizeof(struct drop_push_request) + PATH_MAX + 1;271 sizeof(struct drop_push_request) + PATH_MAX + 1;
270 message_length[C_REQUEST][F_DROP_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =272 message_length[C_REQUEST][F_DROP_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
271 MESSAGE_LEN_JUDGE_RANGE;273 MESSAGE_LEN_JUDGE_RANGE;
274+
275+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_MIN_INDEX] =
276+ sizeof(struct drop_page_push_request);
277+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_MAX_INDEX] =
278+ sizeof(struct drop_page_push_request) + PATH_MAX + 1;
279+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
280+ MESSAGE_LEN_JUDGE_RANGE;
272}281}
273 282 
274static int is_str_msg_valid(char *msg, int str_len[], size_t str_num)283static int is_str_msg_valid(char *msg, int str_len[], size_t str_num)
@@ -306,9 +315,8 @@ static int verify_open_req(size_t msg_len, void *msg)
306 315 
307static int verify_open_resp(size_t msg_len, void *msg)316static int verify_open_resp(size_t msg_len, void *msg)
308{317{
309- struct open_response *resp = msg;318+ if (msg_len != sizeof(struct open_response) &&
310- 319+ msg_len != HMDFS_OPEN_PREFETCH_RESPONSE_SIZE)
311- if (msg_len != sizeof(*resp))
312 return -EINVAL;320 return -EINVAL;
313 321 
314 return 0;322 return 0;
@@ -345,9 +353,8 @@ static int verify_atomic_open_req(size_t msg_len, void *msg)
345 353 
346static int verify_atomic_open_resp(size_t msg_len, void *msg)354static int verify_atomic_open_resp(size_t msg_len, void *msg)
347{355{
348- struct atomic_open_response *resp = msg;356+ if (msg_len != sizeof(struct atomic_open_response) &&
349- 357+ msg_len != HMDFS_ATOMIC_OPEN_PREFETCH_RESPONSE_SIZE)
350- if (msg_len != sizeof(*resp))
351 return -EINVAL;358 return -EINVAL;
352 359 
353 return 0;360 return 0;
@@ -856,6 +863,34 @@ static int hmdfs_drop_push_verify(int flag, size_t msg_len, void *msg)
856 return 0;863 return 0;
857}864}
858 865 
866+static int verify_drop_page_push_req(size_t msg_len, void *msg)
867+{
868+ struct drop_page_push_request *req = msg;
869+ int str_len[] = {req->path_len};
870+
871+ if (req->path_len < 0 || req->path_len >= PATH_MAX)
872+ return -EINVAL;
873+
874+ if (msg_len != sizeof(*req) + req->path_len + 1)
875+ return -EINVAL;
876+
877+ if (is_str_msg_valid(req->buf, str_len, sizeof(str_len) / sizeof(int)))
878+ return -EINVAL;
879+
880+ return 0;
881+}
882+
883+static int hmdfs_drop_page_push_verify(int flag, size_t msg_len, void *msg)
884+{
885+ if (!msg || !msg_len)
886+ return 0;
887+
888+ if (flag == C_REQUEST)
889+ return verify_drop_page_push_req(msg_len, msg);
890+
891+ return 0;
892+}
893+ 
859typedef int (*hmdfs_message_verify_func)(int, size_t, void *);894typedef int (*hmdfs_message_verify_func)(int, size_t, void *);
860 895 
861static const hmdfs_message_verify_func message_verify[F_SIZE] = {896static const hmdfs_message_verify_func message_verify[F_SIZE] = {
@@ -871,6 +906,7 @@ static const hmdfs_message_verify_func message_verify[F_SIZE] = {
871 [F_SETATTR] = hmdfs_setattr_verify,906 [F_SETATTR] = hmdfs_setattr_verify,
872 [F_STATFS] = hmdfs_statfs_verify,907 [F_STATFS] = hmdfs_statfs_verify,
873 [F_DROP_PUSH] = hmdfs_drop_push_verify,908 [F_DROP_PUSH] = hmdfs_drop_push_verify,
909+ [F_DROP_PAGE_PUSH] = hmdfs_drop_page_push_verify,
874 [F_GETATTR] = hmdfs_getattr_verify,910 [F_GETATTR] = hmdfs_getattr_verify,
875 [F_GETXATTR] = hmdfs_getxattr_verify,911 [F_GETXATTR] = hmdfs_getxattr_verify,
876 [F_SETXATTR] = hmdfs_setxattr_verify,912 [F_SETXATTR] = hmdfs_setxattr_verify,
@@ -12,6 +12,10 @@
12#include <linux/wait.h>12#include <linux/wait.h>
13#include <linux/workqueue.h>13#include <linux/workqueue.h>
14#include <linux/namei.h>14#include <linux/namei.h>
15+#include <linux/bits.h>
16+#include <linux/stddef.h>
17+ 
18+#define HMDFS_PREFETCH_FLAG ((__u8)BIT(0))
15 19 
16struct hmdfs_cmd {20struct hmdfs_cmd {
17 __u8 reserved;21 __u8 reserved;
@@ -20,9 +24,21 @@ struct hmdfs_cmd {
20 __u8 reserved2;24 __u8 reserved2;
21} __packed;25} __packed;
22 26 
27+static inline bool hm_isprefetch(__u8 flag)
28+{
29+ return (flag & HMDFS_PREFETCH_FLAG) == HMDFS_PREFETCH_FLAG;
30+}
31+ 
32+static inline void hm_setprefetch(__u8 *flag)
33+{
34+ *flag |= HMDFS_PREFETCH_FLAG;
35+}
36+ 
23#define HMDFS_MSG_MAGIC 0xF737#define HMDFS_MSG_MAGIC 0xF7
24#define HMDFS_MAX_MESSAGE_LEN (8 * 1024 * 1024)38#define HMDFS_MAX_MESSAGE_LEN (8 * 1024 * 1024)
25 39 
40+#define HMDFS_PAGE_SIZE 4096
41+ 
26struct hmdfs_head_cmd {42struct hmdfs_head_cmd {
27 __u8 magic;43 __u8 magic;
28 __u8 version;44 __u8 version;
@@ -181,6 +197,7 @@ enum FILE_CMD {
181 F_RESERVED_7 = 26,197 F_RESERVED_7 = 26,
182 F_RESERVED_8 = 27,198 F_RESERVED_8 = 27,
183 F_ATOMIC_OPEN = 28,199 F_ATOMIC_OPEN = 28,
200+ F_DROP_PAGE_PUSH = 29,
184 F_SIZE,201 F_SIZE,
185};202};
186 203 
@@ -206,6 +223,22 @@ struct open_response {
206 __le64 ichange_count;223 __le64 ichange_count;
207} __packed;224} __packed;
208 225 
226+struct open_prefetch_response {
227+ struct open_response open_resp;
228+ __u8 read_success;
229+ __u8 page[];
230+} __packed;
231+ 
232+#define HMDFS_OPEN_PREFETCH_RESPONSE_SIZE \
233+ (sizeof(struct open_prefetch_response) + HMDFS_PAGE_SIZE)
234+ 
235+static_assert(offsetof(struct open_prefetch_response, read_success) ==
236+ sizeof(struct open_response),
237+ "prefetch data must follow open response");
238+static_assert(offsetof(struct open_prefetch_response, page) ==
239+ sizeof(struct open_response) + sizeof(__u8),
240+ "prefetch page must follow read status");
241+ 
209enum hmdfs_open_flags {242enum hmdfs_open_flags {
210 HMDFS_O_TRUNC = O_TRUNC,243 HMDFS_O_TRUNC = O_TRUNC,
211 HMDFS_O_EXCL = O_EXCL,244 HMDFS_O_EXCL = O_EXCL,
@@ -231,6 +264,22 @@ struct atomic_open_response {
231 struct open_response open_resp;264 struct open_response open_resp;
232} __packed;265} __packed;
233 266 
267+struct atomic_open_prefetch_response {
268+ struct atomic_open_response atomic_open_resp;
269+ __u8 read_success;
270+ __u8 page[];
271+} __packed;
272+ 
273+#define HMDFS_ATOMIC_OPEN_PREFETCH_RESPONSE_SIZE \
274+ (sizeof(struct atomic_open_prefetch_response) + HMDFS_PAGE_SIZE)
275+ 
276+static_assert(offsetof(struct atomic_open_prefetch_response, read_success) ==
277+ sizeof(struct atomic_open_response),
278+ "prefetch data must follow atomic open response");
279+static_assert(offsetof(struct atomic_open_prefetch_response, page) ==
280+ sizeof(struct atomic_open_response) + sizeof(__u8),
281+ "prefetch page must follow read status");
282+ 
234struct release_request {283struct release_request {
235 __le64 file_ver;284 __le64 file_ver;
236 __le32 file_id;285 __le32 file_id;
@@ -332,6 +381,18 @@ struct drop_push_request {
332 char path[0];381 char path[0];
333} __packed;382} __packed;
334 383 
384+struct drop_page_push_request {
385+ __le64 file_size;
386+ __le64 ctime;
387+ __le32 ctime_nsec;
388+ __le64 mtime;
389+ __le32 mtime_nsec;
390+ __le64 stable_ctime;
391+ __le32 stable_ctime_nsec;
392+ __le32 path_len;
393+ char buf[0];
lijiawei
lijiaweilijiawei12 天前

[次要] [fs/hmdfs/comm/protocol.h:362] drop_page_push_request 中使用 char buf[0] 作为柔性数组成员,C99 标准中 buf[0] 不是标准柔性数组,建议使用 char buf[] 替代,与内核编码惯例一致。

likedislike
zengjy
4 天前 评论:
394+} __packed;
395+ 
335struct setattr_request {396struct setattr_request {
336 __le64 size;397 __le64 size;
337 __le32 valid;398 __le32 valid;
@@ -43,6 +43,7 @@ static const request_callback s_recv_callbacks[F_SIZE] = {
43 [F_SETATTR] = hmdfs_server_setattr,43 [F_SETATTR] = hmdfs_server_setattr,
44 [F_STATFS] = hmdfs_server_statfs,44 [F_STATFS] = hmdfs_server_statfs,
45 [F_DROP_PUSH] = hmdfs_server_get_drop_push,45 [F_DROP_PUSH] = hmdfs_server_get_drop_push,
46+ [F_DROP_PAGE_PUSH] = hmdfs_server_get_drop_page_push,
46 [F_GETATTR] = hmdfs_server_getattr,47 [F_GETATTR] = hmdfs_server_getattr,
47 [F_FSYNC] = hmdfs_server_fsync,48 [F_FSYNC] = hmdfs_server_fsync,
48 [F_SYNCFS] = hmdfs_server_syncfs,49 [F_SYNCFS] = hmdfs_server_syncfs,
@@ -835,6 +836,7 @@ static int hmdfs_request_recv(struct hmdfs_peer *con,
835 case F_STATFS:836 case F_STATFS:
836 case F_CONNECT_REKEY:837 case F_CONNECT_REKEY:
837 case F_DROP_PUSH:838 case F_DROP_PUSH:
839+ case F_DROP_PAGE_PUSH:
838 case F_GETATTR:840 case F_GETATTR:
839 case F_FSYNC:841 case F_FSYNC:
840 case F_SYNCFS:842 case F_SYNCFS:
@@ -1045,6 +1047,7 @@ static int hmdfs_response_recv(struct hmdfs_peer *con,
1045 case F_STATFS:1047 case F_STATFS:
1046 case F_CONNECT_REKEY:1048 case F_CONNECT_REKEY:
1047 case F_DROP_PUSH:1049 case F_DROP_PUSH:
1050+ case F_DROP_PAGE_PUSH:
1048 case F_GETATTR:1051 case F_GETATTR:
1049 case F_FSYNC:1052 case F_FSYNC:
1050 case F_SYNCFS:1053 case F_SYNCFS:
@@ -176,7 +176,7 @@ static void hmdfs_dev_d_release(struct dentry *dentry)
176 switch (hmdfs_d(dentry)->dentry_type) {176 switch (hmdfs_d(dentry)->dentry_type) {
177 case HMDFS_LAYER_SECOND_LOCAL:177 case HMDFS_LAYER_SECOND_LOCAL:
178 hmdfs_clear_cache_dents(dentry, false);178 hmdfs_clear_cache_dents(dentry, false);
179- hmdfs_drop_remote_cache_dents(dentry);179+ hmdfs_release_remote_cache(dentry);
180 path_put(&(hmdfs_d(dentry)->lower_path));180 path_put(&(hmdfs_d(dentry)->lower_path));
181 break;181 break;
182 case HMDFS_LAYER_ZERO:182 case HMDFS_LAYER_ZERO:
@@ -214,7 +214,7 @@ static void hmdfs_d_release(struct dentry *dentry)
214 return;214 return;
215 215 
216 hmdfs_clear_cache_dents(dentry, false);216 hmdfs_clear_cache_dents(dentry, false);
217- hmdfs_drop_remote_cache_dents(dentry);217+ hmdfs_release_remote_cache(dentry);
218 hmdfs_put_reset_lower_path(dentry);218 hmdfs_put_reset_lower_path(dentry);
219 kmem_cache_free(hmdfs_dentry_cachep, dentry->d_fsdata);219 kmem_cache_free(hmdfs_dentry_cachep, dentry->d_fsdata);
220 dentry->d_fsdata = NULL;220 dentry->d_fsdata = NULL;
@@ -61,9 +61,23 @@ int hmdfs_file_release_local(struct inode *inode, struct file *file)
61{61{
62 struct hmdfs_file_info *gfi = hmdfs_f(file);62 struct hmdfs_file_info *gfi = hmdfs_f(file);
63 struct hmdfs_inode_info *info = hmdfs_i(inode);63 struct hmdfs_inode_info *info = hmdfs_i(inode);
64+ bool modified;
64 65 
65 if (file->f_flags & (O_RDWR | O_WRONLY))66 if (file->f_flags & (O_RDWR | O_WRONLY))
66 atomic_dec(&info->write_opened);67 atomic_dec(&info->write_opened);
68+ 
69+ /*
70+ * Clear the current state before sending the drop message. A concurrent
71+ * write will set modified again and be handled by a later release.
72+ */
73+ spin_lock(&info->modify_lock);
74+ modified = info->modified;
75+ info->modified = false;
76+ spin_unlock(&info->modify_lock);
77+ 
78+ if (modified)
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/file_local.c:74] hmdfs_file_release_local 中将 hmdfs_drop_remote_cache_pages 的调用从 spinlock 内移到 lock 外,这是正确的改进(hmdfs_drop_remote_cache_pages 内部会调用 hmdfs_send_drop_page_push,可能休眠)。但 modified 变量在 spinlock 内读取并清零,而 hmdfs_drop_remote_cache_pages 在 lock 外执行,如果此时另一个线程设置了 info->modified,该次修改不会被 drop。建议:确认这是可接受的(因为 release 路径上不会再有新的写入),或在注释中说明此设计决策。

likedislike
79+ hmdfs_drop_remote_cache_pages(file_dentry(file));
80+ 
67 file->private_data = NULL;81 file->private_data = NULL;
68 fput(gfi->lower_file);82 fput(gfi->lower_file);
69 kfree(gfi);83 kfree(gfi);
@@ -112,6 +126,7 @@ static ssize_t hmdfs_local_read_iter(struct kiocb *iocb, struct iov_iter *iter)
112static void hmdfs_file_modified(struct file *file)126static void hmdfs_file_modified(struct file *file)
113{127{
114 struct inode *inode = file_inode(file);128 struct inode *inode = file_inode(file);
129+ struct hmdfs_inode_info *info = hmdfs_i(inode);
115 struct dentry *dentry = file_dentry(file);130 struct dentry *dentry = file_dentry(file);
116 struct file *lower_file = hmdfs_f(file)->lower_file;131 struct file *lower_file = hmdfs_f(file)->lower_file;
117 struct inode *lower_inode = file_inode(lower_file);132 struct inode *lower_inode = file_inode(lower_file);
@@ -123,9 +138,13 @@ static void hmdfs_file_modified(struct file *file)
123 138 
124 if (!hmdfs_i_merge(hmdfs_i(inode)))139 if (!hmdfs_i_merge(hmdfs_i(inode)))
125 update_inode_to_dentry(dentry, inode);140 update_inode_to_dentry(dentry, inode);
141+
142+ spin_lock(&info->modify_lock);
143+ info->modified = true;
144+ spin_unlock(&info->modify_lock);
126}145}
127 146 
128-ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,147+ssize_t hmdfs_do_write_iter_local(struct file *file, struct iov_iter *iter,
129 loff_t *ppos)148 loff_t *ppos)
130{149{
131 ssize_t ret;150 ssize_t ret;
@@ -154,7 +173,7 @@ ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,
154 173 
155ssize_t hmdfs_local_write_iter(struct kiocb *iocb, struct iov_iter *iter)174ssize_t hmdfs_local_write_iter(struct kiocb *iocb, struct iov_iter *iter)
156{175{
157- return hmdfs_do_write_iter(iocb->ki_filp, iter, &iocb->ki_pos);176+ return hmdfs_do_write_iter_local(iocb->ki_filp, iter, &iocb->ki_pos);
158}177}
159 178 
160int hmdfs_fsync_local(struct file *file, loff_t start, loff_t end, int datasync)179int hmdfs_fsync_local(struct file *file, loff_t start, loff_t end, int datasync)
@@ -537,9 +537,52 @@ static ssize_t hmdfs_merge_read_iter(struct kiocb *iocb, struct iov_iter *iter)
537 return hmdfs_do_read_iter(iocb->ki_filp, iter, &iocb->ki_pos);537 return hmdfs_do_read_iter(iocb->ki_filp, iter, &iocb->ki_pos);
538}538}
539 539 
540+static void hmdfs_file_modified(struct file *file)
541+{
542+ struct inode *inode = file_inode(file);
543+ struct dentry *dentry = file_dentry(file);
544+ struct file *lower_file = hmdfs_f(file)->lower_file;
545+ struct inode *lower_inode = file_inode(lower_file);
546+ 
547+ inode->i_atime = lower_inode->i_atime;
548+ inode->__i_ctime = lower_inode->__i_ctime;
549+ inode->i_mtime = lower_inode->i_mtime;
550+ i_size_write(inode, i_size_read(lower_inode));
551+ 
552+ if (!hmdfs_i_merge(hmdfs_i(inode)))
553+ update_inode_to_dentry(dentry, inode);
554+}
555+ 
556+ssize_t hmdfs_do_write_iter_merge(struct file *file, struct iov_iter *iter,
lijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/file_merge.c:556] hmdfs_do_write_iter_merge 函数中 hmdfs_file_modified 更新 inode 属性时,未像 file_local.c 中那样设置 info->modified = true 和调用 hmdfs_drop_remote_cache_pages。而 hmdfs_file_release_merge 也未包含 modified 检查和缓存推送逻辑。这意味着 merge 模式下的写操作不会触发远程页缓存失效,可能导致其他设备看到过期数据。需确认 merge 模式是否也需要推送页缓存失效通知。

likedislike
zengjy
4 天前 评论:
557+ loff_t *ppos)
558+{
559+ ssize_t ret;
560+ struct file *lower_file = hmdfs_f(file)->lower_file;
561+ struct inode *inode = file_inode(file);
562+ 
563+ if (!iov_iter_count(iter))
564+ return 0;
565+ 
566+ inode_lock(inode);
567+ 
568+ ret = file_remove_privs(file);
569+ if (ret)
570+ goto out_unlock;
571+ 
572+ file_start_write(lower_file);
573+ ret = vfs_iter_write(lower_file, iter, ppos, 0);
574+ file_end_write(lower_file);
575+ 
576+ hmdfs_file_modified(file);
577+ 
578+out_unlock:
579+ inode_unlock(inode);
580+ return ret;
581+}
582+ 
540ssize_t hmdfs_merge_write_iter(struct kiocb *iocb, struct iov_iter *iter)583ssize_t hmdfs_merge_write_iter(struct kiocb *iocb, struct iov_iter *iter)
541{584{
542- return hmdfs_do_write_iter(iocb->ki_filp, iter, &iocb->ki_pos);585+ return hmdfs_do_write_iter_merge(iocb->ki_filp, iter, &iocb->ki_pos);
543}586}
544 587 
545int hmdfs_file_open_merge(struct inode *inode, struct file *file)588int hmdfs_file_open_merge(struct inode *inode, struct file *file)
@@ -580,6 +623,19 @@ int hmdfs_file_open_merge(struct inode *inode, struct file *file)
580 return err;623 return err;
581}624}
582 625 
626+int hmdfs_file_release_merge(struct inode *inode, struct file *file)
627+{
628+ struct hmdfs_file_info *gfi = hmdfs_f(file);
629+ struct hmdfs_inode_info *info = hmdfs_i(inode);
630+ 
631+ if (file->f_flags & (O_RDWR | O_WRONLY))
632+ atomic_dec(&info->write_opened);
633+ file->private_data = NULL;
634+ fput(gfi->lower_file);
635+ kfree(gfi);
636+ return 0;
637+}
638+ 
583int hmdfs_file_flush_merge(struct file *file, fl_owner_t id)639int hmdfs_file_flush_merge(struct file *file, fl_owner_t id)
584{640{
585 struct hmdfs_file_info *gfi = hmdfs_f(file);641 struct hmdfs_file_info *gfi = hmdfs_f(file);
@@ -832,7 +888,7 @@ const struct file_operations hmdfs_file_fops_merge = {
832 .mmap = hmdfs_file_mmap_local,888 .mmap = hmdfs_file_mmap_local,
833 .open = hmdfs_file_open_merge,889 .open = hmdfs_file_open_merge,
834 .flush = hmdfs_file_flush_merge,890 .flush = hmdfs_file_flush_merge,
835- .release = hmdfs_file_release_local,891+ .release = hmdfs_file_release_merge,
836 .fsync = hmdfs_fsync_local,892 .fsync = hmdfs_fsync_local,
837 .unlocked_ioctl = hmdfs_file_ioctl_merge,893 .unlocked_ioctl = hmdfs_file_ioctl_merge,
838 .compat_ioctl = hmdfs_file_ioctl_merge,894 .compat_ioctl = hmdfs_file_ioctl_merge,
@@ -24,12 +24,21 @@
24#include "hmdfs_dentryfile.h"24#include "hmdfs_dentryfile.h"
25#include "hmdfs_trace.h"25#include "hmdfs_trace.h"
26 26 
27+#define HMDFS_SMALL_FILE_THRESHOLD 1024
28+ 
27static inline bool hmdfs_remote_write_cache_expired(29static inline bool hmdfs_remote_write_cache_expired(
28 struct hmdfs_inode_info *info)30 struct hmdfs_inode_info *info)
29{31{
30 return time_after(jiffies, info->writecache_expire);32 return time_after(jiffies, info->writecache_expire);
31}33}
32 34 
35+static inline bool hmdfs_remote_use_small_file_optimization(struct inode *inode)
36+{
37+ return HMDFS_SMALL_FILE_OPT_SUPPORTED &&
38+ READ_ONCE(hmdfs_sb(inode->i_sb)->small_file_optimization) &&
39+ i_size_read(inode) < HMDFS_SMALL_FILE_THRESHOLD;
40+}
41+ 
33enum expire_reason {42enum expire_reason {
34 ALL_GOOD = 0,43 ALL_GOOD = 0,
35 INO_DISMATCH = 1,44 INO_DISMATCH = 1,
@@ -154,29 +163,64 @@ static int hmdfs_open_final_remote(struct hmdfs_inode_info *info,
154 return 0;163 return 0;
155}164}
156 165 
157-int hmdfs_do_open_remote(struct file *file, bool keep_cache)166+static void hmdfs_set_page_content(struct page *page, const void *data)
167+{
168+ void *addr;
169+ addr = kmap(page);
170+ memcpy(addr, data, HMDFS_PAGE_SIZE);
171+ SetPageUptodate(page);
172+ kunmap(page);
173+}
174+ 
175+int hmdfs_do_open_remote(struct inode *inode, struct file *file,
176+ bool keep_cache, bool prefetch)
158{177{
159 struct hmdfs_inode_info *info = hmdfs_i(file_inode(file));178 struct hmdfs_inode_info *info = hmdfs_i(file_inode(file));
160 struct hmdfs_peer *conn = info->conn;179 struct hmdfs_peer *conn = info->conn;
161 struct hmdfs_open_ret open_ret;180 struct hmdfs_open_ret open_ret;
162 __u8 file_type = hmdfs_d(file->f_path.dentry)->file_type;181 __u8 file_type = hmdfs_d(file->f_path.dentry)->file_type;
163- char *send_buf;182+ struct page *page = NULL;
183+ char *send_buf = NULL;
164 int err = 0;184 int err = 0;
165 185 
186+ open_ret.page = NULL;
187+ if (prefetch) {
188+ open_ret.page = kmalloc(HMDFS_PAGE_SIZE, GFP_KERNEL);
189+ if (!open_ret.page)
190+ prefetch = false;
191+ }
192+ 
166 send_buf = hmdfs_get_dentry_relative_path(file->f_path.dentry);193 send_buf = hmdfs_get_dentry_relative_path(file->f_path.dentry);
167 if (!send_buf) {194 if (!send_buf) {
168 err = -ENOMEM;195 err = -ENOMEM;
169 goto out_free;196 goto out_free;
170 }197 }
171- err = hmdfs_send_open(conn, send_buf, file_type, &open_ret);198+ err = hmdfs_send_open(conn, send_buf, file_type, prefetch, &open_ret);
172 if (err) {199 if (err) {
173 hmdfs_err("hmdfs_send_open return failed with %d", err);200 hmdfs_err("hmdfs_send_open return failed with %d", err);
174 goto out_free;201 goto out_free;
175 }202 }
176 203 
177 err = hmdfs_open_final_remote(info, &open_ret, file, keep_cache);204 err = hmdfs_open_final_remote(info, &open_ret, file, keep_cache);
205+ if (err) {
206+ hmdfs_err("hmdfs_open_final_remote return failed with %d", err);
207+ goto out_free;
208+ }
209+ info->fid_opened = true;
210+ 
211+ if (!prefetch || open_ret.read_success != 1)
212+ goto out_free;
213+ 
214+ page = grab_cache_page(inode->i_mapping, 0);
lijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/file_remote.c:212] grab_cache_page(inode->i_mapping, 0) 获取页面后调用 hmdfs_set_page_content 写入内容,但成功路径中未调用 put_page 释放引用计数。unlock_page 仅解锁而非释放引用,grab_cache_page 获取的页面引用需配对 put_page 释放,否则会导致页面泄漏。需在 unlock_page 后添加 put_page(page) 释放页面引用。

likedislike
215+ if (!IS_ERR_OR_NULL(page)) {
216+ hmdfs_set_page_content(page, open_ret.page);
217+ unlock_page(page);
218+ put_page(page);
219+ info->need_reopen = false;
220+ }
178 221 
179out_free:222out_free:
223+ kfree(open_ret.page);
180 kfree(send_buf);224 kfree(send_buf);
181 return err;225 return err;
182}226}
@@ -245,9 +289,11 @@ static int hmdfs_remote_file_reopen(struct hmdfs_inode_info *info,
245 * send close (fid_1) // In case of leak289 * send close (fid_1) // In case of leak
246 * fid = new fid_2290 * fid = new fid_2
247 */291 */
248- if (fid.id != HMDFS_INODE_INVALID_FILE_ID)292+ if (info->fid_opened && fid.id != HMDFS_INODE_INVALID_FILE_ID)
249 hmdfs_send_close(conn, &fid);293 hmdfs_send_close(conn, &fid);
250- err = hmdfs_do_open_remote(filp, true);294+ info->fid_opened = false;
295+ err = hmdfs_do_open_remote(inode, filp, true,
296+ hmdfs_remote_use_small_file_optimization(inode));
251 inode_unlock(inode);297 inode_unlock(inode);
252 298 
253 spin_lock(&info->fid_lock);299 spin_lock(&info->fid_lock);
@@ -283,7 +329,11 @@ void hmdfs_do_close_remote(struct kref *kref)
283 container_of(kref, struct hmdfs_inode_info, ref);329 container_of(kref, struct hmdfs_inode_info, ref);
284 struct hmdfs_fid fid;330 struct hmdfs_fid fid;
285 331 
332+ if (!info->fid_opened)
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/file_remote.c:331] hmdfs_do_close_remote 中当 !info->fid_opened 时直接 return,但此函数是 kref_put 的释放回调。kref_put 已将引用计数减为 0 并调用此回调,而回调提前返回意味着引用计数为 0 但未做任何清理。后续若再次 kref_get(如 hmdfs_file_open_remotekref_read(ref) > 0 的 else 分支),将触发 use-after-free。建议:移除此处 return,或在 hmdfs_file_release_remote 中先判断 info->fid_opened 再决定是否 kref_put

likedislike
zengjy
4 天前 评论:
333+ return;
334+ 
286 hmdfs_remote_fetch_fid(info, &fid);335 hmdfs_remote_fetch_fid(info, &fid);
336+ info->fid_opened = false;
287 /* This function can return asynchronously */337 /* This function can return asynchronously */
288 hmdfs_send_close(info->conn, &fid);338 hmdfs_send_close(info->conn, &fid);
289}339}
@@ -334,15 +384,44 @@ int hmdfs_file_open_remote(struct inode *inode, struct file *file)
334{384{
335 struct hmdfs_inode_info *info = hmdfs_i(inode);385 struct hmdfs_inode_info *info = hmdfs_i(inode);
336 struct kref *ref = &(info->ref);386 struct kref *ref = &(info->ref);
387+ bool small_file_optimization;
337 int err = 0;388 int err = 0;
338 389 
339 inode_lock(inode);390 inode_lock(inode);
340- if (kref_read(ref) == 0) {391+ small_file_optimization = hmdfs_remote_use_small_file_optimization(inode);
341- err = hmdfs_do_open_remote(file, false);392+ 
342- if (err == 0)393+ if (!info->fid_opened) {
394+ /*
395+ * 1. If small file optimization is disabled, always do remote open.
396+ * 2. Otherwise, only do remote open for a non-read-only open or
397+ * when the file needs to be reopened.
398+ */
399+ if (!small_file_optimization ||
400+ (file->f_flags & O_ACCMODE) != O_RDONLY ||
401+ info->need_reopen) {
402+ err = hmdfs_do_open_remote(inode, file, false,
403+ small_file_optimization);
404+ } else if (info->writecache_expire &&
405+ hmdfs_remote_write_cache_expired(info)) {
406+ /*
407+ * Reset expiration and truncate here so
408+ * hmdfs_open_final_remote() does not need
409+ * to repeat the expiration check.
410+ */
411+ info->writecache_expire = 0;
412+ truncate_inode_pages(inode->i_mapping, 0);
413+ err = hmdfs_do_open_remote(inode, file, false, true);
414+ } else {
415+ atomic64_set(&info->write_counter, 0);
416+ info->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;
417+ }
418+ }
419+ 
420+ if (!err) {
421+ if (kref_read(ref) == 0)
343 kref_init(ref);422 kref_init(ref);
344- } else {423+ else
345- kref_get(ref);424+ kref_get(ref);
346 }425 }
347 inode_unlock(inode);426 inode_unlock(inode);
348 427 
@@ -427,6 +506,7 @@ int hmdfs_file_release_remote(struct inode *inode, struct file *file)
427 506 
428 inode_lock(inode);507 inode_lock(inode);
429 kref_put(&info->ref, hmdfs_do_close_remote);508 kref_put(&info->ref, hmdfs_do_close_remote);
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/file_remote.c:507] hmdfs_file_release_remote 无条件调用 kref_put(&info->ref, hmdfs_do_close_remote),移除了原有的 kref_read(&info->ref) > 0 保护。当 ref 已经为 0 时(例如 hmdfs_do_close_remote 已执行但 fid_opened 被设为 false 后又触发 release),kref_put 会导致引用计数下溢。虽然 hmdfs_do_close_remote 中增加了 !fid_opened 的提前返回来避免重复关闭,但引用计数语义已被破坏。建议恢复 kref_read > 0 的守卫条件,或在 fid_opened == false 时跳过 kref_put

likedislike
zengjy
4 天前 评论:
509+ 
430 hmdfs_remote_keep_writecache(inode, file);510 hmdfs_remote_keep_writecache(inode, file);
431 inode_unlock(inode);511 inode_unlock(inode);
432 512 
@@ -33,8 +33,9 @@
33#define HMDFS_IOC_GET_DST_PATH _IOR(HMDFS_IOC, 3, __u32)33#define HMDFS_IOC_GET_DST_PATH _IOR(HMDFS_IOC, 3, __u32)
34 34 
35 35 
36-#define HMDFS_PAGE_SIZE 409636+ 
37#define HMDFS_PAGE_OFFSET 1237#define HMDFS_PAGE_OFFSET 12
38+#define HMDFS_SMALL_FILE_OPT_SUPPORTED (HMDFS_PAGE_SIZE == PAGE_SIZE)
38 39 
39/* max xattr value size, not include '\0' */40/* max xattr value size, not include '\0' */
40#define HMDFS_XATTR_SIZE_MAX 409641#define HMDFS_XATTR_SIZE_MAX 4096
@@ -149,6 +150,7 @@ struct hmdfs_sb_info {
149 struct list_head server_cache;150 struct list_head server_cache;
150 struct list_head to_delete;151 struct list_head to_delete;
151 struct mutex cache_list_lock;152 struct mutex cache_list_lock;
153+ bool small_file_optimization;
152 154 
153 /* local operation time statistic */155 /* local operation time statistic */
154 struct server_statistic *s_server_statis;156 struct server_statistic *s_server_statis;
@@ -29,14 +29,38 @@ static inline void free_sm_outbuf(struct hmdfs_send_command *sm)
29 sm->out_buf = NULL;29 sm->out_buf = NULL;
30}30}
31 31 
32+static struct hmdfs_time_t msec_to_timespec(unsigned int msec)
33+{
34+ struct hmdfs_time_t timespec = {
35+ .tv_sec = msec / MSEC_PER_SEC,
36+ .tv_nsec = (msec % MSEC_PER_SEC) * NSEC_PER_MSEC,
37+ };
38+ 
39+ return timespec;
40+}
41+ 
42+static struct hmdfs_time_t hmdfs_current_kernel_time(void)
43+{
44+ struct hmdfs_time_t time;
45+ 
46+#if KERNEL_VERSION(4, 18, 0) < LINUX_VERSION_CODE
47+ ktime_get_coarse_real_ts64(&time);
48+#else
49+ time = current_kernel_time();
50+#endif
51+ return time;
52+}
53+ 
32int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,54int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
33- __u8 file_type, struct hmdfs_open_ret *open_ret)55+ __u8 file_type, bool prefetch,
56+ struct hmdfs_open_ret *open_ret)
34{57{
35 int ret;58 int ret;
36 int path_len = strlen(send_buf);59 int path_len = strlen(send_buf);
37 size_t send_len = sizeof(struct open_request) + path_len + 1;60 size_t send_len = sizeof(struct open_request) + path_len + 1;
38 struct open_request *open_req = kzalloc(send_len, GFP_KERNEL);61 struct open_request *open_req = kzalloc(send_len, GFP_KERNEL);
39 struct open_response *resp;62 struct open_response *resp;
63+ struct open_prefetch_response *prefetch_resp;
40 struct hmdfs_send_command sm = {64 struct hmdfs_send_command sm = {
41 .data = open_req,65 .data = open_req,
42 .len = send_len,66 .len = send_len,
@@ -44,6 +68,8 @@ int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
44 .local_filp = NULL,68 .local_filp = NULL,
45 };69 };
46 hmdfs_init_cmd(&sm.operations, F_OPEN);70 hmdfs_init_cmd(&sm.operations, F_OPEN);
71+ if (prefetch)
72+ hm_setprefetch(&sm.operations.reserved);
47 73 
48 if (!open_req) {74 if (!open_req) {
49 ret = -ENOMEM;75 ret = -ENOMEM;
@@ -59,6 +85,20 @@ int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
59 ret = -ENOENT;85 ret = -ENOENT;
60 if (ret)86 if (ret)
61 goto out;87 goto out;
88+ if (sm.out_len == HMDFS_OPEN_PREFETCH_RESPONSE_SIZE) {
89+ prefetch_resp = sm.out_buf;
90+ open_ret->read_success = prefetch_resp->read_success;
91+ if (open_ret->read_success)
92+ memcpy(open_ret->page, prefetch_resp->page,
93+ HMDFS_PAGE_SIZE);
94+ } else if (sm.out_len == sizeof(struct open_response)) {
95+ // Operation not supported
96+ open_ret->read_success = 0;
97+ } else {
98+ hmdfs_warning("invalid open response length %zu", sm.out_len);
99+ ret = -EINVAL;
100+ goto out;
101+ }
62 resp = sm.out_buf;102 resp = sm.out_buf;
63 103 
64 open_ret->ino = le64_to_cpu(resp->ino);104 open_ret->ino = le64_to_cpu(resp->ino);
@@ -1037,8 +1077,10 @@ void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path)
1037 };1077 };
1038 1078 
1039 hmdfs_init_cmd(&sm.operations, F_DROP_PUSH);1079 hmdfs_init_cmd(&sm.operations, F_DROP_PUSH);
1040- if (!dp_req)1080+ if (!dp_req) {
1081+ hmdfs_err("cannot allocate drop_push_request");
1041 return;1082 return;
1083+ }
1042 1084 
1043 dp_req->path_len = cpu_to_le32(path_len);1085 dp_req->path_len = cpu_to_le32(path_len);
1044 strncpy(dp_req->path, path, path_len);1086 strncpy(dp_req->path, path, path_len);
@@ -1047,6 +1089,56 @@ void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path)
1047 kfree(dp_req);1089 kfree(dp_req);
1048}1090}
1049 1091 
1092+void hmdfs_send_drop_page_push(struct hmdfs_peer *con, const char *send_buf, struct inode *inode)
1093+{
1094+ int path_len = strlen(send_buf);
1095+ size_t send_len = sizeof(struct drop_page_push_request) + path_len + 1;
1096+ struct drop_page_push_request *dpp_req = kzalloc(send_len, GFP_KERNEL);
1097+ if (!dpp_req) {
1098+ hmdfs_err("cannot allocate drop_page_push_request");
1099+ return;
1100+ }
1101+ 
1102+ struct hmdfs_send_command sm = {
1103+ .data = dpp_req,
1104+ .len = send_len,
1105+ .out_buf = NULL,
1106+ .local_filp = NULL,
1107+ };
1108+ 
1109+ struct hmdfs_time_t current_time = hmdfs_current_kernel_time();
1110+ struct hmdfs_time_t ctime = inode->__i_ctime;
1111+ struct hmdfs_time_t precision =
1112+ msec_to_timespec(con->sbi->dcache_precision);
1113+ 
1114+ loff_t size = i_size_read(inode);
1115+ 
1116+ hmdfs_init_cmd(&sm.operations, F_DROP_PAGE_PUSH);
1117+ 
1118+ dpp_req->file_size = cpu_to_le64(size);
1119+ dpp_req->ctime = cpu_to_le64(ctime.tv_sec);
1120+ dpp_req->ctime_nsec = cpu_to_le32(ctime.tv_nsec);
1121+ 
1122+ precision = hmdfs_time_add(ctime, precision);
1123+ if (hmdfs_time_compare(&current_time, &ctime) < 0) {
1124+ dpp_req->stable_ctime = cpu_to_le64(0);
1125+ dpp_req->stable_ctime_nsec = cpu_to_le32(0);
1126+ } else if (hmdfs_time_compare(&current_time, &ctime) >= 0 &&
1127+ hmdfs_time_compare(&current_time, &precision) < 0) {
1128+ dpp_req->stable_ctime = dpp_req->ctime;
1129+ dpp_req->stable_ctime_nsec = dpp_req->ctime_nsec;
1130+ } else {
1131+ dpp_req->stable_ctime = cpu_to_le64(precision.tv_sec);
1132+ dpp_req->stable_ctime_nsec = cpu_to_le32(precision.tv_nsec);
1133+ }
1134+ 
1135+ dpp_req->path_len = cpu_to_le32(path_len);
1136+ strscpy(dpp_req->buf, send_buf, send_len - sizeof(struct drop_page_push_request));
1137+ 
1138+ hmdfs_sendmessage_request(con, &sm);
1139+ kfree(dpp_req);
1140+}
1141+ 
1050static void *hmdfs_get_msg_next(struct hmdfs_peer *peer, int *id)1142static void *hmdfs_get_msg_next(struct hmdfs_peer *peer, int *id)
1051{1143{
1052 struct hmdfs_msg_idr_head *head = NULL;1144 struct hmdfs_msg_idr_head *head = NULL;
@@ -18,6 +18,8 @@ struct hmdfs_open_ret {
18 __u64 ino;18 __u64 ino;
19 struct hmdfs_time_t remote_ctime;19 struct hmdfs_time_t remote_ctime;
20 struct hmdfs_time_t stable_ctime;20 struct hmdfs_time_t stable_ctime;
21+ __u8 read_success;
22+ __u8 *page;
21};23};
22 24 
23struct hmdfs_writepage_context {25struct hmdfs_writepage_context {
@@ -90,7 +92,8 @@ void hmdfs_client_writepage_done(struct hmdfs_inode_info *info,
90 struct hmdfs_writepage_context *ctx);92 struct hmdfs_writepage_context *ctx);
91 93 
92int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,94int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
93- __u8 file_type, struct hmdfs_open_ret *open_ret);95+ __u8 file_type, bool prefetch,
96+ struct hmdfs_open_ret *open_ret);
94void hmdfs_send_close(struct hmdfs_peer *con, const struct hmdfs_fid *fid);97void hmdfs_send_close(struct hmdfs_peer *con, const struct hmdfs_fid *fid);
95int hmdfs_send_fsync(struct hmdfs_peer *con, const struct hmdfs_fid *fid,98int hmdfs_send_fsync(struct hmdfs_peer *con, const struct hmdfs_fid *fid,
96 __s64 start, __s64 end, __s32 datasync);99 __s64 start, __s64 end, __s32 datasync);
@@ -2364,6 +2364,38 @@ void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path)
2364 path_put(&root_path);2364 path_put(&root_path);
2365}2365}
2366 2366 
2367+void hmdfs_add_remote_page_cache_list(struct hmdfs_peer *con, struct file *file)
2368+{
2369+ struct remotecache_item *item = NULL;
2370+ struct remotecache_item *item_temp = NULL;
2371+ struct hmdfs_dentry_info *d_info = NULL;
2372+ 
2373+ d_info = hmdfs_d(file->f_path.dentry);
2374+ if (!d_info)
2375+ return;
2376+ 
2377+ /* find duplicate con */
2378+ mutex_lock(&d_info->remote_cache_list_lock);
2379+ list_for_each_entry_safe(item, item_temp,
2380+ &(d_info->remote_cache_list_head), list) {
2381+ if (item->con->device_id == con->device_id) {
2382+ mutex_unlock(&d_info->remote_cache_list_lock);
2383+ return;
2384+ }
2385+ }
2386+ 
2387+ item = kzalloc(sizeof(*item), GFP_KERNEL);
2388+ if (!item) {
2389+ mutex_unlock(&d_info->remote_cache_list_lock);
2390+ return;
2391+ }
2392+ 
2393+ item->con = con;
2394+ item->drop_flag = 0;
2395+ list_add(&(item->list), &(d_info->remote_cache_list_head));
2396+ mutex_unlock(&d_info->remote_cache_list_lock);
2397+}
2398+ 
2367int hmdfs_drop_remote_cache_dents(struct dentry *dentry)2399int hmdfs_drop_remote_cache_dents(struct dentry *dentry)
2368{2400{
2369 struct path lower_path;2401 struct path lower_path;
@@ -2420,6 +2452,97 @@ int hmdfs_drop_remote_cache_dents(struct dentry *dentry)
2420 return 0;2452 return 0;
2421}2453}
2422 2454 
2455+int hmdfs_drop_remote_cache_pages(struct dentry *dentry)
2456+{
2457+ struct remotecache_item *item = NULL;
2458+ struct remotecache_item *item_temp = NULL;
2459+ struct hmdfs_dentry_info *d_info = NULL;
2460+ char *relative_path = NULL;
2461+ LIST_HEAD(drop_list);
2462+ 
2463+ if (!dentry) {
2464+ hmdfs_err("dentry null and return");
2465+ return 0;
2466+ }
2467+ 
2468+ d_info = hmdfs_d(dentry);
2469+ if (!d_info) {
2470+ hmdfs_err("d_info null and return");
2471+ return 0;
2472+ }
2473+ relative_path = hmdfs_get_dentry_relative_path(dentry);
2474+ if (!relative_path) {
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/hmdfs_dentryfile.c:2474] hmdfs_release_remote_cachehmdfs_get_dentry_relative_path 失败时仅打印错误日志,但并未 return 或设置标志,后续 list_for_each_entry_safe 循环仍会在 relative_path && lower_inode 条件下跳过发送。这意味着缓存项被从链表移除并 kfree,但 drop 消息未发送,导致对端缓存不会被清理。建议:relative_path 为 NULL 时直接 return,或在循环中仅移除但保留 item 供后续重试。

likedislike
zengjy
4 天前 评论:
2475+ hmdfs_err("get dentry relative path failed");
2476+ return 0;
2477+ }
2478+ mutex_lock(&d_info->remote_cache_list_lock);
2479+ list_for_each_entry_safe(item, item_temp,
2480+ &(d_info->remote_cache_list_head), list) {
2481+ if (item->drop_flag == 1) {
2482+ item->drop_flag = 0;
2483+ continue;
2484+ }
2485+ list_move_tail(&item->list, &drop_list);
2486+ }
2487+ mutex_unlock(&d_info->remote_cache_list_lock);
2488+ 
2489+ list_for_each_entry_safe(item, item_temp, &drop_list, list) {
2490+ hmdfs_send_drop_page_push(item->con, relative_path,
2491+ dentry->d_inode);
2492+ list_del(&item->list);
lijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/hmdfs_dentryfile.c:2486] hmdfs_drop_remote_cache_pages 中在 list_for_each_entry_safe 循环内先调用 hmdfs_send_drop_page_push(可能睡眠),然后 list_delkfree。但 hmdfs_send_drop_page_push 内部调用 hmdfs_sendmessage_request 发送网络消息,若发送失败或阻塞,持有 remote_cache_list_lock 互斥锁的时间会很长。建议先收集需要发送的 item 到临时列表,释放锁后再发送消息,减少锁持有时间。

likedislike
2493+ kfree(item);
2494+ }
2495+ 
2496+ kfree(relative_path);
2497+ return 0;
2498+}
2499+ 
2500+void hmdfs_release_remote_cache(struct dentry *dentry)
2501+{
2502+ struct remotecache_item *item = NULL;
2503+ struct remotecache_item *item_temp = NULL;
2504+ struct hmdfs_dentry_info *d_info = NULL;
2505+ struct path lower_path;
2506+ struct inode *lower_inode;
2507+ char *relative_path;
2508+ LIST_HEAD(release_list);
2509+ 
2510+ if (!dentry)
2511+ return;
2512+ 
2513+ d_info = hmdfs_d(dentry);
2514+ if (!d_info)
2515+ return;
2516+ 
2517+ mutex_lock(&d_info->remote_cache_list_lock);
2518+ list_splice_init(&d_info->remote_cache_list_head, &release_list);
2519+ mutex_unlock(&d_info->remote_cache_list_lock);
2520+ if (list_empty(&release_list))
2521+ return;
2522+ 
2523+ hmdfs_get_lower_path(dentry, &lower_path);
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/hmdfs_dentryfile.c:2523] hmdfs_release_remote_cache 中调用 hmdfs_get_lower_path(dentry, &lower_path) 后直接使用 lower_path.dentry,但若 dentry 无下层路径,lower_path 可能未初始化或 lower_path.dentry 为 NULL。虽然后续有 lower_path.dentry ? 的三元判断,但 hmdfs_put_lower_path 仍会被无条件调用,此时若 lower_path 无效可能导致问题。建议:检查 hmdfs_get_lower_path 返回值或在 lower_path.dentry == NULL 时跳过 hmdfs_put_lower_path

likedislike
zengjy
4 天前 评论:
2524+ lower_inode = lower_path.dentry ? d_inode(lower_path.dentry) : NULL;
2525+ relative_path = hmdfs_get_dentry_relative_path(dentry);
2526+ if (!relative_path)
2527+ hmdfs_err("get dentry relative path failed");
2528+ 
2529+ list_for_each_entry_safe(item, item_temp, &release_list, list) {
2530+ if (relative_path && lower_inode) {
2531+ if (S_ISDIR(lower_inode->i_mode))
2532+ hmdfs_send_drop_push(item->con, relative_path);
2533+ else if (S_ISREG(lower_inode->i_mode))
2534+ hmdfs_send_drop_page_push(item->con,
2535+ relative_path,
2536+ lower_inode);
2537+ }
2538+ list_del(&item->list);
2539+ kfree(item);
2540+ }
2541+ 
2542+ kfree(relative_path);
2543+ hmdfs_put_lower_path(&lower_path);
2544+}
2545+ 
2423/* Clear the dentry cache files of target directory */2546/* Clear the dentry cache files of target directory */
2424int hmdfs_clear_cache_dents(struct dentry *dentry, bool remove_cache)2547int hmdfs_clear_cache_dents(struct dentry *dentry, bool remove_cache)
2425{2548{
@@ -211,6 +211,7 @@ struct clearcache_item {
211};211};
212 212 
213void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path);213void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path);
214+void hmdfs_add_remote_page_cache_list(struct hmdfs_peer *con, struct file *file);
214 215 
215struct remotecache_item {216struct remotecache_item {
216 struct hmdfs_peer *con;217 struct hmdfs_peer *con;
@@ -245,7 +246,10 @@ struct cache_file_callback {
245};246};
246 247 
247int hmdfs_drop_remote_cache_dents(struct dentry *dentry);248int hmdfs_drop_remote_cache_dents(struct dentry *dentry);
249+int hmdfs_drop_remote_cache_pages(struct dentry *dentry);
250+void hmdfs_release_remote_cache(struct dentry *dentry);
248void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path);251void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path);
252+void hmdfs_send_drop_page_push(struct hmdfs_peer *con, const char *send_buf, struct inode *inode);
249void hmdfs_mark_drop_flag(uint64_t device_id, struct dentry *dentry);253void hmdfs_mark_drop_flag(uint64_t device_id, struct dentry *dentry);
250void hmdfs_clear_drop_flag(struct dentry *dentry);254void hmdfs_clear_drop_flag(struct dentry *dentry);
251void delete_in_cache_file(uint64_t dev_id, struct dentry *dentry);255void delete_in_cache_file(uint64_t dev_id, struct dentry *dentry);
@@ -128,7 +128,9 @@ loff_t hmdfs_file_llseek_local(struct file *file, loff_t offset, int whence);
128 128 
129ssize_t hmdfs_do_read_iter(struct file *file, struct iov_iter *iter,129ssize_t hmdfs_do_read_iter(struct file *file, struct iov_iter *iter,
130 loff_t *ppos);130 loff_t *ppos);
131-ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,131+ssize_t hmdfs_do_write_iter_merge(struct file *file, struct iov_iter *iter,
132+ loff_t *ppos);
133+ssize_t hmdfs_do_write_iter_local(struct file *file, struct iov_iter *iter,
132 loff_t *ppos);134 loff_t *ppos);
133 135 
134int hmdfs_file_release_local(struct inode *inode, struct file *file);136int hmdfs_file_release_local(struct inode *inode, struct file *file);
@@ -461,6 +461,7 @@ static void hmdfs_update_open_response(struct hmdfs_peer *con,
461 loff_t size = info->stat_valid ? info->stat.size :461 loff_t size = info->stat_valid ? info->stat.size :
462 i_size_read(info->inode);462 i_size_read(info->inode);
463 463 
464+ // NB: info->real_ino is the lower_file's i_ino and i_igeneration, not the info->file's
464 resp->ino = cpu_to_le64(info->real_ino);465 resp->ino = cpu_to_le64(info->real_ino);
465 resp->file_ver = cpu_to_le64(hmdfs_server_pack_fid_ver(con, cmd));466 resp->file_ver = cpu_to_le64(hmdfs_server_pack_fid_ver(con, cmd));
466 resp->file_id = cpu_to_le32(info->file_id);467 resp->file_id = cpu_to_le32(info->file_id);
@@ -490,6 +491,22 @@ static void hmdfs_update_open_response(struct hmdfs_peer *con,
490 }491 }
491}492}
492 493 
494+static void hmdfs_update_open_prefetch(struct hmdfs_open_info *info,
495+ __u8 *read_success, __u8 *page)
496+{
497+ loff_t pos = 0;
498+ ssize_t readsize;
499+ 
500+ readsize = kernel_read(info->file, page, HMDFS_PAGE_SIZE, &pos);
501+ if (readsize < 0) {
502+ *read_success = 0;
503+ } else {
504+ *read_success = 1;
505+ if (readsize != HMDFS_PAGE_SIZE)
506+ memset(page + readsize, 0, HMDFS_PAGE_SIZE - readsize);
507+ }
508+}
509+ 
493static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,510static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,
494 const char *filename,511 const char *filename,
495 struct hmdfs_open_info *info)512 struct hmdfs_open_info *info)
@@ -534,66 +551,6 @@ static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,
534 return 0;551 return 0;
535}552}
536 553 
537-void hmdfs_server_open(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
538- void *data)
539-{
540- struct open_request *recv = data;
541- int sizeread = sizeof(struct open_response);
542- struct open_response *resp = NULL;
543- struct hmdfs_open_info *info = NULL;
544- int ret = 0;
545- 
546- trace_hmdfs_server_open_enter(con, recv);
547- 
548- resp = kzalloc(sizeread, GFP_KERNEL);
549- info = kmalloc(sizeof(*info), GFP_KERNEL);
550- if (!resp || !info) {
551- ret = -ENOMEM;
552- goto err_free;
553- }
554- 
555- if (path_contain_dotdot(recv->buf, recv->path_len)) {
556- ret = -EINVAL;
557- goto err_free;
558- }
559- 
560- info->file = hmdfs_open_file(con, recv->buf, recv->file_type,
561- &info->file_id);
562- if (IS_ERR(info->file)) {
563- ret = PTR_ERR(info->file);
564- goto err_free;
565- }
566- 
567- ret = hmdfs_get_open_info(con, recv->file_type, recv->buf, info);
568- if (ret)
569- goto err_close;
570- 
571- hmdfs_update_open_response(con, cmd, info, resp);
572- 
573- trace_hmdfs_server_open_exit(con, resp, info->file, 0);
574- ret = hmdfs_sendmessage_response(con, cmd, sizeread, resp, 0);
575- if (ret) {
576- hmdfs_err("sending msg response failed, file_id %d, err %d",
577- info->file_id, ret);
578- remove_file_from_conn(con, info->file_id);
579- hmdfs_close_path(info->file);
580- }
581- hmdfs_close_path(info->file);
582- kfree(resp);
583- kfree(info);
584- return;
585- 
586-err_close:
587- hmdfs_close_path(info->file);
588- remove_file_from_conn(con, info->file_id);
589- hmdfs_close_path(info->file);
590-err_free:
591- kfree(resp);
592- kfree(info);
593- trace_hmdfs_server_open_exit(con, NULL, NULL, ret);
594- hmdfs_send_err_response(con, cmd, ret);
595-}
596- 
597static int hmdfs_check_and_create(struct path *path_parent,554static int hmdfs_check_and_create(struct path *path_parent,
598 struct dentry *dentry, uint64_t device_id,555 struct dentry *dentry, uint64_t device_id,
599 umode_t mode, bool is_excl)556 umode_t mode, bool is_excl)
@@ -744,8 +701,14 @@ void hmdfs_server_atomic_open(struct hmdfs_peer *con,
744 struct hmdfs_head_cmd *cmd, void *data)701 struct hmdfs_head_cmd *cmd, void *data)
745{702{
746 int err;703 int err;
704+ bool is_prefetch = hm_isprefetch(cmd->operations.reserved);
705+ size_t resp_size = is_prefetch ?
706+ HMDFS_ATOMIC_OPEN_PREFETCH_RESPONSE_SIZE :
707+ sizeof(struct atomic_open_response);
747 struct atomic_open_request *recv = data;708 struct atomic_open_request *recv = data;
748- struct atomic_open_response *resp = NULL;709+ void *resp_buf = NULL;
710+ struct atomic_open_response *resp;
711+ struct atomic_open_prefetch_response *prefetch_resp;
749 struct hmdfs_open_info *info = NULL;712 struct hmdfs_open_info *info = NULL;
750 char *file_path = recv->buf;713 char *file_path = recv->buf;
751 char *file = recv->buf + recv->path_len + 1;714 char *file = recv->buf + recv->path_len + 1;
@@ -760,19 +723,28 @@ void hmdfs_server_atomic_open(struct hmdfs_peer *con,
760 }723 }
761 724 
762 info = kmalloc(sizeof(*info), GFP_KERNEL);725 info = kmalloc(sizeof(*info), GFP_KERNEL);
763- resp = kzalloc(sizeof(*resp), GFP_KERNEL);726+ resp_buf = kzalloc(resp_size, GFP_KERNEL);
764- if (!resp || !info) {727+ if (!resp_buf || !info) {
765 err = -ENOMEM;728 err = -ENOMEM;
766 goto out;729 goto out;
767 }730 }
731+ if (is_prefetch) {
732+ prefetch_resp = resp_buf;
733+ resp = &prefetch_resp->atomic_open_resp;
734+ } else {
735+ resp = resp_buf;
736+ }
768 737 
769 err = hmdfs_server_do_atomic_open(con, cmd, recv, info, resp);738 err = hmdfs_server_do_atomic_open(con, cmd, recv, info, resp);
739+ if (!err && is_prefetch)
740+ hmdfs_update_open_prefetch(info, &prefetch_resp->read_success,
741+ prefetch_resp->page);
770 742 
771out:743out:
772 if (err) {744 if (err) {
773 hmdfs_send_err_response(con, cmd, err);745 hmdfs_send_err_response(con, cmd, err);
774 } else {746 } else {
775- err = hmdfs_sendmessage_response(con, cmd, sizeof(*resp), resp,747+ err = hmdfs_sendmessage_response(con, cmd, resp_size, resp_buf,
776 0);748 0);
777 if (err) {749 if (err) {
778 hmdfs_err("sending msg response failed, file_id %d, err %d",750 hmdfs_err("sending msg response failed, file_id %d, err %d",
@@ -782,7 +754,7 @@ void hmdfs_server_atomic_open(struct hmdfs_peer *con,
782 }754 }
783 }755 }
784 kfree(info);756 kfree(info);
785- kfree(resp);757+ kfree(resp_buf);
786}758}
787 759 
788void hmdfs_server_release(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,760void hmdfs_server_release(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
@@ -1087,6 +1059,84 @@ static char *server_lookup_lower(struct hmdfs_peer *peer, const char *req_path,
1087 return err ? ERR_PTR(err) : lo_p_name;1059 return err ? ERR_PTR(err) : lo_p_name;
1088}1060}
1089 1061 
1062+void hmdfs_server_open(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1063+ void *data)
1064+{
1065+ struct open_request *recv = data;
1066+ bool is_prefetch = hm_isprefetch(cmd->operations.reserved);
1067+ size_t resp_size = is_prefetch ? HMDFS_OPEN_PREFETCH_RESPONSE_SIZE :
1068+ sizeof(struct open_response);
1069+ void *resp_buf = NULL;
1070+ struct open_response *resp;
1071+ struct open_prefetch_response *prefetch_resp;
lijiawei
lijiaweilijiawei4 天前

[fs/hmdfs/hmdfs_server.c:1066] hmdfs_server_openresp 类型为 struct open_response *,但实际分配大小取决于 is_prefetch。当 is_prefetch 为 true 时,prefetch_resp = (struct open_prefetch_response *)resp 进行了不兼容类型的强制转换。与 hmdfs_server_atomic_open 同样存在严格别名规则违规问题。建议:使用 void * 中间指针或两个独立指针变量。

likedislike
1072+ struct hmdfs_open_info *info = NULL;
1073+ int ret = 0;
1074+ 
1075+ trace_hmdfs_server_open_enter(con, recv);
1076+ 
1077+ resp_buf = kzalloc(resp_size, GFP_KERNEL);
1078+ info = kmalloc(sizeof(*info), GFP_KERNEL);
1079+ if (!resp_buf || !info) {
1080+ ret = -ENOMEM;
1081+ goto err_free;
1082+ }
1083+ if (is_prefetch) {
1084+ prefetch_resp = resp_buf;
1085+ resp = &prefetch_resp->open_resp;
1086+ } else {
1087+ resp = resp_buf;
1088+ }
1089+ 
1090+ if (path_contain_dotdot(recv->buf, recv->path_len)) {
1091+ ret = -EINVAL;
1092+ goto err_free;
1093+ }
1094+ 
1095+ info->file = hmdfs_open_file(con, recv->buf, recv->file_type,
1096+ &info->file_id);
1097+ 
1098+ if (IS_ERR(info->file)) {
1099+ ret = PTR_ERR(info->file);
1100+ goto err_free;
1101+ }
1102+ 
1103+ ret = hmdfs_get_open_info(con, recv->file_type, recv->buf, info);
1104+ if (ret)
1105+ goto err_close;
1106+ 
1107+ hmdfs_update_open_response(con, cmd, info, resp);
1108+ if (is_prefetch)
1109+ hmdfs_update_open_prefetch(info, &prefetch_resp->read_success,
1110+ prefetch_resp->page);
1111+ 
1112+ trace_hmdfs_server_open_exit(con, resp, info->file, 0);
1113+ ret = hmdfs_sendmessage_response(con, cmd, resp_size, resp_buf, 0);
1114+ if (ret) {
1115+ hmdfs_err("sending msg response failed, file_id %d, err %d",
1116+ info->file_id, ret);
1117+ remove_file_from_conn(con, info->file_id);
1118+ goto out;
1119+ }
1120+ 
1121+ if (is_prefetch)
1122+ hmdfs_add_remote_page_cache_list(con, info->file);
lijiaweilijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/hmdfs_server.c:1093] hmdfs_server_open 中成功发送响应后才调用 hmdfs_add_remote_page_cache_list 注册缓存项,但传入的是 info->file(即将在下一行 hmdfs_close_path 关闭的文件指针)。hmdfs_add_remote_page_cache_list 通过 file->f_path.dentry 获取 d_info,而关闭后 file 指针可能已失效。虽然 dentry 本身有独立引用计数,但使用已关闭的 file 指针访问 f_path.dentry 不够安全。建议在关闭前提取 dentry 或改用其他方式传递。

likedislike
zengjy
4 天前 评论:
lijiaweilijiawei3 天前

[fs/hmdfs/hmdfs_server.c:1122] hmdfs_add_remote_page_cache_list(con, info->file) 在 out 标签处的 hmdfs_close_path(info->file) 之前调用。如果 hmdfs_add_remote_page_cache_list 仅保存 info->file 指针而不增加引用计数(如 get_file()),则 hmdfs_close_path 释放文件后,远端页面缓存列表中将持有悬垂指针,导致 use-after-free。请确认该函数内部是否正确管理了文件引用计数。

likedislike
zengjy
3 天前 评论:
1123+ 
1124+out:
1125+ hmdfs_close_path(info->file);
1126+ kfree(resp_buf);
1127+ kfree(info);
1128+ return;
1129+ 
1130+err_close:
1131+ hmdfs_close_path(info->file);
1132+ remove_file_from_conn(con, info->file_id);
1133+err_free:
1134+ kfree(resp_buf);
1135+ kfree(info);
1136+ trace_hmdfs_server_open_exit(con, NULL, NULL, ret);
1137+ hmdfs_send_err_response(con, cmd, ret);
1138+}
1139+ 
1090void hmdfs_server_readdir(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,1140void hmdfs_server_readdir(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1091 void *data)1141 void *data)
1092{1142{
@@ -1441,6 +1491,7 @@ static bool hmdfs_filldir_real(struct dir_context *ctx, const char *name,
1441 }1491 }
1442 1492 
1443 if (d_type == DT_REG || d_type == DT_DIR) {1493 if (d_type == DT_REG || d_type == DT_DIR) {
1494+ // gc->file is dentry_file, the anonymous tmp_file
1444 create_dentry(child, d_inode(child), gc->file, gc->sbi);1495 create_dentry(child, d_inode(child), gc->file, gc->sbi);
1445 gc->num++;1496 gc->num++;
1446 } else if (d_type == DT_LNK) {1497 } else if (d_type == DT_LNK) {
@@ -1500,12 +1551,14 @@ struct file *hmdfs_server_rebuild_dents(struct hmdfs_sb_info *sbi,
1500 struct file *dentry_file = NULL;1551 struct file *dentry_file = NULL;
1501 struct hmdfs_dcache_header header;1552 struct hmdfs_dcache_header header;
1502 1553 
1554+ // create a anonymous tmp_file in sbi->cache_dir
1503 dentry_file = create_local_dentry_file_cache(sbi);1555 dentry_file = create_local_dentry_file_cache(sbi);
1504 if (IS_ERR(dentry_file)) {1556 if (IS_ERR(dentry_file)) {
1505 hmdfs_err("file create failed err=%ld", PTR_ERR(dentry_file));1557 hmdfs_err("file create failed err=%ld", PTR_ERR(dentry_file));
1506 return dentry_file;1558 return dentry_file;
1507 }1559 }
1508 1560 
1561+ // path is the lower path of the target dir
1509 file = dentry_open(path, O_RDONLY | O_DIRECTORY, current_cred());1562 file = dentry_open(path, O_RDONLY | O_DIRECTORY, current_cred());
1510 if (IS_ERR(file)) {1563 if (IS_ERR(file)) {
1511 err = PTR_ERR(file);1564 err = PTR_ERR(file);
@@ -1556,6 +1609,10 @@ void hmdfs_server_writepage(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1556 ssize_t ret;1609 ssize_t ret;
1557 int err = 0;1610 int err = 0;
1558 1611 
1612+ struct remotecache_item *item = NULL;
1613+ struct remotecache_item *item_temp = NULL;
1614+ struct hmdfs_dentry_info *d_info = NULL;
1615+ 
1559 file_id = le32_to_cpu(writepage_recv->file_id);1616 file_id = le32_to_cpu(writepage_recv->file_id);
1560 file_ver = le64_to_cpu(writepage_recv->file_ver);1617 file_ver = le64_to_cpu(writepage_recv->file_ver);
1561 file = get_file_by_fid_and_ver(con, cmd, file_id, file_ver);1618 file = get_file_by_fid_and_ver(con, cmd, file_id, file_ver);
@@ -1567,12 +1624,24 @@ void hmdfs_server_writepage(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1567 err = PTR_ERR(file);1624 err = PTR_ERR(file);
1568 goto out;1625 goto out;
1569 }1626 }
1627+ d_info = hmdfs_d(file->f_path.dentry);
1570 1628 
1571 pos = (loff_t)le64_to_cpu(writepage_recv->index) << HMDFS_PAGE_OFFSET;1629 pos = (loff_t)le64_to_cpu(writepage_recv->index) << HMDFS_PAGE_OFFSET;
1572 count = le32_to_cpu(writepage_recv->count);1630 count = le32_to_cpu(writepage_recv->count);
1573 ret = kernel_write(file, writepage_recv->buf, count, &pos);1631 ret = kernel_write(file, writepage_recv->buf, count, &pos);
1574 if (ret != count)1632 if (ret != count)
1575 err = -EIO;1633 err = -EIO;
1634+ else if (d_info) {
1635+ mutex_lock(&d_info->remote_cache_list_lock);
1636+ list_for_each_entry_safe(item, item_temp,
1637+ &(d_info->remote_cache_list_head), list) {
1638+ if (item->con->device_id == con->device_id) {
1639+ item->drop_flag = 1;
lijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/hmdfs_server.c:1610] hmdfs_server_writepage 中当服务端写入数据后,将同一设备的 drop_flag 设为 1 以跳过后续 drop_page_push。但 hmdfs_drop_remote_cache_pagesdrop_flag == 1 时仅重置为 0 并 continue,不发送推送也不删除 item。这意味着该 item 会永久保留在链表中,直到下一次 writepage 才会被再次设为 1。如果文件之后不再被写入,该 item 永远不会被释放,造成内存泄漏。建议在 drop_flag == 1 时也发送推送并删除 item,或在 writepage 路径中删除而非仅设标记。

likedislike
zengjy
4 天前 评论:
1640+ break;
1641+ }
1642+ }
1643+ mutex_unlock(&d_info->remote_cache_list_lock);
1644+ }
1576 1645 
1577 hmdfs_close_path(file);1646 hmdfs_close_path(file);
1578out:1647out:
@@ -2123,3 +2192,88 @@ void hmdfs_server_get_drop_push(struct hmdfs_peer *con,
2123quickack:2192quickack:
2124 set_conn_sock_quickack(con);2193 set_conn_sock_quickack(con);
2125}2194}
2195+ 
2196+static struct drop_page_push_recv init_drop_page_push_recv(void *data)
2197+{
2198+ struct drop_page_push_request *dpp_recv = data;
2199+ struct drop_page_push_recv dppr = {
2200+ .file_size = le64_to_cpu(dpp_recv->file_size),
2201+ .path_len = le32_to_cpu(dpp_recv->path_len),
2202+ .buf = dpp_recv->buf,
2203+ };
2204+ dppr.remote_ctime.tv_sec = le64_to_cpu(dpp_recv->ctime);
2205+ dppr.remote_ctime.tv_nsec = le32_to_cpu(dpp_recv->ctime_nsec);
2206+ dppr.stable_ctime.tv_sec = le64_to_cpu(dpp_recv->stable_ctime);
2207+ dppr.stable_ctime.tv_nsec = le32_to_cpu(dpp_recv->stable_ctime_nsec);
2208+ 
2209+ return dppr;
2210+}
2211+ 
2212+void hmdfs_server_get_drop_page_push(struct hmdfs_peer *con,
2213+ struct hmdfs_head_cmd *cmd, void *data)
2214+{
2215+ struct drop_page_push_recv dpp_recv = init_drop_page_push_recv(data);
2216+ struct path root_path, path;
2217+ struct inode *inode;
2218+ struct hmdfs_inode_info *info;
2219+ char *tmp_path = NULL;
2220+ int err;
2221+ 
2222+ // Learn from hmdfs_server_get_drop_push
2223+ const char *root_name = con->sbi->real_dst;
2224+ if (path_contain_dotdot(dpp_recv.buf, dpp_recv.path_len)) {
2225+ err = -EINVAL;
2226+ goto quickack;
2227+ }
2228+ 
2229+ err = kern_path(root_name, 0, &root_path);
2230+ if (err) {
2231+ hmdfs_err("kern_path failed at %s, err = %d", root_name, err);
2232+ goto quickack;
2233+ }
2234+ 
2235+ tmp_path = kzalloc(PATH_MAX, GFP_KERNEL);
2236+ if (!tmp_path)
2237+ goto out_kfree;
2238+ snprintf(tmp_path, PATH_MAX, "/" DEVICE_VIEW_ROOT "/%s%s", con->cid,
2239+ dpp_recv.buf);
2240+ 
2241+ err = vfs_path_lookup(root_path.dentry, root_path.mnt, tmp_path, 0,
2242+ &path);
2243+ if (err) {
2244+ hmdfs_info("path found failed at %s, err = %d", dpp_recv.buf, err);
2245+ goto out_tfree;
2246+ }
2247+ 
2248+ inode = d_inode(path.dentry);
2249+ info = hmdfs_i(inode);
2250+ 
2251+ inode_lock(inode);
2252+ 
2253+ /*
2254+ * It doesn't make sense to update metadata here but leaving the data obsolete.
2255+ * But how to design here? Should we also transport the data here?
2256+ * Transporting data means we may never open the file twice,
2257+ * which may corrupt file reference counter in server when the file is not opened but the page cache is missing.
2258+ */
2259+ // inode->i_ctime = dpp_recv.remote_ctime;
2260+ // info->remote_ctime = dpp_recv.remote_ctime;
2261+ // info->stable_ctime = dpp_recv.stable_ctime;
2262+ 
2263+ // i_size_write(inode, dpp_recv.file_size);
2264+ 
2265+ info->need_reopen = true;
lijiawei
lijiaweilijiawei12 天前

[重要] [fs/hmdfs/hmdfs_server.c:2236] hmdfs_server_get_drop_page_push 中设置 info->need_reopen = true,但未对 inode->i_mapping 的页缓存做失效处理(如 invalidate_inode_pages2)。如果远程设备写入了新数据并推送了 drop_page_push,本地 inode 的页缓存可能包含过期数据。仅设置 need_reopen 标志,在下次 open 时才会重新读取,但已映射的页面不会被自动刷新。建议在此处同时清空页缓存,确保后续读取获取最新数据。

likedislike
zengjy
4 天前 评论:
2266+ 
2267+ inode_unlock(inode);
2268+ 
2269+ path_put(&path);
2270+ 
2271+out_tfree:
2272+ kfree(tmp_path);
2273+ 
2274+out_kfree:
2275+ path_put(&root_path);
2276+ 
2277+quickack:
2278+ set_conn_sock_quickack(con);
2279+}
@@ -20,6 +20,14 @@
20#define DATA_SEC_LEVEL_LABEL "user.security"20#define DATA_SEC_LEVEL_LABEL "user.security"
21#define DATA_SEC_LEVEL_LENGTH 1021#define DATA_SEC_LEVEL_LENGTH 10
22 22 
23+struct drop_page_push_recv {
24+ __u64 file_size;
25+ struct hmdfs_time_t remote_ctime;
26+ struct hmdfs_time_t stable_ctime;
27+ __u32 path_len;
28+ char *buf;
29+};
30+ 
23static inline void hmdfs_send_err_response(struct hmdfs_peer *con,31static inline void hmdfs_send_err_response(struct hmdfs_peer *con,
24 struct hmdfs_head_cmd *cmd, int err)32 struct hmdfs_head_cmd *cmd, int err)
25{33{
@@ -74,6 +82,8 @@ void hmdfs_server_listxattr(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
74 void *data);82 void *data);
75void hmdfs_server_get_drop_push(struct hmdfs_peer *con,83void hmdfs_server_get_drop_push(struct hmdfs_peer *con,
76 struct hmdfs_head_cmd *cmd, void *data);84 struct hmdfs_head_cmd *cmd, void *data);
85+void hmdfs_server_get_drop_page_push(struct hmdfs_peer *con,
86+ struct hmdfs_head_cmd *cmd, void *data);
77 87 
78void __init hmdfs_server_add_node_evt_cb(void);88void __init hmdfs_server_add_node_evt_cb(void);
79#endif89#endif
@@ -69,8 +69,12 @@ struct hmdfs_inode_info {
69 struct inode *lower_inode; // for local/merge inode69 struct inode *lower_inode; // for local/merge inode
70 struct hmdfs_peer *conn; // for remote inode70 struct hmdfs_peer *conn; // for remote inode
71 struct kref ref;71 struct kref ref;
72+ bool need_reopen;
73+ spinlock_t modify_lock;
74+ bool modified;
72 spinlock_t fid_lock;75 spinlock_t fid_lock;
73 struct hmdfs_fid fid;76 struct hmdfs_fid fid;
77+ bool fid_opened;
74 unsigned long fid_flags;78 unsigned long fid_flags;
75 wait_queue_head_t fid_wq;79 wait_queue_head_t fid_wq;
76 __u8 inode_type; // deprecated: use ino system instead80 __u8 inode_type; // deprecated: use ino system instead
@@ -315,6 +315,9 @@ static struct inode *hmdfs_alloc_inode(struct super_block *sb)
315 if (!gi)315 if (!gi)
316 return NULL;316 return NULL;
317 memset(gi, 0, offsetof(struct hmdfs_inode_info, vfs_inode));317 memset(gi, 0, offsetof(struct hmdfs_inode_info, vfs_inode));
318+ gi->need_reopen = true;
319+ gi->modified = false;
320+ spin_lock_init(&gi->modify_lock);
318 INIT_LIST_HEAD(&gi->wb_list);321 INIT_LIST_HEAD(&gi->wb_list);
319 init_rwsem(&gi->wpage_sem);322 init_rwsem(&gi->wpage_sem);
320 gi->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;323 gi->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;
@@ -698,6 +701,7 @@ static void hmdfs_init_cmd_timeout(struct hmdfs_sb_info *sbi)
698 set_cmd_timeout(sbi, F_STATFS, TIMEOUT_COMMON);701 set_cmd_timeout(sbi, F_STATFS, TIMEOUT_COMMON);
699 set_cmd_timeout(sbi, F_CONNECT_REKEY, TIMEOUT_NONE);702 set_cmd_timeout(sbi, F_CONNECT_REKEY, TIMEOUT_NONE);
700 set_cmd_timeout(sbi, F_DROP_PUSH, TIMEOUT_NONE);703 set_cmd_timeout(sbi, F_DROP_PUSH, TIMEOUT_NONE);
704+ set_cmd_timeout(sbi, F_DROP_PAGE_PUSH, TIMEOUT_NONE);
701 set_cmd_timeout(sbi, F_GETATTR, TIMEOUT_COMMON);705 set_cmd_timeout(sbi, F_GETATTR, TIMEOUT_COMMON);
702 set_cmd_timeout(sbi, F_FSYNC, TIMEOUT_90S);706 set_cmd_timeout(sbi, F_FSYNC, TIMEOUT_90S);
703 set_cmd_timeout(sbi, F_SYNCFS, TIMEOUT_30S);707 set_cmd_timeout(sbi, F_SYNCFS, TIMEOUT_30S);
@@ -747,6 +751,7 @@ static int hmdfs_init_sbi(struct hmdfs_sb_info *sbi)
747 sbi->dcache_precision = DEFAULT_DCACHE_PRECISION;751 sbi->dcache_precision = DEFAULT_DCACHE_PRECISION;
748 sbi->dcache_timeout = DEFAULT_DCACHE_TIMEOUT;752 sbi->dcache_timeout = DEFAULT_DCACHE_TIMEOUT;
749 sbi->write_cache_timeout = DEFAULT_WRITE_CACHE_TIMEOUT;753 sbi->write_cache_timeout = DEFAULT_WRITE_CACHE_TIMEOUT;
754+ sbi->small_file_optimization = false;
750 hmdfs_init_cmd_timeout(sbi);755 hmdfs_init_cmd_timeout(sbi);
751 sbi->async_cb_delay = HMDFS_NODE_EVT_CB_DELAY;756 sbi->async_cb_delay = HMDFS_NODE_EVT_CB_DELAY;
752 sbi->async_req_max_active = DEFAULT_SRV_REQ_MAX_ACTIVE;757 sbi->async_req_max_active = DEFAULT_SRV_REQ_MAX_ACTIVE;