已关闭
基于HMDFS的批量小文件共享优化 #321
基于HMDFS的批量小文件共享优化 #321
已关闭
zengjy创建于 3月19日关闭于 15 天前
17 个文件变更+623-79
@@ -938,6 +938,35 @@ static struct sbi_attribute sbi_write_cache_timeout_sec_attr =
938 __ATTR(write_cache_timeout_sec, 0664, sbi_write_cache_timeout_sec_show,938 __ATTR(write_cache_timeout_sec, 0664, sbi_write_cache_timeout_sec_show,
939 sbi_write_cache_timeout_sec_store);939 sbi_write_cache_timeout_sec_store);
940 940 
941+static ssize_t sbi_small_file_optimization_show(struct kobject *kobj,
942+ struct sbi_attribute *attr, char *buf)
943+{
944+ const struct hmdfs_sb_info *sbi = to_sbi(kobj);
945+ 
946+ return snprintf(buf, PAGE_SIZE, "%d\n",
947+ READ_ONCE(sbi->small_file_optimization));
948+}
949+ 
950+static ssize_t sbi_small_file_optimization_store(struct kobject *kobj,
951+ struct sbi_attribute *attr,
952+ const char *buf, size_t len)
953+{
954+ struct hmdfs_sb_info *sbi = to_sbi(kobj);
955+ bool small_file_optimization;
956+ int err;
957+ 
958+ err = kstrtobool(buf, &small_file_optimization);
959+ if (err)
960+ return err;
961+ 
962+ WRITE_ONCE(sbi->small_file_optimization, small_file_optimization);
963+ return len;
964+}
965+ 
966+static struct sbi_attribute sbi_small_file_optimization_attr =
967+ __ATTR(small_file_optimization, 0644, sbi_small_file_optimization_show,
968+ sbi_small_file_optimization_store);
969+ 
941static ssize_t sbi_node_evt_cb_delay_show(struct kobject *kobj,970static ssize_t sbi_node_evt_cb_delay_show(struct kobject *kobj,
942 struct sbi_attribute *attr,971 struct sbi_attribute *attr,
943 char *buf)972 char *buf)
@@ -1213,6 +1242,7 @@ static struct attribute *sbi_attrs[] = {
1213 &sbi_dcache_threshold_attr.attr,1242 &sbi_dcache_threshold_attr.attr,
1214 &sbi_dcache_timeout_attr.attr,1243 &sbi_dcache_timeout_attr.attr,
1215 &sbi_write_cache_timeout_sec_attr.attr,1244 &sbi_write_cache_timeout_sec_attr.attr,
1245+ &sbi_small_file_optimization_attr.attr,
1216 &sbi_local_op_attr.attr,1246 &sbi_local_op_attr.attr,
1217 &sbi_delay_resp_attr.attr,1247 &sbi_delay_resp_attr.attr,
1218 &sbi_wb_timeout_ms_attr.attr,1248 &sbi_wb_timeout_ms_attr.attr,
@@ -1335,6 +1365,7 @@ HMDFS_CMD_ATTR(rename, F_RENAME);
1335HMDFS_CMD_ATTR(setattr, F_SETATTR);1365HMDFS_CMD_ATTR(setattr, F_SETATTR);
1336HMDFS_CMD_ATTR(statfs, F_STATFS);1366HMDFS_CMD_ATTR(statfs, F_STATFS);
1337HMDFS_CMD_ATTR(drop_push, F_DROP_PUSH);1367HMDFS_CMD_ATTR(drop_push, F_DROP_PUSH);
1368+HMDFS_CMD_ATTR(drop_page_push, F_DROP_PAGE_PUSH);
1338HMDFS_CMD_ATTR(getattr, F_GETATTR);1369HMDFS_CMD_ATTR(getattr, F_GETATTR);
1339HMDFS_CMD_ATTR(fsync, F_FSYNC);1370HMDFS_CMD_ATTR(fsync, F_FSYNC);
1340HMDFS_CMD_ATTR(syncfs, F_SYNCFS);1371HMDFS_CMD_ATTR(syncfs, F_SYNCFS);
@@ -1354,6 +1385,7 @@ static struct attribute *sbi_timeout_attrs[] = {
1354 ATTR_LIST(getattr), ATTR_LIST(fsync),1385 ATTR_LIST(getattr), ATTR_LIST(fsync),
1355 ATTR_LIST(syncfs), ATTR_LIST(getxattr),1386 ATTR_LIST(syncfs), ATTR_LIST(getxattr),
1356 ATTR_LIST(setxattr), ATTR_LIST(listxattr),1387 ATTR_LIST(setxattr), ATTR_LIST(listxattr),
1388+ ATTR_LIST(drop_page_push),
1357 NULL1389 NULL
1358};1390};
1359ATTRIBUTE_GROUPS(sbi_timeout);1391ATTRIBUTE_GROUPS(sbi_timeout);
@@ -27,6 +27,7 @@ void hmdfs_message_verify_init(void)
27 need_response[F_RELEASE] = false;27 need_response[F_RELEASE] = false;
28 need_response[F_CONNECT_REKEY] = false;28 need_response[F_CONNECT_REKEY] = false;
29 need_response[F_DROP_PUSH] = false;29 need_response[F_DROP_PUSH] = false;
30+ need_response[F_DROP_PAGE_PUSH] = false;
30 31 
31 for (flag = 0; flag < C_FLAG_SIZE; flag++) {32 for (flag = 0; flag < C_FLAG_SIZE; flag++) {
32 for (cmd = 0; cmd < F_SIZE; cmd++) {33 for (cmd = 0; cmd < F_SIZE; cmd++) {
@@ -43,11 +44,12 @@ void hmdfs_message_verify_init(void)
43 sizeof(struct open_request) + PATH_MAX + 1;44 sizeof(struct open_request) + PATH_MAX + 1;
44 message_length[C_REQUEST][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =45 message_length[C_REQUEST][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
45 MESSAGE_LEN_JUDGE_RANGE;46 MESSAGE_LEN_JUDGE_RANGE;
46- message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MIN_INDEX] = 0;47+ message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MIN_INDEX] = sizeof(
48+ struct open_response) - HMDFS_PAGE_SIZE - 1;
47 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MAX_INDEX] =49 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_MAX_INDEX] =
48 sizeof(struct open_response);50 sizeof(struct open_response);
49 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =51 message_length[C_RESPONSE][F_OPEN][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
50- MESSAGE_LEN_JUDGE_BIN;52+ MESSAGE_LEN_JUDGE_RANGE;
51 53 
52 message_length[C_REQUEST][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] =54 message_length[C_REQUEST][F_ATOMIC_OPEN][HMDFS_MESSAGE_MIN_INDEX] =
53 sizeof(struct atomic_open_request);55 sizeof(struct atomic_open_request);
@@ -269,6 +271,13 @@ void hmdfs_message_verify_init(void)
269 sizeof(struct drop_push_request) + PATH_MAX + 1;271 sizeof(struct drop_push_request) + PATH_MAX + 1;
270 message_length[C_REQUEST][F_DROP_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =272 message_length[C_REQUEST][F_DROP_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
271 MESSAGE_LEN_JUDGE_RANGE;273 MESSAGE_LEN_JUDGE_RANGE;
274+
275+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_MIN_INDEX] =
276+ sizeof(struct drop_page_push_request);
277+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_MAX_INDEX] =
278+ sizeof(struct drop_page_push_request) + PATH_MAX + 1;
279+ message_length[C_REQUEST][F_DROP_PAGE_PUSH][HMDFS_MESSAGE_LEN_JUDGE_INDEX] =
280+ MESSAGE_LEN_JUDGE_RANGE;
272}281}
273 282 
274static int is_str_msg_valid(char *msg, int str_len[], size_t str_num)283static int is_str_msg_valid(char *msg, int str_len[], size_t str_num)
@@ -308,7 +317,7 @@ static int verify_open_resp(size_t msg_len, void *msg)
308{317{
309 struct open_response *resp = msg;318 struct open_response *resp = msg;
310 319 
311- if (msg_len != sizeof(*resp))320+ if (msg_len != sizeof(*resp) && msg_len != sizeof(*resp) - HMDFS_PAGE_SIZE - 1)
312 return -EINVAL;321 return -EINVAL;
313 322 
314 return 0;323 return 0;
@@ -856,6 +865,34 @@ static int hmdfs_drop_push_verify(int flag, size_t msg_len, void *msg)
856 return 0;865 return 0;
857}866}
858 867 
868+static int verify_drop_page_push_req(size_t msg_len, void *msg)
869+{
870+ struct drop_page_push_request *req = msg;
871+ int str_len[] = {req->path_len};
872+
873+ if (req->path_len < 0 || req->path_len >= PATH_MAX)
874+ return -EINVAL;
875+
876+ if (msg_len != sizeof(*req) + req->path_len + 1)
877+ return -EINVAL;
878+
879+ if (is_str_msg_valid(req->buf, str_len, sizeof(str_len) / sizeof(int)))
880+ return -EINVAL;
881+
882+ return 0;
883+}
884+
885+static int hmdfs_drop_page_push_verify(int flag, size_t msg_len, void *msg)
886+{
887+ if (!msg || !msg_len)
888+ return 0;
889+
890+ if (flag == C_REQUEST)
891+ return verify_drop_page_push_req(msg_len, msg);
892+
893+ return 0;
894+}
895+ 
859typedef int (*hmdfs_message_verify_func)(int, size_t, void *);896typedef int (*hmdfs_message_verify_func)(int, size_t, void *);
860 897 
861static const hmdfs_message_verify_func message_verify[F_SIZE] = {898static const hmdfs_message_verify_func message_verify[F_SIZE] = {
@@ -871,6 +908,7 @@ static const hmdfs_message_verify_func message_verify[F_SIZE] = {
871 [F_SETATTR] = hmdfs_setattr_verify,908 [F_SETATTR] = hmdfs_setattr_verify,
872 [F_STATFS] = hmdfs_statfs_verify,909 [F_STATFS] = hmdfs_statfs_verify,
873 [F_DROP_PUSH] = hmdfs_drop_push_verify,910 [F_DROP_PUSH] = hmdfs_drop_push_verify,
911+ [F_DROP_PAGE_PUSH] = hmdfs_drop_page_push_verify,
874 [F_GETATTR] = hmdfs_getattr_verify,912 [F_GETATTR] = hmdfs_getattr_verify,
875 [F_GETXATTR] = hmdfs_getxattr_verify,913 [F_GETXATTR] = hmdfs_getxattr_verify,
876 [F_SETXATTR] = hmdfs_setxattr_verify,914 [F_SETXATTR] = hmdfs_setxattr_verify,
@@ -12,6 +12,9 @@
12#include <linux/wait.h>12#include <linux/wait.h>
13#include <linux/workqueue.h>13#include <linux/workqueue.h>
14#include <linux/namei.h>14#include <linux/namei.h>
15+#include <linux/bits.h>
16+ 
17+#define HMDFS_PREFETCH_FLAG ((__u8)BIT(0))
15 18 
16struct hmdfs_cmd {19struct hmdfs_cmd {
17 __u8 reserved;20 __u8 reserved;
@@ -20,9 +23,21 @@ struct hmdfs_cmd {
20 __u8 reserved2;23 __u8 reserved2;
21} __packed;24} __packed;
22 25 
26+static inline bool hm_isprefetch(__u8 flag)
27+{
28+ return (flag & HMDFS_PREFETCH_FLAG) == HMDFS_PREFETCH_FLAG;
29+}
30+ 
31+static inline void hm_setprefetch(__u8 *flag)
32+{
33+ *flag |= HMDFS_PREFETCH_FLAG;
34+}
35+ 
23#define HMDFS_MSG_MAGIC 0xF736#define HMDFS_MSG_MAGIC 0xF7
24#define HMDFS_MAX_MESSAGE_LEN (8 * 1024 * 1024)37#define HMDFS_MAX_MESSAGE_LEN (8 * 1024 * 1024)
25 38 
39+#define HMDFS_PAGE_SIZE 4096
oh_create_jiawei

【建议】宏定义写死,是否有考虑未来的兼容演进,如支持内存大页?

likedislike
zengjy
6月10日 评论:
40+ 
26struct hmdfs_head_cmd {41struct hmdfs_head_cmd {
27 __u8 magic;42 __u8 magic;
28 __u8 version;43 __u8 version;
@@ -181,6 +196,7 @@ enum FILE_CMD {
181 F_RESERVED_7 = 26,196 F_RESERVED_7 = 26,
182 F_RESERVED_8 = 27,197 F_RESERVED_8 = 27,
183 F_ATOMIC_OPEN = 28,198 F_ATOMIC_OPEN = 28,
199+ F_DROP_PAGE_PUSH = 29,
184 F_SIZE,200 F_SIZE,
185};201};
186 202 
@@ -204,6 +220,8 @@ struct open_response {
204 __le64 stable_ctime;220 __le64 stable_ctime;
205 __le32 stable_ctime_nsec;221 __le32 stable_ctime_nsec;
206 __le64 ichange_count;222 __le64 ichange_count;
223+ __u8 read_success;
224+ __u8 page[HMDFS_PAGE_SIZE];
lijiawei
lijiaweilijiawei7月3日

[Critical] open_response 中嵌入 4096 字节 page 数组,导致 hmdfs_open_ret 栈变量有栈溢出风险\n\nfs/hmdfs/comm/protocol.h:224 - __u8 page[HMDFS_PAGE_SIZE]\n\nstruct open_response__packed 结构体,新增 __u8 page[HMDFS_PAGE_SIZE] 使其大小膨胀到约 4KB+。而 hmdfs_open_ret(hmdfs_client.h)同样嵌入了 __u8 page[HMDFS_PAGE_SIZE],该结构体在 hmdfs_do_open_remote() 中作为栈变量 struct hmdfs_open_ret open_ret 分配。4KB 的栈数组在内核中非常危险,可能导致栈溢出。\n\n建议:将 hmdfs_open_ret.page 改为动态分配(kmalloc),或者仅在 prefetch 场景下分配 page 缓冲。

likedislike
zengjy
7月10日 评论:
lijiawei
lijiawei
7月13日 评论:
zengjy
7月18日 评论:
207} __packed;225} __packed;
208 226 
209enum hmdfs_open_flags {227enum hmdfs_open_flags {
@@ -332,6 +350,18 @@ struct drop_push_request {
332 char path[0];350 char path[0];
333} __packed;351} __packed;
334 352 
353+struct drop_page_push_request {
354+ __le64 file_size;
355+ __le64 ctime;
356+ __le32 ctime_nsec;
357+ __le64 mtime;
358+ __le32 mtime_nsec;
359+ __le64 stable_ctime;
360+ __le32 stable_ctime_nsec;
361+ __le32 path_len;
362+ char buf[0];
363+} __packed;
364+ 
335struct setattr_request {365struct setattr_request {
336 __le64 size;366 __le64 size;
337 __le32 valid;367 __le32 valid;
@@ -43,6 +43,7 @@ static const request_callback s_recv_callbacks[F_SIZE] = {
43 [F_SETATTR] = hmdfs_server_setattr,43 [F_SETATTR] = hmdfs_server_setattr,
44 [F_STATFS] = hmdfs_server_statfs,44 [F_STATFS] = hmdfs_server_statfs,
45 [F_DROP_PUSH] = hmdfs_server_get_drop_push,45 [F_DROP_PUSH] = hmdfs_server_get_drop_push,
46+ [F_DROP_PAGE_PUSH] = hmdfs_server_get_drop_page_push,
46 [F_GETATTR] = hmdfs_server_getattr,47 [F_GETATTR] = hmdfs_server_getattr,
47 [F_FSYNC] = hmdfs_server_fsync,48 [F_FSYNC] = hmdfs_server_fsync,
48 [F_SYNCFS] = hmdfs_server_syncfs,49 [F_SYNCFS] = hmdfs_server_syncfs,
@@ -835,6 +836,7 @@ static int hmdfs_request_recv(struct hmdfs_peer *con,
835 case F_STATFS:836 case F_STATFS:
836 case F_CONNECT_REKEY:837 case F_CONNECT_REKEY:
837 case F_DROP_PUSH:838 case F_DROP_PUSH:
839+ case F_DROP_PAGE_PUSH:
838 case F_GETATTR:840 case F_GETATTR:
839 case F_FSYNC:841 case F_FSYNC:
840 case F_SYNCFS:842 case F_SYNCFS:
@@ -1045,6 +1047,7 @@ static int hmdfs_response_recv(struct hmdfs_peer *con,
1045 case F_STATFS:1047 case F_STATFS:
1046 case F_CONNECT_REKEY:1048 case F_CONNECT_REKEY:
1047 case F_DROP_PUSH:1049 case F_DROP_PUSH:
1050+ case F_DROP_PAGE_PUSH:
1048 case F_GETATTR:1051 case F_GETATTR:
1049 case F_FSYNC:1052 case F_FSYNC:
1050 case F_SYNCFS:1053 case F_SYNCFS:
@@ -64,6 +64,14 @@ int hmdfs_file_release_local(struct inode *inode, struct file *file)
64 64 
65 if (file->f_flags & (O_RDWR | O_WRONLY))65 if (file->f_flags & (O_RDWR | O_WRONLY))
66 atomic_dec(&info->write_opened);66 atomic_dec(&info->write_opened);
67+ 
68+ spin_lock(&info->modify_lock);
69+ if (info->modified){
70+ hmdfs_drop_remote_cache_pages(file_dentry(file));
71+ info->modified = false;
72+ }
73+ spin_unlock(&info->modify_lock);
74+ 
67 file->private_data = NULL;75 file->private_data = NULL;
68 fput(gfi->lower_file);76 fput(gfi->lower_file);
69 kfree(gfi);77 kfree(gfi);
@@ -112,6 +120,7 @@ static ssize_t hmdfs_local_read_iter(struct kiocb *iocb, struct iov_iter *iter)
112static void hmdfs_file_modified(struct file *file)120static void hmdfs_file_modified(struct file *file)
113{121{
114 struct inode *inode = file_inode(file);122 struct inode *inode = file_inode(file);
123+ struct hmdfs_inode_info *info = hmdfs_i(inode);
115 struct dentry *dentry = file_dentry(file);124 struct dentry *dentry = file_dentry(file);
116 struct file *lower_file = hmdfs_f(file)->lower_file;125 struct file *lower_file = hmdfs_f(file)->lower_file;
117 struct inode *lower_inode = file_inode(lower_file);126 struct inode *lower_inode = file_inode(lower_file);
@@ -123,9 +132,13 @@ static void hmdfs_file_modified(struct file *file)
123 132 
124 if (!hmdfs_i_merge(hmdfs_i(inode)))133 if (!hmdfs_i_merge(hmdfs_i(inode)))
125 update_inode_to_dentry(dentry, inode);134 update_inode_to_dentry(dentry, inode);
135+
136+ spin_lock(&info->modify_lock);
137+ info->modified = true;
138+ spin_unlock(&info->modify_lock);
126}139}
127 140 
128-ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,141+ssize_t hmdfs_do_write_iter_local(struct file *file, struct iov_iter *iter,
129 loff_t *ppos)142 loff_t *ppos)
130{143{
131 ssize_t ret;144 ssize_t ret;
@@ -154,7 +167,7 @@ ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,
154 167 
155ssize_t hmdfs_local_write_iter(struct kiocb *iocb, struct iov_iter *iter)168ssize_t hmdfs_local_write_iter(struct kiocb *iocb, struct iov_iter *iter)
156{169{
157- return hmdfs_do_write_iter(iocb->ki_filp, iter, &iocb->ki_pos);170+ return hmdfs_do_write_iter_local(iocb->ki_filp, iter, &iocb->ki_pos);
158}171}
159 172 
160int hmdfs_fsync_local(struct file *file, loff_t start, loff_t end, int datasync)173int hmdfs_fsync_local(struct file *file, loff_t start, loff_t end, int datasync)
@@ -537,9 +537,52 @@ static ssize_t hmdfs_merge_read_iter(struct kiocb *iocb, struct iov_iter *iter)
537 return hmdfs_do_read_iter(iocb->ki_filp, iter, &iocb->ki_pos);537 return hmdfs_do_read_iter(iocb->ki_filp, iter, &iocb->ki_pos);
538}538}
539 539 
540+static void hmdfs_file_modified(struct file *file)
541+{
542+ struct inode *inode = file_inode(file);
543+ struct dentry *dentry = file_dentry(file);
544+ struct file *lower_file = hmdfs_f(file)->lower_file;
545+ struct inode *lower_inode = file_inode(lower_file);
546+ 
547+ inode->i_atime = lower_inode->i_atime;
548+ inode->__i_ctime = lower_inode->__i_ctime;
549+ inode->i_mtime = lower_inode->i_mtime;
550+ i_size_write(inode, i_size_read(lower_inode));
551+ 
552+ if (!hmdfs_i_merge(hmdfs_i(inode)))
553+ update_inode_to_dentry(dentry, inode);
554+}
555+ 
556+ssize_t hmdfs_do_write_iter_merge(struct file *file, struct iov_iter *iter,
557+ loff_t *ppos)
558+{
559+ ssize_t ret;
560+ struct file *lower_file = hmdfs_f(file)->lower_file;
561+ struct inode *inode = file_inode(file);
562+ 
563+ if (!iov_iter_count(iter))
564+ return 0;
565+ 
566+ inode_lock(inode);
567+ 
568+ ret = file_remove_privs(file);
569+ if (ret)
570+ goto out_unlock;
571+ 
572+ file_start_write(lower_file);
573+ ret = vfs_iter_write(lower_file, iter, ppos, 0);
574+ file_end_write(lower_file);
575+ 
576+ hmdfs_file_modified(file);
577+ 
578+out_unlock:
579+ inode_unlock(inode);
580+ return ret;
581+}
582+ 
540ssize_t hmdfs_merge_write_iter(struct kiocb *iocb, struct iov_iter *iter)583ssize_t hmdfs_merge_write_iter(struct kiocb *iocb, struct iov_iter *iter)
541{584{
542- return hmdfs_do_write_iter(iocb->ki_filp, iter, &iocb->ki_pos);585+ return hmdfs_do_write_iter_merge(iocb->ki_filp, iter, &iocb->ki_pos);
543}586}
544 587 
545int hmdfs_file_open_merge(struct inode *inode, struct file *file)588int hmdfs_file_open_merge(struct inode *inode, struct file *file)
@@ -580,6 +623,19 @@ int hmdfs_file_open_merge(struct inode *inode, struct file *file)
580 return err;623 return err;
581}624}
582 625 
626+int hmdfs_file_release_merge(struct inode *inode, struct file *file)
627+{
628+ struct hmdfs_file_info *gfi = hmdfs_f(file);
629+ struct hmdfs_inode_info *info = hmdfs_i(inode);
630+ 
631+ if (file->f_flags & (O_RDWR | O_WRONLY))
632+ atomic_dec(&info->write_opened);
633+ file->private_data = NULL;
634+ fput(gfi->lower_file);
635+ kfree(gfi);
636+ return 0;
637+}
638+ 
583int hmdfs_file_flush_merge(struct file *file, fl_owner_t id)639int hmdfs_file_flush_merge(struct file *file, fl_owner_t id)
584{640{
585 struct hmdfs_file_info *gfi = hmdfs_f(file);641 struct hmdfs_file_info *gfi = hmdfs_f(file);
@@ -832,7 +888,7 @@ const struct file_operations hmdfs_file_fops_merge = {
832 .mmap = hmdfs_file_mmap_local,888 .mmap = hmdfs_file_mmap_local,
833 .open = hmdfs_file_open_merge,889 .open = hmdfs_file_open_merge,
834 .flush = hmdfs_file_flush_merge,890 .flush = hmdfs_file_flush_merge,
835- .release = hmdfs_file_release_local,891+ .release = hmdfs_file_release_merge,
836 .fsync = hmdfs_fsync_local,892 .fsync = hmdfs_fsync_local,
837 .unlocked_ioctl = hmdfs_file_ioctl_merge,893 .unlocked_ioctl = hmdfs_file_ioctl_merge,
838 .compat_ioctl = hmdfs_file_ioctl_merge,894 .compat_ioctl = hmdfs_file_ioctl_merge,
@@ -24,12 +24,20 @@
24#include "hmdfs_dentryfile.h"24#include "hmdfs_dentryfile.h"
25#include "hmdfs_trace.h"25#include "hmdfs_trace.h"
26 26 
27+#define HMDFS_SMALL_FILE_THRESHOLD 1024
28+ 
27static inline bool hmdfs_remote_write_cache_expired(29static inline bool hmdfs_remote_write_cache_expired(
28 struct hmdfs_inode_info *info)30 struct hmdfs_inode_info *info)
29{31{
30 return time_after(jiffies, info->writecache_expire);32 return time_after(jiffies, info->writecache_expire);
31}33}
32 34 
35+static inline bool hmdfs_remote_use_small_file_optimization(struct inode *inode)
36+{
37+ return READ_ONCE(hmdfs_sb(inode->i_sb)->small_file_optimization) &&
38+ i_size_read(inode) < HMDFS_SMALL_FILE_THRESHOLD;
39+}
40+ 
33enum expire_reason {41enum expire_reason {
34 ALL_GOOD = 0,42 ALL_GOOD = 0,
35 INO_DISMATCH = 1,43 INO_DISMATCH = 1,
@@ -154,29 +162,62 @@ static int hmdfs_open_final_remote(struct hmdfs_inode_info *info,
154 return 0;162 return 0;
155}163}
156 164 
157-int hmdfs_do_open_remote(struct file *file, bool keep_cache)165+static void hmdfs_set_page_content(struct page *page, const void *data)
166+{
167+ void *addr;
168+ addr = kmap(page);
169+ memcpy(addr, data, HMDFS_PAGE_SIZE);
170+ SetPageUptodate(page);
171+ kunmap(page);
172+}
173+ 
174+int hmdfs_do_open_remote(struct inode *inode, struct file *file,
175+ bool keep_cache, bool prefetch)
158{176{
159 struct hmdfs_inode_info *info = hmdfs_i(file_inode(file));177 struct hmdfs_inode_info *info = hmdfs_i(file_inode(file));
160 struct hmdfs_peer *conn = info->conn;178 struct hmdfs_peer *conn = info->conn;
161 struct hmdfs_open_ret open_ret;179 struct hmdfs_open_ret open_ret;
162 __u8 file_type = hmdfs_d(file->f_path.dentry)->file_type;180 __u8 file_type = hmdfs_d(file->f_path.dentry)->file_type;
163- char *send_buf;181+ struct page *page = NULL;
182+ char *send_buf = NULL;
164 int err = 0;183 int err = 0;
165 184 
185+ open_ret.page = NULL;
186+ if (prefetch) {
187+ open_ret.page = kmalloc(HMDFS_PAGE_SIZE, GFP_KERNEL);
188+ if (!open_ret.page)
189+ prefetch = false;
190+ }
191+ 
166 send_buf = hmdfs_get_dentry_relative_path(file->f_path.dentry);192 send_buf = hmdfs_get_dentry_relative_path(file->f_path.dentry);
167 if (!send_buf) {193 if (!send_buf) {
168 err = -ENOMEM;194 err = -ENOMEM;
169 goto out_free;195 goto out_free;
170 }196 }
171- err = hmdfs_send_open(conn, send_buf, file_type, &open_ret);197+ err = hmdfs_send_open(conn, send_buf, file_type, prefetch, &open_ret);
172 if (err) {198 if (err) {
173 hmdfs_err("hmdfs_send_open return failed with %d", err);199 hmdfs_err("hmdfs_send_open return failed with %d", err);
174 goto out_free;200 goto out_free;
175 }201 }
176 202 
177 err = hmdfs_open_final_remote(info, &open_ret, file, keep_cache);203 err = hmdfs_open_final_remote(info, &open_ret, file, keep_cache);
204+ if (err) {
205+ hmdfs_err("hmdfs_open_final_remote return failed with %d", err);
206+ goto out_free;
207+ }
208+ 
209+ if (!prefetch || open_ret.read_success != 1)
210+ goto out_free;
211+ 
212+ page = grab_cache_page(inode->i_mapping, 0);
213+ if (!IS_ERR_OR_NULL(page)) {
214+ hmdfs_set_page_content(page, open_ret.page);
215+ unlock_page(page);
216+ info->need_reopen = false;
217+ }
178 218 
179out_free:219out_free:
220+ kfree(open_ret.page);
180 kfree(send_buf);221 kfree(send_buf);
181 return err;222 return err;
182}223}
@@ -247,7 +288,8 @@ static int hmdfs_remote_file_reopen(struct hmdfs_inode_info *info,
247 */288 */
248 if (fid.id != HMDFS_INODE_INVALID_FILE_ID)289 if (fid.id != HMDFS_INODE_INVALID_FILE_ID)
249 hmdfs_send_close(conn, &fid);290 hmdfs_send_close(conn, &fid);
250- err = hmdfs_do_open_remote(filp, true);291+ err = hmdfs_do_open_remote(inode, filp, true,
292+ hmdfs_remote_use_small_file_optimization(inode));
251 inode_unlock(inode);293 inode_unlock(inode);
252 294 
253 spin_lock(&info->fid_lock);295 spin_lock(&info->fid_lock);
@@ -334,13 +376,38 @@ int hmdfs_file_open_remote(struct inode *inode, struct file *file)
334{376{
335 struct hmdfs_inode_info *info = hmdfs_i(inode);377 struct hmdfs_inode_info *info = hmdfs_i(inode);
336 struct kref *ref = &(info->ref);378 struct kref *ref = &(info->ref);
379+ bool small_file_optimization;
337 int err = 0;380 int err = 0;
338 381 
339 inode_lock(inode);382 inode_lock(inode);
383+ small_file_optimization = hmdfs_remote_use_small_file_optimization(inode);
384+ 
340 if (kref_read(ref) == 0) {385 if (kref_read(ref) == 0) {
341- err = hmdfs_do_open_remote(file, false);386+ /*
342- if (err == 0)387+ * 1. If small file optimization is disabled, we always do remote open.
343- kref_init(ref);388+ * 2. If small file optimization is enabled, we only do remote open when it's not a read-only open,
389+ * or the file needs to be reopened.
390+ */
391+ if (!small_file_optimization || (file->f_flags & O_ACCMODE) != O_RDONLY || info->need_reopen) {
392+ err = hmdfs_do_open_remote(inode, file, false, small_file_optimization);
393+ if (err == 0)
394+ kref_init(ref);
395+ } else if (info->writecache_expire && hmdfs_remote_write_cache_expired(info)) {
396+ /*
397+ * Reset expiration and truncate here so
398+ * hmdfs_open_final_remote() does not need
399+ * to repeat the expiration check.
400+ */
401+ info->writecache_expire = 0;
402+ truncate_inode_pages(inode->i_mapping, 0);
403+ err = hmdfs_do_open_remote(inode, file, false, true);
404+ if (err == 0) {
405+ kref_init(ref);
406+ }
407+ } else {
408+ atomic64_set(&info->write_counter, 0);
lijiawei
lijiaweilijiawei7月3日

[Major] hmdfs_file_open_remote() 中 else 分支跳过 remote open 时未初始化 kref\n\nfs/hmdfs/file_remote.c:400 - } else {\n\n当 small_file_optimization 启用且文件是只读打开且不需要 reopen 时,代码进入 else 分支直接设置 write_counter 和 getattr_isize,但没有调用 kref_init(ref)。后续 hmdfs_file_release_remote() 会调用 kref_put(&info->ref, ...),如果 kref 从未被 init,kref_put 的行为是未定义的,可能导致引用计数错误或 use-after-free。\n\n需要在 else 分支中添加 kref_init(ref)

likedislike
zengjy
7月9日 评论:
409+ info->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;
410+ }
344 } else {411 } else {
345 kref_get(ref);412 kref_get(ref);
346 }413 }
@@ -426,7 +493,9 @@ int hmdfs_file_release_remote(struct inode *inode, struct file *file)
426 hmdfs_remote_del_wr_opened_inode(info->conn, info);493 hmdfs_remote_del_wr_opened_inode(info->conn, info);
427 494 
428 inode_lock(inode);495 inode_lock(inode);
429- kref_put(&info->ref, hmdfs_do_close_remote);496+ if (kref_read(&info->ref) > 0)
497+ kref_put(&info->ref, hmdfs_do_close_remote);
498+ 
430 hmdfs_remote_keep_writecache(inode, file);499 hmdfs_remote_keep_writecache(inode, file);
431 inode_unlock(inode);500 inode_unlock(inode);
432 501 
@@ -33,7 +33,7 @@
33#define HMDFS_IOC_GET_DST_PATH _IOR(HMDFS_IOC, 3, __u32)33#define HMDFS_IOC_GET_DST_PATH _IOR(HMDFS_IOC, 3, __u32)
34 34 
35 35 
36-#define HMDFS_PAGE_SIZE 409636+ 
37#define HMDFS_PAGE_OFFSET 1237#define HMDFS_PAGE_OFFSET 12
38 38 
39/* max xattr value size, not include '\0' */39/* max xattr value size, not include '\0' */
@@ -148,6 +148,7 @@ struct hmdfs_sb_info {
148 struct list_head server_cache;148 struct list_head server_cache;
149 struct list_head to_delete;149 struct list_head to_delete;
150 struct mutex cache_list_lock;150 struct mutex cache_list_lock;
151+ bool small_file_optimization;
151 152 
152 /* local operation time statistic */153 /* local operation time statistic */
153 struct server_statistic *s_server_statis;154 struct server_statistic *s_server_statis;
@@ -29,8 +29,31 @@ static inline void free_sm_outbuf(struct hmdfs_send_command *sm)
29 sm->out_buf = NULL;29 sm->out_buf = NULL;
30}30}
31 31 
32+static struct hmdfs_time_t msec_to_timespec(unsigned int msec)
33+{
34+ struct hmdfs_time_t timespec = {
35+ .tv_sec = msec / MSEC_PER_SEC,
36+ .tv_nsec = (msec % MSEC_PER_SEC) * NSEC_PER_MSEC,
37+ };
38+ 
39+ return timespec;
40+}
41+ 
42+static struct hmdfs_time_t hmdfs_current_kernel_time(void)
43+{
44+ struct hmdfs_time_t time;
45+ 
46+#if KERNEL_VERSION(4, 18, 0) < LINUX_VERSION_CODE
47+ ktime_get_coarse_real_ts64(&time);
48+#else
49+ time = current_kernel_time();
50+#endif
51+ return time;
52+}
53+ 
32int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,54int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
33- __u8 file_type, struct hmdfs_open_ret *open_ret)55+ __u8 file_type, bool prefetch,
56+ struct hmdfs_open_ret *open_ret)
34{57{
35 int ret;58 int ret;
36 int path_len = strlen(send_buf);59 int path_len = strlen(send_buf);
@@ -44,6 +67,8 @@ int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
44 .local_filp = NULL,67 .local_filp = NULL,
45 };68 };
46 hmdfs_init_cmd(&sm.operations, F_OPEN);69 hmdfs_init_cmd(&sm.operations, F_OPEN);
70+ if (prefetch)
71+ hm_setprefetch(&sm.operations.reserved);
47 72 
48 if (!open_req) {73 if (!open_req) {
49 ret = -ENOMEM;74 ret = -ENOMEM;
@@ -69,6 +94,13 @@ int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
69 open_ret->remote_ctime.tv_nsec = le32_to_cpu(resp->ctime_nsec);94 open_ret->remote_ctime.tv_nsec = le32_to_cpu(resp->ctime_nsec);
70 open_ret->stable_ctime.tv_sec = le64_to_cpu(resp->stable_ctime);95 open_ret->stable_ctime.tv_sec = le64_to_cpu(resp->stable_ctime);
71 open_ret->stable_ctime.tv_nsec = le32_to_cpu(resp->stable_ctime_nsec);96 open_ret->stable_ctime.tv_nsec = le32_to_cpu(resp->stable_ctime_nsec);
97+ if (sm.out_len == sizeof(struct open_response)) {
98+ open_ret->read_success = resp->read_success;
99+ if (open_ret->read_success)
100+ memcpy(open_ret->page, resp->page, HMDFS_PAGE_SIZE);
101+ } else {
102+ open_ret->read_success = EOPNOTSUPP;
103+ }
72 104 
73out:105out:
74 free_sm_outbuf(&sm);106 free_sm_outbuf(&sm);
@@ -1047,6 +1079,53 @@ void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path)
1047 kfree(dp_req);1079 kfree(dp_req);
1048}1080}
1049 1081 
1082+void hmdfs_send_drop_page_push(struct hmdfs_peer *con, const char *send_buf, struct inode *inode)
1083+{
1084+ int path_len = strlen(send_buf);
1085+ size_t send_len = sizeof(struct drop_page_push_request) + path_len + 1;
1086+ struct drop_page_push_request *dpp_req = kzalloc(send_len, GFP_KERNEL);
lijiawei
lijiaweilijiawei7月3日

[Major] hmdfs_send_drop_page_push() 中 kzalloc 的 NULL 检查在使用 dpp_req 之后\n\nfs/hmdfs/hmdfs_client.c:1086 - struct drop_page_push_request *dpp_req = kzalloc(send_len, GFP_KERNEL);\n\n代码顺序为:\n1. dpp_req = kzalloc(send_len, GFP_KERNEL) — 分配内存\n2. 初始化 sm 结构体,其中 sm.data = dpp_req — 使用了可能为 NULL 的指针\n3. hmdfs_init_cmd(&sm.operations, F_DROP_PAGE_PUSH) — 初始化命令\n4. if (!dpp_req) return — 才检查 NULL\n\n如果 kzalloc 返回 NULL,步骤 2 中 sm.data = dpp_req 会将 NULL 赋给 sm.data。应将 NULL 检查提前到 kzalloc 之后、使用 dpp_req 之前。\n\n建议:将 if (!dpp_req) return 移到 sm 初始化之前。

likedislike
zengjy
7月10日 评论:
1087+ if (!dpp_req) return;
1088+ 
1089+ struct hmdfs_send_command sm = {
1090+ .data = dpp_req,
1091+ .len = send_len,
1092+ .out_buf = NULL,
1093+ .local_filp = NULL,
1094+ };
1095+ 
1096+ struct hmdfs_time_t current_time = hmdfs_current_kernel_time();
1097+ struct hmdfs_time_t ctime = inode->__i_ctime;
1098+ struct hmdfs_time_t precision =
1099+ msec_to_timespec(con->sbi->dcache_precision);
1100+ 
1101+ loff_t size = i_size_read(inode);
1102+ 
1103+ hmdfs_init_cmd(&sm.operations, F_DROP_PAGE_PUSH);
1104+ 
1105+ dpp_req->file_size = cpu_to_le64(size);
1106+ dpp_req->ctime = cpu_to_le64(ctime.tv_sec);
1107+ dpp_req->ctime_nsec = cpu_to_le32(ctime.tv_nsec);
1108+ 
1109+ precision = hmdfs_time_add(ctime, precision);
1110+ if (hmdfs_time_compare(&current_time, &ctime) < 0) {
1111+ dpp_req->stable_ctime = cpu_to_le64(0);
1112+ dpp_req->stable_ctime_nsec = cpu_to_le32(0);
1113+ } else if (hmdfs_time_compare(&current_time, &ctime) >= 0 &&
1114+ hmdfs_time_compare(&current_time, &precision) < 0) {
1115+ dpp_req->stable_ctime = dpp_req->ctime;
1116+ dpp_req->stable_ctime_nsec = dpp_req->ctime_nsec;
1117+ } else {
1118+ dpp_req->stable_ctime = cpu_to_le64(precision.tv_sec);
1119+ dpp_req->stable_ctime_nsec = cpu_to_le32(precision.tv_nsec);
1120+ }
1121+ 
1122+ dpp_req->path_len = cpu_to_le32(path_len);
1123+ strscpy(dpp_req->buf, send_buf, send_len - sizeof(struct drop_page_push_request));
1124+ 
1125+ hmdfs_sendmessage_request(con, &sm);
1126+ kfree(dpp_req);
1127+}
1128+ 
1050static void *hmdfs_get_msg_next(struct hmdfs_peer *peer, int *id)1129static void *hmdfs_get_msg_next(struct hmdfs_peer *peer, int *id)
1051{1130{
1052 struct hmdfs_msg_idr_head *head = NULL;1131 struct hmdfs_msg_idr_head *head = NULL;
@@ -18,6 +18,8 @@ struct hmdfs_open_ret {
18 __u64 ino;18 __u64 ino;
19 struct hmdfs_time_t remote_ctime;19 struct hmdfs_time_t remote_ctime;
20 struct hmdfs_time_t stable_ctime;20 struct hmdfs_time_t stable_ctime;
21+ __u8 read_success;
22+ __u8 *page;
21};23};
22 24 
23struct hmdfs_writepage_context {25struct hmdfs_writepage_context {
@@ -90,7 +92,8 @@ void hmdfs_client_writepage_done(struct hmdfs_inode_info *info,
90 struct hmdfs_writepage_context *ctx);92 struct hmdfs_writepage_context *ctx);
91 93 
92int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,94int hmdfs_send_open(struct hmdfs_peer *con, const char *send_buf,
93- __u8 file_type, struct hmdfs_open_ret *open_ret);95+ __u8 file_type, bool prefetch,
96+ struct hmdfs_open_ret *open_ret);
94void hmdfs_send_close(struct hmdfs_peer *con, const struct hmdfs_fid *fid);97void hmdfs_send_close(struct hmdfs_peer *con, const struct hmdfs_fid *fid);
95int hmdfs_send_fsync(struct hmdfs_peer *con, const struct hmdfs_fid *fid,98int hmdfs_send_fsync(struct hmdfs_peer *con, const struct hmdfs_fid *fid,
96 __s64 start, __s64 end, __s32 datasync);99 __s64 start, __s64 end, __s32 datasync);
@@ -2364,6 +2364,38 @@ void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path)
2364 path_put(&root_path);2364 path_put(&root_path);
2365}2365}
2366 2366 
2367+void hmdfs_add_remote_page_cache_list(struct hmdfs_peer *con, struct file *file)
2368+{
2369+ struct remotecache_item *item = NULL;
2370+ struct remotecache_item *item_temp = NULL;
2371+ struct hmdfs_dentry_info *d_info = NULL;
2372+ 
2373+ d_info = hmdfs_d(file->f_path.dentry);
2374+ if (!d_info)
2375+ return;
2376+ 
2377+ /* find duplicate con */
2378+ mutex_lock(&d_info->remote_cache_list_lock);
2379+ list_for_each_entry_safe(item, item_temp,
2380+ &(d_info->remote_cache_list_head), list) {
2381+ if (item->con->device_id == con->device_id) {
2382+ mutex_unlock(&d_info->remote_cache_list_lock);
2383+ return;
2384+ }
2385+ }
2386+ 
2387+ item = kzalloc(sizeof(*item), GFP_KERNEL);
2388+ if (!item) {
2389+ mutex_unlock(&d_info->remote_cache_list_lock);
2390+ return;
2391+ }
2392+ 
2393+ item->con = con;
2394+ item->drop_flag = 0;
2395+ list_add(&(item->list), &(d_info->remote_cache_list_head));
2396+ mutex_unlock(&d_info->remote_cache_list_lock);
2397+}
2398+ 
2367int hmdfs_drop_remote_cache_dents(struct dentry *dentry)2399int hmdfs_drop_remote_cache_dents(struct dentry *dentry)
2368{2400{
2369 struct path lower_path;2401 struct path lower_path;
@@ -2420,6 +2452,46 @@ int hmdfs_drop_remote_cache_dents(struct dentry *dentry)
2420 return 0;2452 return 0;
2421}2453}
2422 2454 
2455+int hmdfs_drop_remote_cache_pages(struct dentry *dentry)
2456+{
2457+ struct remotecache_item *item = NULL;
2458+ struct remotecache_item *item_temp = NULL;
2459+ struct hmdfs_dentry_info *d_info = NULL;
2460+ char *relative_path = NULL;
2461+ 
2462+ if (!dentry) {
2463+ hmdfs_err("dentry null and return");
2464+ return 0;
2465+ }
2466+ 
2467+ d_info = hmdfs_d(dentry);
2468+ if (!d_info) {
2469+ hmdfs_err("d_info null and return");
2470+ return 0;
2471+ }
2472+ relative_path = hmdfs_get_dentry_relative_path(dentry);
2473+ if (!relative_path) {
2474+ hmdfs_err("get dentry relative path failed");
2475+ return 0;
2476+ }
2477+ mutex_lock(&d_info->remote_cache_list_lock);
2478+ list_for_each_entry_safe(item, item_temp,
2479+ &(d_info->remote_cache_list_head), list) {
2480+ if (item->drop_flag == 1) {
2481+ item->drop_flag = 0;
2482+ continue;
2483+ }
2484+ hmdfs_send_drop_page_push(item->con, relative_path,
2485+ dentry->d_inode);
2486+ list_del(&item->list);
2487+ kfree(item);
2488+ }
2489+ mutex_unlock(&d_info->remote_cache_list_lock);
2490+ 
2491+ kfree(relative_path);
2492+ return 0;
2493+}
2494+ 
2423/* Clear the dentry cache files of target directory */2495/* Clear the dentry cache files of target directory */
2424int hmdfs_clear_cache_dents(struct dentry *dentry, bool remove_cache)2496int hmdfs_clear_cache_dents(struct dentry *dentry, bool remove_cache)
2425{2497{
@@ -211,6 +211,7 @@ struct clearcache_item {
211};211};
212 212 
213void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path);213void hmdfs_add_remote_cache_list(struct hmdfs_peer *con, const char *dir_path);
214+void hmdfs_add_remote_page_cache_list(struct hmdfs_peer *con, struct file *file);
214 215 
215struct remotecache_item {216struct remotecache_item {
216 struct hmdfs_peer *con;217 struct hmdfs_peer *con;
@@ -245,7 +246,9 @@ struct cache_file_callback {
245};246};
246 247 
247int hmdfs_drop_remote_cache_dents(struct dentry *dentry);248int hmdfs_drop_remote_cache_dents(struct dentry *dentry);
249+int hmdfs_drop_remote_cache_pages(struct dentry *dentry);
248void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path);250void hmdfs_send_drop_push(struct hmdfs_peer *con, const char *path);
251+void hmdfs_send_drop_page_push(struct hmdfs_peer *con, const char *send_buf, struct inode *inode);
249void hmdfs_mark_drop_flag(uint64_t device_id, struct dentry *dentry);252void hmdfs_mark_drop_flag(uint64_t device_id, struct dentry *dentry);
250void hmdfs_clear_drop_flag(struct dentry *dentry);253void hmdfs_clear_drop_flag(struct dentry *dentry);
251void delete_in_cache_file(uint64_t dev_id, struct dentry *dentry);254void delete_in_cache_file(uint64_t dev_id, struct dentry *dentry);
@@ -128,7 +128,9 @@ loff_t hmdfs_file_llseek_local(struct file *file, loff_t offset, int whence);
128 128 
129ssize_t hmdfs_do_read_iter(struct file *file, struct iov_iter *iter,129ssize_t hmdfs_do_read_iter(struct file *file, struct iov_iter *iter,
130 loff_t *ppos);130 loff_t *ppos);
131-ssize_t hmdfs_do_write_iter(struct file *file, struct iov_iter *iter,131+ssize_t hmdfs_do_write_iter_merge(struct file *file, struct iov_iter *iter,
132+ loff_t *ppos);
133+ssize_t hmdfs_do_write_iter_local(struct file *file, struct iov_iter *iter,
132 loff_t *ppos);134 loff_t *ppos);
133 135 
134int hmdfs_file_release_local(struct inode *inode, struct file *file);136int hmdfs_file_release_local(struct inode *inode, struct file *file);
@@ -460,7 +460,10 @@ static void hmdfs_update_open_response(struct hmdfs_peer *con,
460 msec_to_timespec(con->sbi->dcache_precision);460 msec_to_timespec(con->sbi->dcache_precision);
461 loff_t size = info->stat_valid ? info->stat.size :461 loff_t size = info->stat_valid ? info->stat.size :
462 i_size_read(info->inode);462 i_size_read(info->inode);
463+ loff_t pos = 0;
464+ ssize_t readsize = 0;
463 465 
466+ // NB: info->real_ino is the lower_file's i_ino and i_igeneration, not the info->file's
464 resp->ino = cpu_to_le64(info->real_ino);467 resp->ino = cpu_to_le64(info->real_ino);
465 resp->file_ver = cpu_to_le64(hmdfs_server_pack_fid_ver(con, cmd));468 resp->file_ver = cpu_to_le64(hmdfs_server_pack_fid_ver(con, cmd));
466 resp->file_id = cpu_to_le32(info->file_id);469 resp->file_id = cpu_to_le32(info->file_id);
@@ -488,6 +491,17 @@ static void hmdfs_update_open_response(struct hmdfs_peer *con,
488 resp->stable_ctime = cpu_to_le64(precision.tv_sec);491 resp->stable_ctime = cpu_to_le64(precision.tv_sec);
489 resp->stable_ctime_nsec = cpu_to_le32(precision.tv_nsec);492 resp->stable_ctime_nsec = cpu_to_le32(precision.tv_nsec);
490 }493 }
494+ 
495+ if (hm_isprefetch(cmd->operations.reserved)) {
oh_create_jiawei

【一般】未看到仅使能小文件的特性规格限制,且未看到1KB规格定义的选型理由,建议补充

likedislike
oh_create_jiawei
oh_create_jiawei
6月7日 评论:
zengjy
6月10日 评论:
496+ readsize = kernel_read(info->file, resp->page, HMDFS_PAGE_SIZE, &pos);
oh_create_jiawei

【一般】从open语义出发,当前open中增加了读行为,语义发生了转变,接口平均耗时的性能影响是否有针对性测试,同时对应用的兼容性行为是否进行了充分评估,是否能保证应用行为不发生任何非预期的异常

likedislike
zengjy
6月10日 评论:
497+ if (readsize < 0) {
498+ resp->read_success = 0;
499+ } else {
500+ resp->read_success = 1;
oh_create_jiawei

【一般】将读行为提前,是对整体所有应用的使用均带来了改变,是否有考虑对整机IO层面,CPU层面,器件功耗层面等的影响,在工程应用层面,open是比read更热的应用执行路径,热路径下的行为变更,需要考虑对整个设备的影响

likedislike
zengjy
6月10日 评论:
501+ if (readsize != HMDFS_PAGE_SIZE)
502+ memset(resp->page + readsize, 0, HMDFS_PAGE_SIZE - readsize);
503+ }
504+ }
491}505}
492 506 
493static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,507static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,
@@ -534,66 +548,6 @@ static int hmdfs_get_open_info(struct hmdfs_peer *con, uint8_t file_type,
534 return 0;548 return 0;
535}549}
536 550 
537-void hmdfs_server_open(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
538- void *data)
539-{
540- struct open_request *recv = data;
541- int sizeread = sizeof(struct open_response);
542- struct open_response *resp = NULL;
543- struct hmdfs_open_info *info = NULL;
544- int ret = 0;
545- 
546- trace_hmdfs_server_open_enter(con, recv);
547- 
548- resp = kzalloc(sizeread, GFP_KERNEL);
549- info = kmalloc(sizeof(*info), GFP_KERNEL);
550- if (!resp || !info) {
551- ret = -ENOMEM;
552- goto err_free;
553- }
554- 
555- if (path_contain_dotdot(recv->buf, recv->path_len)) {
556- ret = -EINVAL;
557- goto err_free;
558- }
559- 
560- info->file = hmdfs_open_file(con, recv->buf, recv->file_type,
561- &info->file_id);
562- if (IS_ERR(info->file)) {
563- ret = PTR_ERR(info->file);
564- goto err_free;
565- }
566- 
567- ret = hmdfs_get_open_info(con, recv->file_type, recv->buf, info);
568- if (ret)
569- goto err_close;
570- 
571- hmdfs_update_open_response(con, cmd, info, resp);
572- 
573- trace_hmdfs_server_open_exit(con, resp, info->file, 0);
574- ret = hmdfs_sendmessage_response(con, cmd, sizeread, resp, 0);
575- if (ret) {
576- hmdfs_err("sending msg response failed, file_id %d, err %d",
577- info->file_id, ret);
578- remove_file_from_conn(con, info->file_id);
579- hmdfs_close_path(info->file);
580- }
581- hmdfs_close_path(info->file);
582- kfree(resp);
583- kfree(info);
584- return;
585- 
586-err_close:
587- hmdfs_close_path(info->file);
588- remove_file_from_conn(con, info->file_id);
589- hmdfs_close_path(info->file);
590-err_free:
591- kfree(resp);
592- kfree(info);
593- trace_hmdfs_server_open_exit(con, NULL, NULL, ret);
594- hmdfs_send_err_response(con, cmd, ret);
595-}
596- 
597static int hmdfs_check_and_create(struct path *path_parent,551static int hmdfs_check_and_create(struct path *path_parent,
598 struct dentry *dentry, uint64_t device_id,552 struct dentry *dentry, uint64_t device_id,
599 umode_t mode, bool is_excl)553 umode_t mode, bool is_excl)
@@ -1087,6 +1041,73 @@ static char *server_lookup_lower(struct hmdfs_peer *peer, const char *req_path,
1087 return err ? ERR_PTR(err) : lo_p_name;1041 return err ? ERR_PTR(err) : lo_p_name;
1088}1042}
1089 1043 
1044+void hmdfs_server_open(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1045+ void *data)
1046+{
1047+ struct open_request *recv = data;
1048+ bool is_prefetch = hm_isprefetch(cmd->operations.reserved);
1049+ int sizeread = sizeof(struct open_response) -
1050+ (is_prefetch ? 0 : HMDFS_PAGE_SIZE + 1);
1051+ struct open_response *resp = NULL;
1052+ struct hmdfs_open_info *info = NULL;
1053+ int ret = 0;
1054+ 
1055+ trace_hmdfs_server_open_enter(con, recv);
1056+ 
1057+ resp = kzalloc(sizeread, GFP_KERNEL);
1058+ info = kmalloc(sizeof(*info), GFP_KERNEL);
1059+ if (!resp || !info) {
1060+ ret = -ENOMEM;
1061+ goto err_free;
1062+ }
1063+ 
1064+ if (path_contain_dotdot(recv->buf, recv->path_len)) {
1065+ ret = -EINVAL;
1066+ goto err_free;
1067+ }
1068+ 
1069+ info->file = hmdfs_open_file(con, recv->buf, recv->file_type,
1070+ &info->file_id);
1071+ 
1072+ if (IS_ERR(info->file)) {
1073+ ret = PTR_ERR(info->file);
1074+ goto err_free;
1075+ }
1076+ 
1077+ ret = hmdfs_get_open_info(con, recv->file_type, recv->buf, info);
1078+ if (ret)
1079+ goto err_close;
1080+ 
1081+ hmdfs_update_open_response(con, cmd, info, resp);
1082+ 
1083+ trace_hmdfs_server_open_exit(con, resp, info->file, 0);
oh_create_jiawei

【建议】流程还没执行完,trace_exit的原因是什么?

likedislike
zengjy
6月10日 评论:
1084+ ret = hmdfs_sendmessage_response(con, cmd, sizeread, resp, 0);
1085+ if (ret) {
1086+ hmdfs_err("sending msg response failed, file_id %d, err %d",
1087+ info->file_id, ret);
1088+ remove_file_from_conn(con, info->file_id);
1089+ goto out;
1090+ }
1091+ 
1092+ if (is_prefetch)
1093+ hmdfs_add_remote_page_cache_list(con, info->file);
1094+ 
1095+out:
1096+ hmdfs_close_path(info->file);
1097+ kfree(resp);
1098+ kfree(info);
1099+ return;
1100+ 
1101+err_close:
1102+ hmdfs_close_path(info->file);
1103+ remove_file_from_conn(con, info->file_id);
lijiawei
lijiaweilijiawei7月3日

[Major] hmdfs_server_open() 的 err_close 标签中 hmdfs_close_path(info->file) 被调用了两次\n\nfs/hmdfs/hmdfs_server.c:1103 - hmdfs_close_path(info->file);\n\n在重构后的 hmdfs_server_open() 中,err_close 标签下的代码:\nc\nerr_close:\n\thmdfs_close_path(info->file);\n\tremove_file_from_conn(con, info->file_id);\n\thmdfs_close_path(info->file);\n\nhmdfs_close_path(info->file) 被连续调用了两次,对同一个 file 做两次 close 操作是错误的,可能导致 double free 或引用计数异常。这个问题是从旧代码直接搬过来的,建议在此次重构中一并修复。\n\n应删除其中一个 hmdfs_close_path(info->file) 调用。

likedislike
zengjy
7月10日 评论:
1104+err_free:
1105+ kfree(resp);
1106+ kfree(info);
1107+ trace_hmdfs_server_open_exit(con, NULL, NULL, ret);
1108+ hmdfs_send_err_response(con, cmd, ret);
1109+}
1110+ 
1090void hmdfs_server_readdir(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,1111void hmdfs_server_readdir(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1091 void *data)1112 void *data)
1092{1113{
@@ -1441,6 +1462,7 @@ static bool hmdfs_filldir_real(struct dir_context *ctx, const char *name,
1441 }1462 }
1442 1463 
1443 if (d_type == DT_REG || d_type == DT_DIR) {1464 if (d_type == DT_REG || d_type == DT_DIR) {
1465+ // gc->file is dentry_file, the anonymous tmp_file
1444 create_dentry(child, d_inode(child), gc->file, gc->sbi);1466 create_dentry(child, d_inode(child), gc->file, gc->sbi);
1445 gc->num++;1467 gc->num++;
1446 } else if (d_type == DT_LNK) {1468 } else if (d_type == DT_LNK) {
@@ -1500,12 +1522,14 @@ struct file *hmdfs_server_rebuild_dents(struct hmdfs_sb_info *sbi,
1500 struct file *dentry_file = NULL;1522 struct file *dentry_file = NULL;
1501 struct hmdfs_dcache_header header;1523 struct hmdfs_dcache_header header;
1502 1524 
1525+ // create a anonymous tmp_file in sbi->cache_dir
1503 dentry_file = create_local_dentry_file_cache(sbi);1526 dentry_file = create_local_dentry_file_cache(sbi);
1504 if (IS_ERR(dentry_file)) {1527 if (IS_ERR(dentry_file)) {
1505 hmdfs_err("file create failed err=%ld", PTR_ERR(dentry_file));1528 hmdfs_err("file create failed err=%ld", PTR_ERR(dentry_file));
1506 return dentry_file;1529 return dentry_file;
1507 }1530 }
1508 1531 
1532+ // path is the lower path of the target dir
1509 file = dentry_open(path, O_RDONLY | O_DIRECTORY, current_cred());1533 file = dentry_open(path, O_RDONLY | O_DIRECTORY, current_cred());
1510 if (IS_ERR(file)) {1534 if (IS_ERR(file)) {
1511 err = PTR_ERR(file);1535 err = PTR_ERR(file);
@@ -1556,6 +1580,10 @@ void hmdfs_server_writepage(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1556 ssize_t ret;1580 ssize_t ret;
1557 int err = 0;1581 int err = 0;
1558 1582 
1583+ struct remotecache_item *item = NULL;
1584+ struct remotecache_item *item_temp = NULL;
1585+ struct hmdfs_dentry_info *d_info = NULL;
1586+ 
1559 file_id = le32_to_cpu(writepage_recv->file_id);1587 file_id = le32_to_cpu(writepage_recv->file_id);
1560 file_ver = le64_to_cpu(writepage_recv->file_ver);1588 file_ver = le64_to_cpu(writepage_recv->file_ver);
1561 file = get_file_by_fid_and_ver(con, cmd, file_id, file_ver);1589 file = get_file_by_fid_and_ver(con, cmd, file_id, file_ver);
@@ -1567,12 +1595,24 @@ void hmdfs_server_writepage(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
1567 err = PTR_ERR(file);1595 err = PTR_ERR(file);
1568 goto out;1596 goto out;
1569 }1597 }
1598+ d_info = hmdfs_d(file->f_path.dentry);
1570 1599 
1571 pos = (loff_t)le64_to_cpu(writepage_recv->index) << HMDFS_PAGE_OFFSET;1600 pos = (loff_t)le64_to_cpu(writepage_recv->index) << HMDFS_PAGE_OFFSET;
1572 count = le32_to_cpu(writepage_recv->count);1601 count = le32_to_cpu(writepage_recv->count);
1573 ret = kernel_write(file, writepage_recv->buf, count, &pos);1602 ret = kernel_write(file, writepage_recv->buf, count, &pos);
1574 if (ret != count)1603 if (ret != count)
1575 err = -EIO;1604 err = -EIO;
1605+ else if (d_info) {
1606+ mutex_lock(&d_info->remote_cache_list_lock);
1607+ list_for_each_entry_safe(item, item_temp,
1608+ &(d_info->remote_cache_list_head), list) {
1609+ if (item->con->device_id == con->device_id) {
1610+ item->drop_flag = 1;
1611+ break;
1612+ }
1613+ }
1614+ mutex_unlock(&d_info->remote_cache_list_lock);
1615+ }
1576 1616 
1577 hmdfs_close_path(file);1617 hmdfs_close_path(file);
1578out:1618out:
@@ -2123,3 +2163,88 @@ void hmdfs_server_get_drop_push(struct hmdfs_peer *con,
2123quickack:2163quickack:
2124 set_conn_sock_quickack(con);2164 set_conn_sock_quickack(con);
2125}2165}
2166+ 
2167+static struct drop_page_push_recv init_drop_page_push_recv(void *data)
2168+{
2169+ struct drop_page_push_request *dpp_recv = data;
2170+ struct drop_page_push_recv dppr = {
2171+ .file_size = le64_to_cpu(dpp_recv->file_size),
2172+ .path_len = le32_to_cpu(dpp_recv->path_len),
2173+ .buf = dpp_recv->buf,
2174+ };
2175+ dppr.remote_ctime.tv_sec = le64_to_cpu(dpp_recv->ctime);
2176+ dppr.remote_ctime.tv_nsec = le32_to_cpu(dpp_recv->ctime_nsec);
2177+ dppr.stable_ctime.tv_sec = le64_to_cpu(dpp_recv->stable_ctime);
lijiawei
lijiaweilijiawei7月3日

[Critical] init_drop_page_push_recv() 中 ctime_nsec 使用了错误的字节序转换函数\n\nfs/hmdfs/hmdfs_server.c:2178 - dppr.remote_ctime.tv_nsec = le64_to_cpu(dpp_recv->ctime_nsec);\n\n在 drop_page_push_request 结构体(protocol.h)中,ctime_nsecstable_ctime_nsec 定义为 __le32,但此处使用了 le64_to_cpu() 进行转换。__le32 字段应使用 le32_to_cpu()。使用 le64_to_cpu() 会读取 8 字节(越界读取),且转换结果不正确,可能导致纳秒值被错误地左移或读取到相邻字段的数据。\n\n同样的问题存在于 stable_ctime_nsec 的转换(第2180行)。应改为 le32_to_cpu()

likedislike
zengjy
7月10日 评论:
2178+ dppr.stable_ctime.tv_nsec = le32_to_cpu(dpp_recv->stable_ctime_nsec);
2179+ 
2180+ return dppr;
2181+}
2182+ 
2183+void hmdfs_server_get_drop_page_push(struct hmdfs_peer *con,
2184+ struct hmdfs_head_cmd *cmd, void *data)
2185+{
2186+ struct drop_page_push_recv dpp_recv = init_drop_page_push_recv(data);
2187+ struct path root_path, path;
2188+ struct inode *inode;
2189+ struct hmdfs_inode_info *info;
2190+ char *tmp_path = NULL;
2191+ int err;
2192+ 
2193+ // Learn from hmdfs_server_get_drop_push
2194+ const char *root_name = con->sbi->real_dst;
2195+ if (path_contain_dotdot(dpp_recv.buf, dpp_recv.path_len)) {
2196+ err = -EINVAL;
2197+ goto quickack;
2198+ }
2199+ 
2200+ err = kern_path(root_name, 0, &root_path);
2201+ if (err) {
2202+ hmdfs_err("kern_path failed at %s, err = %d", root_name, err);
2203+ goto quickack;
2204+ }
2205+ 
2206+ tmp_path = kzalloc(PATH_MAX, GFP_KERNEL);
2207+ if (!tmp_path)
2208+ goto out_kfree;
2209+ snprintf(tmp_path, PATH_MAX, "/" DEVICE_VIEW_ROOT "/%s%s", con->cid,
2210+ dpp_recv.buf);
2211+ 
2212+ err = vfs_path_lookup(root_path.dentry, root_path.mnt, tmp_path, 0,
2213+ &path);
2214+ if (err) {
2215+ hmdfs_info("path found failed at %s, err = %d", dpp_recv.buf, err);
2216+ goto out_tfree;
2217+ }
2218+ 
2219+ inode = d_inode(path.dentry);
2220+ info = hmdfs_i(inode);
2221+ 
2222+ inode_lock(inode);
2223+ 
2224+ /*
2225+ * It doesn't make sense to update metadata here but leaving the data obsolete.
2226+ * But how to design here? Should we also transport the data here?
2227+ * Transporting data means we may never open the file twice,
2228+ * which may corrupt file reference counter in server when the file is not opened but the page cache is missing.
2229+ */
2230+ // inode->i_ctime = dpp_recv.remote_ctime;
2231+ // info->remote_ctime = dpp_recv.remote_ctime;
2232+ // info->stable_ctime = dpp_recv.stable_ctime;
2233+ 
2234+ // i_size_write(inode, dpp_recv.file_size);
2235+ 
2236+ info->need_reopen = true;
2237+ 
2238+ inode_unlock(inode);
2239+ 
2240+ path_put(&path);
2241+ 
2242+out_tfree:
2243+ kfree(tmp_path);
2244+ 
2245+out_kfree:
2246+ path_put(&root_path);
2247+ 
2248+quickack:
2249+ set_conn_sock_quickack(con);
2250+}
@@ -20,6 +20,14 @@
20#define DATA_SEC_LEVEL_LABEL "user.security"20#define DATA_SEC_LEVEL_LABEL "user.security"
21#define DATA_SEC_LEVEL_LENGTH 1021#define DATA_SEC_LEVEL_LENGTH 10
22 22 
23+struct drop_page_push_recv {
24+ __u64 file_size;
25+ struct hmdfs_time_t remote_ctime;
26+ struct hmdfs_time_t stable_ctime;
27+ __u32 path_len;
28+ char *buf;
29+};
30+ 
23static inline void hmdfs_send_err_response(struct hmdfs_peer *con,31static inline void hmdfs_send_err_response(struct hmdfs_peer *con,
24 struct hmdfs_head_cmd *cmd, int err)32 struct hmdfs_head_cmd *cmd, int err)
25{33{
@@ -74,6 +82,8 @@ void hmdfs_server_listxattr(struct hmdfs_peer *con, struct hmdfs_head_cmd *cmd,
74 void *data);82 void *data);
75void hmdfs_server_get_drop_push(struct hmdfs_peer *con,83void hmdfs_server_get_drop_push(struct hmdfs_peer *con,
76 struct hmdfs_head_cmd *cmd, void *data);84 struct hmdfs_head_cmd *cmd, void *data);
85+void hmdfs_server_get_drop_page_push(struct hmdfs_peer *con,
86+ struct hmdfs_head_cmd *cmd, void *data);
77 87 
78void __init hmdfs_server_add_node_evt_cb(void);88void __init hmdfs_server_add_node_evt_cb(void);
79#endif89#endif
@@ -69,6 +69,9 @@ struct hmdfs_inode_info {
69 struct inode *lower_inode; // for local/merge inode69 struct inode *lower_inode; // for local/merge inode
70 struct hmdfs_peer *conn; // for remote inode70 struct hmdfs_peer *conn; // for remote inode
71 struct kref ref;71 struct kref ref;
72+ bool need_reopen;
oh_create_jiawei

【建议】是否有对整机内存底噪的性能影响测试

likedislike
zengjy
6月10日 评论:
73+ spinlock_t modify_lock;
74+ bool modified;
72 spinlock_t fid_lock;75 spinlock_t fid_lock;
73 struct hmdfs_fid fid;76 struct hmdfs_fid fid;
74 unsigned long fid_flags;77 unsigned long fid_flags;
@@ -314,6 +314,9 @@ static struct inode *hmdfs_alloc_inode(struct super_block *sb)
314 if (!gi)314 if (!gi)
315 return NULL;315 return NULL;
316 memset(gi, 0, offsetof(struct hmdfs_inode_info, vfs_inode));316 memset(gi, 0, offsetof(struct hmdfs_inode_info, vfs_inode));
317+ gi->need_reopen = true;
318+ gi->modified = false;
319+ spin_lock_init(&gi->modify_lock);
317 INIT_LIST_HEAD(&gi->wb_list);320 INIT_LIST_HEAD(&gi->wb_list);
318 init_rwsem(&gi->wpage_sem);321 init_rwsem(&gi->wpage_sem);
319 gi->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;322 gi->getattr_isize = HMDFS_STALE_REMOTE_ISIZE;
@@ -697,6 +700,7 @@ static void hmdfs_init_cmd_timeout(struct hmdfs_sb_info *sbi)
697 set_cmd_timeout(sbi, F_STATFS, TIMEOUT_COMMON);700 set_cmd_timeout(sbi, F_STATFS, TIMEOUT_COMMON);
698 set_cmd_timeout(sbi, F_CONNECT_REKEY, TIMEOUT_NONE);701 set_cmd_timeout(sbi, F_CONNECT_REKEY, TIMEOUT_NONE);
699 set_cmd_timeout(sbi, F_DROP_PUSH, TIMEOUT_NONE);702 set_cmd_timeout(sbi, F_DROP_PUSH, TIMEOUT_NONE);
703+ set_cmd_timeout(sbi, F_DROP_PAGE_PUSH, TIMEOUT_NONE);
700 set_cmd_timeout(sbi, F_GETATTR, TIMEOUT_COMMON);704 set_cmd_timeout(sbi, F_GETATTR, TIMEOUT_COMMON);
701 set_cmd_timeout(sbi, F_FSYNC, TIMEOUT_90S);705 set_cmd_timeout(sbi, F_FSYNC, TIMEOUT_90S);
702 set_cmd_timeout(sbi, F_SYNCFS, TIMEOUT_30S);706 set_cmd_timeout(sbi, F_SYNCFS, TIMEOUT_30S);
@@ -746,6 +750,7 @@ static int hmdfs_init_sbi(struct hmdfs_sb_info *sbi)
746 sbi->dcache_precision = DEFAULT_DCACHE_PRECISION;750 sbi->dcache_precision = DEFAULT_DCACHE_PRECISION;
747 sbi->dcache_timeout = DEFAULT_DCACHE_TIMEOUT;751 sbi->dcache_timeout = DEFAULT_DCACHE_TIMEOUT;
748 sbi->write_cache_timeout = DEFAULT_WRITE_CACHE_TIMEOUT;752 sbi->write_cache_timeout = DEFAULT_WRITE_CACHE_TIMEOUT;
753+ sbi->small_file_optimization = true;
749 hmdfs_init_cmd_timeout(sbi);754 hmdfs_init_cmd_timeout(sbi);
750 sbi->async_cb_delay = HMDFS_NODE_EVT_CB_DELAY;755 sbi->async_cb_delay = HMDFS_NODE_EVT_CB_DELAY;
751 sbi->async_req_max_active = DEFAULT_SRV_REQ_MAX_ACTIVE;756 sbi->async_req_max_active = DEFAULT_SRV_REQ_MAX_ACTIVE;